From ab66f8bbaef4ff4b1f2c070db539f051850fc9ae Mon Sep 17 00:00:00 2001
From: RobHanna-NOAA <robert.hanna@noaa.gov>
Date: Wed, 31 Jul 2024 16:51:37 +0000
Subject: [PATCH 1/6] change Post proc perms updates

---
 fim_post_processing.sh           | 4 +++-
 src/src_adjust_ras2fim_rating.py | 5 +++--
 tools/hash_compare.py            | 4 ++++
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index e17850cb9..b2e3b35c9 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -268,7 +268,9 @@ date -u
 
 l_echo $startDiv"Resetting Permissions"
 Tstart
-    find $outputDestDir -type d -exec chmod -R 777 {} +
+    find $outputDestDir/logs/ -type d -exec chmod -R 777 {} +
+    find $outputDestDir/branch_errors/ -type d -exec chmod -R 777 {} +
+    find $outputDestDir/unit_errors/ -type d -exec chmod -R 777 {} +
     find $outputDestDir -type f -exec chmod 777 {} +  # just root level files
 Tcount
 
diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
index 22a47b33e..a5ead6c59 100644
--- a/src/src_adjust_ras2fim_rating.py
+++ b/src/src_adjust_ras2fim_rating.py
@@ -48,7 +48,7 @@
 
 
 def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir):
-    start_time = dt.datetime.now()
+    start_time = dt.datetime.now(dt.timezone.utc)
     print('Reading RAS2FIM rating curves from csv...')
     log_text = 'Processing database for RAS2FIM flow/WSE at NWM flow recur intervals...\n'
     col_filter = ["fid_xs", "flow", "wse"]
@@ -57,7 +57,8 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
     )  # , nrows=30000)
     ras_rc_df.rename(columns={'fid_xs': 'location_id'}, inplace=True)
     # ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
-    print('Duration (read ras_rc_csv): {}'.format(dt.datetime.now() - start_time))
+    run_time = dt.datetime.now(dt.timezone.utc) - start_time
+    print(f"Duration (read ras_rc_csv): {str(run_time).split('.')[0]}")
 
     # convert WSE navd88 values to meters
     ras_rc_df.rename(
diff --git a/tools/hash_compare.py b/tools/hash_compare.py
index 33c2dfad1..be23ade07 100755
--- a/tools/hash_compare.py
+++ b/tools/hash_compare.py
@@ -20,6 +20,10 @@ def main(arg1, arg2, image_only, log_file, gpkg):
     Please be advised that this feature has not been thoroughly tested to ensure robustness.
 
     When arg1 and arg2 are single files, the files names need not match but the extensions must match.
+
+    Note: If any file has a date/time field, it will likely never be successful on a hash compare. This
+    is seen often in gpkgs.
+
     """
 
     if log_file is not None:

From 0494b6e6974fa819a80565d2094c885d98b4f6b6 Mon Sep 17 00:00:00 2001
From: RobHanna-NOAA <robert.hanna@noaa.gov>
Date: Wed, 31 Jul 2024 17:21:30 +0000
Subject: [PATCH 2/6] minor adj

---
 src/src_adjust_ras2fim_rating.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py
index a5ead6c59..832949e46 100644
--- a/src/src_adjust_ras2fim_rating.py
+++ b/src/src_adjust_ras2fim_rating.py
@@ -48,7 +48,7 @@
 
 
 def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_filepath, log_dir):
-    start_time = dt.datetime.now(dt.timezone.utc)
+    start_time = dt.datetime.now()
     print('Reading RAS2FIM rating curves from csv...')
     log_text = 'Processing database for RAS2FIM flow/WSE at NWM flow recur intervals...\n'
     col_filter = ["fid_xs", "flow", "wse"]
@@ -57,7 +57,7 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
     )  # , nrows=30000)
     ras_rc_df.rename(columns={'fid_xs': 'location_id'}, inplace=True)
     # ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
-    run_time = dt.datetime.now(dt.timezone.utc) - start_time
+    run_time = dt.datetime.now() - start_time
     print(f"Duration (read ras_rc_csv): {str(run_time).split('.')[0]}")
 
     # convert WSE navd88 values to meters

From fa68aea879e557057951948001fdee7654609ef6 Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Wed, 31 Jul 2024 11:30:32 -0600
Subject: [PATCH 3/6] Update CHANGELOG.md

---
 docs/CHANGELOG.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index f4bc51bee..56cddd9f5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,20 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.5.x.x - 2024-07-31 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
+
+Prior to this fix, fim_post_processing.sh took just under 4 hours to reset permissions on all files and folder under the entire run. On closer inspection, it was updating permissions for all HUC folders where were already correct. A few other folders needed to have permission updates added. This will speed that up significantly.
+
+Also, use this opportunity to added a new note to hash_compare.py and fix an annoying duration time showing milliseconds.
+
+### Changes
+- `fim_post_processing.sh`:  permissions reset fix
+- `src\src_adjust_ras2fim_rating.py`: minor duration display msg change.
+- `tools\hash_compare.py`: Added note
+
+ <br/><br/>
+
+
 ## v4.5.3.1 - 2024-07-24 - [PR#1233](https://github.com/NOAA-OWP/inundation-mapping/pull/1233)
 
 In a PR [1217](https://github.com/NOAA-OWP/inundation-mapping/pull/1217), which is about to be merged, it updates a bunch of python packages. One is numpy. This has triggered a very large amount of on-screen output from a new numpy warning while running `synthesize_test_cases.py`.

From d8e3588ce0a64af72aa0b80e62844a9ad4a96e81 Mon Sep 17 00:00:00 2001
From: RobHanna-NOAA <robert.hanna@noaa.gov>
Date: Thu, 1 Aug 2024 20:02:21 +0000
Subject: [PATCH 4/6] linting fix

---
 fim_pipeline.sh        |  5 ++---
 fim_post_processing.sh | 13 +++++--------
 src/bash_functions.env | 21 ++++++++++++++++-----
 src/run_by_branch.sh   |  2 +-
 src/run_unit_wb.sh     |  6 +++---
 5 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/fim_pipeline.sh b/fim_pipeline.sh
index 16139fdc7..3934c27a7 100755
--- a/fim_pipeline.sh
+++ b/fim_pipeline.sh
@@ -115,7 +115,7 @@ fi
 echo
 echo "---- Unit (HUC) processing is complete"
 date -u
-Calc_Duration $pipeline_start_time
+Calc_Duration "Duration : " $pipeline_start_time
 echo "---------------------------------------------------"
 
 ## POST PROCESSING
@@ -130,8 +130,7 @@ echo
 
 echo "======================== End of fim_pipeline for $runName =========="
 date -u
-echo "Total Duration is ..."
-Calc_Duration $pipeline_start_time
+Calc_Duration "Total Duration is ... " $pipeline_start_time
 echo
 
 # Exit the script
diff --git a/fim_post_processing.sh b/fim_post_processing.sh
index b2e3b35c9..6a58c075c 100755
--- a/fim_post_processing.sh
+++ b/fim_post_processing.sh
@@ -92,6 +92,7 @@ l_echo ""
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 l_echo "---- Start of fim_post_processing"
 l_echo "---- Started: `date -u`"
+echo ""
 T_total_start
 post_proc_start_time=`date +%s`
 
@@ -134,10 +135,10 @@ if [ "$COUNTER" -gt 1 ]; then
     # Execute the Python file
     l_echo "Updating hydroTable & scr_full_crosswalked for branches"
     python3 $srcDir/update_htable_src.py -d $outputDestDir
+    Tcount
 else
     l_echo "Execution count is $COUNTER, not executing the update_htable_src.py file."
 fi
-Tcount
 
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
@@ -207,7 +208,6 @@ if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$s
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ RAS2FIM CROSS SECTION RATING CURVES ##
@@ -221,7 +221,6 @@ if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINTS (.parquet files) ##
@@ -230,11 +229,11 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [
     l_echo $startDiv"Performing SRC adjustments using benchmark point .parquet files"
     python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputDestDir -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## AGGREGATE BRANCH TABLES ##
 l_echo $startDiv"Aggregating branch hydrotables"
+
 Tstart
 python3 $srcDir/aggregate_by_huc.py \
     -fim $outputDestDir \
@@ -243,7 +242,7 @@ python3 $srcDir/aggregate_by_huc.py \
     -bridge \
     -j $jobLimit
 Tcount
-date -u
+
 
 ## PERFORM MANUAL CALIBRATION
 if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
@@ -253,7 +252,6 @@ if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
         -fim_dir $outputDestDir \
         -calb_file $man_calb_file
     Tcount
-    date -u
 fi
 
 
@@ -263,7 +261,6 @@ python3 $toolsDir/combine_crosswalk_tables.py \
     -d $outputDestDir \
     -o $outputDestDir/crosswalk_table.csv
 Tcount
-date -u
 
 
 l_echo $startDiv"Resetting Permissions"
@@ -292,5 +289,5 @@ echo
 l_echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 l_echo "---- End of fim_post_processing"
 l_echo "---- Ended: `date -u`"
-Calc_Duration $post_proc_start_time
+Calc_Duration "Post Processing Duration:" $post_proc_start_time
 echo
diff --git a/src/bash_functions.env b/src/bash_functions.env
index e8452003a..961a46285 100644
--- a/src/bash_functions.env
+++ b/src/bash_functions.env
@@ -68,15 +68,26 @@ Tcount () {
 }
 
 # This expects a start time submitted as an arg and assumes the end time as now
-Calc_Duration() {
-    local start_time=$1
+Calc_Duration( ) {
+    local duration_msg_prefix=$1
+    local start_time=$2
     local end_time=`date +%s`
 
     local total_sec=$(( $end_time - $start_time ))
-    local dur_min=$((total_sec / 60))
-    local dur_remainder_sec=$((total_sec % 60))
+    local dur_hrs=$((total_sec / 3600 ))
+    local dur_min=$((total_sec %3600 / 60))
+    local dur_remainder_sec=$((total_sec %60))
+
+    if [[ "$dur_hrs" = "0" ]]; then
+        dur_hrs=""
+    else
+        dur_hrs="${dur_hrs} hrs"
+    fi
+
+#    local msg="${duration_msg_prefix} $dur_min min(s) and $dur_remainder_sec sec"
+    local msg="${duration_msg_prefix} ${dur_hrs} $dur_min min(s) and $dur_remainder_sec seconds"
+
 
-    local msg="Duration : $dur_min min(s) and $dur_remainder_sec sec"
     # if an arg comes in and is a file path, we can display and record to a file
     if [[ $log_file_path != "" ]]; then
         echo -e $msg ; echo -e $msg >> $log_file_path
diff --git a/src/run_by_branch.sh b/src/run_by_branch.sh
index 69151a74c..28977c2e0 100755
--- a/src/run_by_branch.sh
+++ b/src/run_by_branch.sh
@@ -128,5 +128,5 @@ fi
 
 echo -e $startDiv"End Branch Processing $hucNumber $current_branch_id ..."
 date -u
-Calc_Duration $branch_start_time
+Calc_Duration "Duration : " $branch_start_time
 echo
diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh
index 3c7a1b460..60c7a3812 100755
--- a/src/run_unit_wb.sh
+++ b/src/run_unit_wb.sh
@@ -305,8 +305,8 @@ if [ -f $deny_unit_list ]; then
 fi
 
 echo "---- HUC $hucNumber - branches have now been processed"
-Calc_Duration $branch_processing_start_time
-echo
+Calc_Duration "Duration for processing branches : " $branch_processing_start_time
+#echo
 
 # WRITE TO LOG FILE CONTAINING ALL HUC PROCESSING TIMES
 total_duration_display="$hucNumber,$(Calc_Time $huc_start_time),$(Calc_Time_Minutes_in_Percent $huc_start_time)"
@@ -314,5 +314,5 @@ echo "$total_duration_display" >> "$outputDestDir/logs/unit/total_duration_run_b
 
 date -u
 echo "---- HUC processing for $hucNumber is complete"
-Calc_Duration $huc_start_time
+Calc_Duration "Duration for huc processing: " $huc_start_time
 echo

From 1993fa17ef33453db706cad438e81f0dd3054b2b Mon Sep 17 00:00:00 2001
From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com>
Date: Thu, 1 Aug 2024 14:53:19 -0600
Subject: [PATCH 5/6] Update CHANGELOG.md

---
 docs/CHANGELOG.md | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 56cddd9f5..af30e733f 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -9,10 +9,14 @@ Prior to this fix, fim_post_processing.sh took just under 4 hours to reset permi
 Also, use this opportunity to added a new note to hash_compare.py and fix an annoying duration time showing milliseconds.
 
 ### Changes
-- `fim_post_processing.sh`:  permissions reset fix
-- `src\src_adjust_ras2fim_rating.py`: minor duration display msg change.
+- `fim_pipeline.sh`: fix duration msgs.
+- `fim_post_processing.sh`:  permissions reset fix, a bit of output cleanup and fix duration msgs.
+- `src`
+    - `bash_functions.env`: update the Calc duration to allow for a msg prefix to be added to the duration calcs. Also adjusted the duration message to show hours as well, previously only min and seconds.
+    - `run_by_branch.sh`: fix duration msgs.
+    - `run_unit_wb.sh`: fix duration msgs.
+    - `src\src_adjust_ras2fim_rating.py`: minor duration display msg change.
 - `tools\hash_compare.py`: Added note
-
  <br/><br/>
 
 

From c14baf6de6b69caa79e57ba31629a72f8d3db9f7 Mon Sep 17 00:00:00 2001
From: Rob Hanna <Robert.hanna@noaa.gov>
Date: Fri, 2 Aug 2024 19:37:48 +0000
Subject: [PATCH 6/6] update changelog

---
 docs/CHANGELOG.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4f71408ac..537a7d2ee 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
-## v4.5.x.x - 2024-07-31 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
+## v4.5.4.4 - 2024-08-02 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
 
 Prior to this fix, fim_post_processing.sh took just under 4 hours to reset permissions on all files and folder under the entire run. On closer inspection, it was updating permissions for all HUC folders where were already correct. A few other folders needed to have permission updates added. This will speed that up significantly.
 
@@ -17,6 +17,7 @@ Also, use this opportunity to added a new note to hash_compare.py and fix an ann
     - `run_unit_wb.sh`: fix duration msgs.
     - `src\src_adjust_ras2fim_rating.py`: minor duration display msg change.
 - `tools\hash_compare.py`: Added note
+
  <br/><br/>