[1pt] PR: Change Post proc perms updates

docs/CHANGELOG.md

@@ -2,6 +2,25 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.5.4.4 - 2024-08-02 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
+
+Prior to this fix, fim_post_processing.sh took just under 4 hours to reset permissions on all files and folder under the entire run. On closer inspection, it was updating permissions for all HUC folders where were already correct. A few other folders needed to have permission updates added. This will speed that up significantly.
+
+Also, use this opportunity to added a new note to hash_compare.py and fix an annoying duration time showing milliseconds.
+
+### Changes
+- `fim_pipeline.sh`: fix duration msgs.
+- `fim_post_processing.sh`:  permissions reset fix, a bit of output cleanup and fix duration msgs.
+- `src`
+    - `bash_functions.env`: update the Calc duration to allow for a msg prefix to be added to the duration calcs. Also adjusted the duration message to show hours as well, previously only min and seconds.
+    - `run_by_branch.sh`: fix duration msgs.
+    - `run_unit_wb.sh`: fix duration msgs.
+    - `src\src_adjust_ras2fim_rating.py`: minor duration display msg change.
+- `tools\hash_compare.py`: Added note
+
+ <br/><br/>
+
+
 ## v4.5.4.3 - 2024-08-02 - [PR#1136](https://github.com/NOAA-OWP/inundation-mapping/pull/1136)
 
 Levee-protected areas are associated with levelpaths based on a 1000 m buffer on each side of the levee line. However, not all levees are designed to protect against all associated levelpaths, especially where the levelpath flows through the levee-protected area. Levee-protected areas are unmasked by removing levelpaths from association that don't intersect levees but instead flow around them which allows inundation by these branches.

fim_pipeline.sh

@@ -115,7 +115,7 @@ fi
 echo
 echo "---- Unit (HUC) processing is complete"
 date -u
-Calc_Duration $pipeline_start_time
+Calc_Duration "Duration : " $pipeline_start_time
 echo "---------------------------------------------------"
 
 ## POST PROCESSING
@@ -130,8 +130,7 @@ echo
 
 echo "======================== End of fim_pipeline for $runName =========="
 date -u
-echo "Total Duration is ..."
-Calc_Duration $pipeline_start_time
+Calc_Duration "Total Duration is ... " $pipeline_start_time
 echo
 
 # Exit the script

fim_post_processing.sh

@@ -92,6 +92,7 @@ l_echo ""
 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 l_echo "---- Start of fim_post_processing"
 l_echo "---- Started: `date -u`"
+echo ""
 T_total_start
 post_proc_start_time=`date +%s`
 
@@ -134,10 +135,10 @@ if [ "$COUNTER" -gt 1 ]; then
     # Execute the Python file
     l_echo "Updating hydroTable & scr_full_crosswalked for branches"
     python3 $srcDir/update_htable_src.py -d $outputDestDir
+    Tcount
 else
     l_echo "Execution count is $COUNTER, not executing the update_htable_src.py file."
 fi
-Tcount
 
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
@@ -207,7 +208,6 @@ if [ "$src_adjust_usgs" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [ "$s
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ RAS2FIM CROSS SECTION RATING CURVES ##
@@ -221,7 +221,6 @@ if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## RUN SYNTHETIC RATING CURVE CALIBRATION W/ BENCHMARK POINTS (.parquet files) ##
@@ -230,11 +229,11 @@ if [ "$src_adjust_spatial" = "True" ] && [ "$src_subdiv_toggle" = "True" ]  && [
     l_echo $startDiv"Performing SRC adjustments using benchmark point .parquet files"
     python3 $srcDir/src_adjust_spatial_obs.py -fim_dir $outputDestDir -j $jobLimit
     Tcount
-    date -u
 fi
 
 ## AGGREGATE BRANCH TABLES ##
 l_echo $startDiv"Aggregating branch hydrotables"
+
 Tstart
 python3 $srcDir/aggregate_by_huc.py \
     -fim $outputDestDir \
@@ -243,7 +242,7 @@ python3 $srcDir/aggregate_by_huc.py \
     -bridge \
     -j $jobLimit
 Tcount
-date -u
+
 
 ## PERFORM MANUAL CALIBRATION
 if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
@@ -253,7 +252,6 @@ if [ "$manual_calb_toggle" = "True" ] && [ -f $man_calb_file ]; then
         -fim_dir $outputDestDir \
         -calb_file $man_calb_file
     Tcount
-    date -u
 fi
 
 
@@ -263,12 +261,13 @@ python3 $toolsDir/combine_crosswalk_tables.py \
     -d $outputDestDir \
     -o $outputDestDir/crosswalk_table.csv
 Tcount
-date -u
 
 
 l_echo $startDiv"Resetting Permissions"
 Tstart
-    find $outputDestDir -type d -exec chmod -R 777 {} +
+    find $outputDestDir/logs/ -type d -exec chmod -R 777 {} +
+    find $outputDestDir/branch_errors/ -type d -exec chmod -R 777 {} +
+    find $outputDestDir/unit_errors/ -type d -exec chmod -R 777 {} +
     find $outputDestDir -type f -exec chmod 777 {} +  # just root level files
 Tcount
 
@@ -290,5 +289,5 @@ echo
 l_echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
 l_echo "---- End of fim_post_processing"
 l_echo "---- Ended: `date -u`"
-Calc_Duration $post_proc_start_time
+Calc_Duration "Post Processing Duration:" $post_proc_start_time
 echo

src/bash_functions.env

@@ -68,15 +68,26 @@ Tcount () {
 }
 
 # This expects a start time submitted as an arg and assumes the end time as now
-Calc_Duration() {
-    local start_time=$1
+Calc_Duration( ) {
+    local duration_msg_prefix=$1
+    local start_time=$2
     local end_time=`date +%s`
 
     local total_sec=$(( $end_time - $start_time ))
-    local dur_min=$((total_sec / 60))
-    local dur_remainder_sec=$((total_sec % 60))
+    local dur_hrs=$((total_sec / 3600 ))
+    local dur_min=$((total_sec %3600 / 60))
+    local dur_remainder_sec=$((total_sec %60))
+
+    if [[ "$dur_hrs" = "0" ]]; then
+        dur_hrs=""
+    else
+        dur_hrs="${dur_hrs} hrs"
+    fi
+
+#    local msg="${duration_msg_prefix} $dur_min min(s) and $dur_remainder_sec sec"
+    local msg="${duration_msg_prefix} ${dur_hrs} $dur_min min(s) and $dur_remainder_sec seconds"
+
 
-    local msg="Duration : $dur_min min(s) and $dur_remainder_sec sec"
     # if an arg comes in and is a file path, we can display and record to a file
     if [[ $log_file_path != "" ]]; then
         echo -e $msg ; echo -e $msg >> $log_file_path

src/run_by_branch.sh

@@ -128,5 +128,5 @@ fi
 
 echo -e $startDiv"End Branch Processing $hucNumber $current_branch_id ..."
 date -u
-Calc_Duration $branch_start_time
+Calc_Duration "Duration : " $branch_start_time
 echo

src/run_unit_wb.sh

@@ -305,14 +305,14 @@ if [ -f $deny_unit_list ]; then
 fi
 
 echo "---- HUC $hucNumber - branches have now been processed"
-Calc_Duration $branch_processing_start_time
-echo
+Calc_Duration "Duration for processing branches : " $branch_processing_start_time
+#echo
 
 # WRITE TO LOG FILE CONTAINING ALL HUC PROCESSING TIMES
 total_duration_display="$hucNumber,$(Calc_Time $huc_start_time),$(Calc_Time_Minutes_in_Percent $huc_start_time)"
 echo "$total_duration_display" >> "$outputDestDir/logs/unit/total_duration_run_by_unit_all_HUCs.csv"
 
 date -u
 echo "---- HUC processing for $hucNumber is complete"
-Calc_Duration $huc_start_time
+Calc_Duration "Duration for huc processing: " $huc_start_time
 echo

src/src_adjust_ras2fim_rating.py

@@ -57,7 +57,8 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
     )  # , nrows=30000)
     ras_rc_df.rename(columns={'fid_xs': 'location_id'}, inplace=True)
     # ras_rc_df['location_id'] = ras_rc_df['feature_id'].astype(object)
-    print('Duration (read ras_rc_csv): {}'.format(dt.datetime.now() - start_time))
+    run_time = dt.datetime.now() - start_time
+    print(f"Duration (read ras_rc_csv): {str(run_time).split('.')[0]}")
 
     # convert WSE navd88 values to meters
     ras_rc_df.rename(

tools/hash_compare.py

@@ -20,6 +20,10 @@ def main(arg1, arg2, image_only, log_file, gpkg):
     Please be advised that this feature has not been thoroughly tested to ensure robustness.
 
     When arg1 and arg2 are single files, the files names need not match but the extensions must match.
+
+    Note: If any file has a date/time field, it will likely never be successful on a hash compare. This
+    is seen often in gpkgs.
+
     """
 
     if log_file is not None: