merge in dev

.gitattributes

@@ -1,2 +1,3 @@
 # Set the default behavior for end of lines.
 * text eol=lf
+docs/CHANGELOG.md merge=union

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,6 +8,16 @@
 
 ### Testing
 
+### Deployment Plan (For developer use)
+
+_How does the changes affect the product?_
+- [ ] Code only?
+- [ ] Require new or adjusted data inputs? Does it have start, end and duration code (in UTC)?
+- [ ] If new or updated data sets, has the FIM code been updated and tested with the new/adjusted data (subset is fine, but must be a subset of the new data)?
+- [ ] Require new pre-clip set?
+- [ ] Has new or updated python packages?
+- [ ] If applicable, has a deployment plan be created with the deployment person/team?
+
 
 ### Issuer Checklist (For developer use)
 

.github/workflows/lint_and_format.yaml

@@ -0,0 +1,22 @@
+name: Lint and Format Using Pre-Commit
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  lint-and-format:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version-file: pyproject.toml
+    - uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -6,29 +6,26 @@ repos:
     rev: v4.4.0
     hooks:
         -   id: trailing-whitespace
-            exclude: CHANGELOG.md
+            # Below is python regex to exclude all .md files
+            exclude: .*md$
         -   id: end-of-file-fixer
-            exclude: Pipfile.lock
+            exclude:  Pipfile.lock
         -   id: check-added-large-files
             args: ['--maxkb=5000']
         -   id: check-json
+
 -   repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
         -   id: flake8
-            entry: pflake8
-            additional_dependencies: [pyproject-flake8]
+            additional_dependencies: [flake8-pyproject]
+
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 24.3.0
     hooks:
         -   id: black
             args: ['--line-length=110', '--skip-magic-trailing-comma', '--skip-string-normalization']
-            exclude: |
-                (?x)^(
-                    src/add_crosswalk.py|
-                    src/bathymetric_adjustment.py|
-                    src/identify_src_bankfull.py
-                )$
+
 -   repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:

CITATION.cff

@@ -4,5 +4,5 @@ authors:
 - family-names: "NOAA Office of Water Prediction"
 title: "Inundation Mapping"
 url: "https://github.com/NOAA-OWP/inundation-mapping"
-version: 4.4.1.0
+version: 4.4.2.1
 date-released: 2023

CONTRIBUTING.md

@@ -3,7 +3,7 @@
 > All contributions to this project will be released to the public domain.
 > By submitting a pull request or filing a bug, issue, or
 > feature request, you are agreeing to comply with this waiver of copyright interest.
-> Details can be found in our [TERMS](TERMS.md) and [LICENSE](LICENSE).
+> Details can be found in our [LICENSE](LICENSE).
 
 
 There are two primary ways to help:
@@ -45,46 +45,92 @@ If you would like to contribute, please follow these steps:
     ```
     git checkout -b <dev-your-bugfix-or-feature>
     ```
-4. [pre-commit](https://pre-commit.com/) is used to run auto-formatting and linting tools. Please follow the steps in the link provided to install `pre-commit` locally.  `pre-commit` can be installed as a git hook to verify your staged changes adhere to the project's style and format requirements (configuration defined in [pyproject.toml](/pyproject.toml)).
-5. After you've installed `pre-commit` via `pip` , `homebrew`, or `conda`, check the version to verify installation:
-    ```
-    $ pre-commit --version
-    ```
-
-6. Initialize the pre-commit hooks included within the root directory of `inundation-mapping`:
-
-    ```
-    $ pre-commit install
-    ```
-7. At this point, you should be set up with `pre-commit`. When a commit is made it will run the pre-commit hooks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). For reference, you may run any of the pre-commit hooks manually before issuing the `git commit` command (see below). Some tools used by the pre commit git hook (`isort`, `flake8`, & `black`) are also available as command line executables within the Docker container, however, it is recommend to run them through `pre-commit` outside of the container, as it picks up the correct configuration.
 
+4. Build the Docker container (if not already done in the [README.md](./README.md) page.):
     ```
-    # Check only the staged changes
-    pre-commit run
-
-    # Check all files in the repo
-    pre-commit run -a
-
-    # Run only the black formatting tool
-    pre-commit run -a flake8
+    Docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>
     ```
 
-8. Build the Docker container:
+5. Pre-commit installation:
+
+   [pre-commit](https://pre-commit.com/) is used to run auto-formatting and enforce styling rules.
+   It is a critical part of development and is enforced at the 'git commit' step. Key tools are included **inside the docker container** if you want to execute the correctly configured linting and formatting command line executables there. If you intend to execute `flake8`, `black` or `isort` from the command line **outside of the docker container**, additional configuration and installation is required, which will not be described here.
+
+   **Note: These steps below are similar to another required critical step (pre-commit configuration) later in this document, which also needs to be run**.
+
+   If pre-commit is not already installed on your system:
+   ```
+   pip install pre-commit
+   ```
+   All related tools (git hook scripts) are installed under the `pre-commit install` step, not at this level. See https://pre-commit.com/#install
+
+   If you get an error message during the install of pre-commit which says:
+
+   *Installing collected packages: pre-commit
+       WARNING: The script pre-commit is installed in '/home/{your_user_name}/.local/bin' which is not on PATH.
+       Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.`,
+   then you will need to do some additional configuration. You need to adjust your path.*
+   ```
+   (Adjusting the path to be exactly the path you see in the WARNING message above from your console output).
+   export PATH="/home/{your_user_name}/.local/bin:$PATH"
+   ```
+   To test that it installed correctly, is pathed correctly and check the version:
+   ```
+   pre-commit --version
+   ```
+   It should respond with the phrase *pre-commit 3.6.0* (version may not be exact).
+
+
+6. pre-commit configuration:
+
+   Now, you need to configure your local clone of the repo to honor the pre-commit hooks.
+   The `pre-commit` package is used to pick up the pre-commit hooks which verify your staged changes adhere to the project's style and format requirements (configuration defined in [pyproject.toml](/pyproject.toml)).
+
+   Initialize the pre-commit hooks included within the root directory of this code repository (`inundation-mapping`):
     ```
-    Docker build -f Dockerfile -t <image_name>:<tag> <path/to/repository>
+    $ pre-commit install
     ```
-9. [Within the container](README.md#startrun-the-docker-container), ensure sure unit tests pass ([instructions here](/unit_tests/README.md)).
+
+7. At this point, you should be set up with `pre-commit`. When a commit is made it will run the pre-commit hooks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). For reference, you may run any of the pre-commit hooks manually before issuing the `git commit` command (see below). Some tools used by the pre commit git hook (`isort`, `flake8`, & `black`) are also available as command line executables **within the Docker container***, however, it is recommended to run them through `pre-commit` **outside of the container**, as it picks up the correct configuration.
+
+   ```
+   There are multiple ways to run pre-commit tests. Here are three:
+   
+   # Check only the staged changes
+   pre-commit run
+
+   # Check all files in the repo
+   pre-commit run -a
+
+   # Run only the isort, black, flake8 (in order).
+   pre-commit run -a isort
+   pre-commit run -a black  (If it fails the first time, run it again and now apply a fix if required)
+   pre-commit run -a flake8
+   ```
+    **Note: The pre-commit system can be a bit tricky. If you have trouble with pre-commit steps above, you have another option.** Some environments may require add the word **sudo** in front commands .
+   ```
+   # Open up a docker container
+   cd /foss_fim
+   isort .
+   black .
+   flake8 .
+
+   # optionally close the container
+   # Back on your terminal console (outside the container), use the typical git add, git commit, git push
+
+ 
+8.  [Within the container](README.md#startrun-the-docker-container), ensure sure unit tests pass ([instructions here](/unit_tests/README.md)).
     ```
     pytest unit_tests/
     ```
 
-10. Outside of the Docker container, commit your changes:
+9. Outside of the Docker container, commit your changes:
     ```
-    git commit -m <descriptive sentence or two changes>
+    git commit -m "<descriptive sentence or two of changes>"
     ```
-    This will invoke pre-commit hooks mentioned in step 7 that will lint & format the code. In many cases non-compliant code will be rectified automatically, but in some cases manual changes will be necessary. Make sure all of these checks pass, if not, make necessary changes and re-issue `git commit -m <...>`.
-
-9. Push to your forked branch:
+    This will invoke pre-commit hooks mentioned in step 6 that will lint & format the code (some others as well). In many cases non-compliant code will be rectified automatically, but in some cases manual changes will be necessary. Make sure all of these checks pass. If not, make necessary changes (`git add <...>`), and re-issue `git commit -m "<...>"`.
+   
+10. Push to your forked branch:
     ```
     git push -u origin
     ```
@@ -93,4 +139,5 @@ If you would like to contribute, please follow these steps:
     git push --set-upstream origin <your branch>
     ```
 
-10. Submit a pull request on [inundation-mapping's GitHub page](https://github.com/NOAA-OWP/inundation-mapping) (please review checklist in [PR template](/.github/PULL_REQUEST_TEMPLATE.md) for additional PR guidance).
+11. Submit a pull request on [inundation-mapping's GitHub page](https://github.com/NOAA-OWP/inundation-mapping) (please review checklist in [PR template](/.github/PULL_REQUEST_TEMPLATE.md) for additional PR guidance).
+   

Dockerfile

@@ -1,5 +1,5 @@
 ## Temporary image to build the libraries and only save the needed artifacts
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.4.3 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0 AS builder
 WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
@@ -9,6 +9,9 @@ ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
+# remove reference to missing repo
+RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+
 RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/*
 
 RUN git clone https://github.com/dtarb/taudem.git
@@ -36,24 +39,20 @@ RUN mkdir -p $taudemDir
 RUN mkdir -p $taudemDir2
 
 ## Move needed binaries to the next stage of the image
-RUN cd taudem/bin && mv -t $taudemDir flowdircond aread8 threshold streamnet gagewatershed catchhydrogeo dinfdistdown
+RUN cd taudem/bin && mv -t $taudemDir flowdircond streamnet gagewatershed catchhydrogeo dinfdistdown
 RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d8flowdir dinfflowdir
 
 
-
-
 ###############################################################################################
-
-
-
 # Base Image that has GDAL, PROJ, etc
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.4.3
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0
 ARG dataDir=/data
 ENV projectDir=/foss_fim
 ARG depDir=/dependencies
 ENV inputsDir=$dataDir/inputs
 ENV outputsDir=/outputs
 ENV srcDir=$projectDir/src
+ENV toolsDir=$projectDir/tools
 ENV workDir=/fim_temp
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
@@ -70,8 +69,13 @@ RUN mkdir -p $workDir
 RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
+# remove reference to missing repo
+RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+
+RUN apt-get update --fix-missing && apt-get install -y openjdk-19-jdk && rm -rf /var/lib/apt/lists/*
+
 RUN apt update --fix-missing
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync tzdata
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata
 
 RUN apt auto-remove
 
@@ -93,7 +97,7 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/unit_tests:$projectDir/tools
 
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
+RUN pip3 install pipenv==2023.12.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
 
 # ----------------------------------
 # Mar 2023
@@ -106,11 +110,11 @@ RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off pipenv install --syste
 # We download and unzip it to the same file folder that pip deployed the whitebox library.
 # Whitebox also attempts to always download a folder called testdata regardless of use.
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
-RUN wbox_path=/usr/local/lib/python3.8/dist-packages/whitebox/ && \
-    wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
-    unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
-    cp $wbox_path/WBT/whitebox_tools $wbox_path && \
-    mkdir $wbox_path/testdata
+# RUN wbox_path=/usr/local/lib/python3.10/dist-packages/whitebox/WBT && \
+#     wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
+#     unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
+#     cp $wbox_path/whitebox_tools $wbox_path && \
+#     mkdir $wbox_path/testdata
 # ----------------------------------
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##

Pipfile

@@ -4,45 +4,54 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-ipython = "==8.12.0"
+ipython = "==8.24.0"
 
 [packages]
 certifi = "==2023.7.22"
 fiona = "==1.8.22"
-geopandas = "==0.12.2"
+geopandas = "==0.14.3"
 numba = "==0.56.4"
 numpy = "==1.23.5"
 pandas = "==2.0.2"
 rasterio = "==1.3.6"
 rasterstats = "==0.18.0"
 richdem = "==0.3.4"
-tqdm = "==4.65.0"
+tqdm = "==4.66.3"
 seaborn = "==0.12.2"
 python-dotenv = "==1.0.0"
 natsort = "==8.3.1"
-memory-profiler = "==0.61.0"
 xarray = "==2023.1.0"
 netcdf4 = "==1.6.3"
 tables = "==3.8.0"
 pyproj = "==3.5.0"
 psycopg2-binary = "==2.9.6"
 boto3 = "==1.26.109"
 jupyter = "==1.0.0"
-jupyterlab = "==3.6.3"
+jupyterlab = "==3.6.7"
 ipympl = "==0.9.3"
 pytest = "==7.3.0"
-whitebox = "==2.3.1"
+whitebox = "2.3.4"
 shapely = "==2.0.1"
-pyarrow = "==11.0.0"
+pyarrow = "==14.0.1"
 rtree = "==1.0.1"
 py7zr = "==0.20.4"
 scipy = "==1.10.1"
-gval = "==0.1.2"
+gval = "==0.2.3"
 flake8 = "==6.0.0"
-black = "==23.7.0"
-pyproject-flake8 = "==6.0.0.post1"
+black = "==24.3.0"
+flake8-pyproject = "==1.2.3"
 pre-commit = "==3.3.3"
 isort = "==5.12.0"
+urllib3 = "==1.26.18"
+pyflwdir = "==0.5.8"
+pillow = "==10.3.0"
+pyogrio = "==0.7.2"
+openpyxl = "==3.1.2"
+osmnx = "==1.9.3"
+lmoments3 = "==1.0.6"
+zarr = "==2.18.0"
+requests = "==2.32.3"
+aiohttp = "==3.9.5"
 
 [requires]
-python_version = "3.8"
+python_version = "3.10"