added dask and hermes support

.github/workflows/run_tests.yml

@@ -0,0 +1,71 @@
+name: test
+
+on:
+  push:
+    branches:
+      - "main"
+      - "master"
+      - "dev"
+    paths:
+      - "**.py"
+      - ".github/workflows/*test*.yml"
+      - "pyproject.toml"
+      - "poetry.lock"
+      - "requirements/requirements*.txt"
+      - "requirements/requirements-dask.txt"
+  pull_request:
+    paths:
+      - "**.py"
+      - ".github/workflows/*test*.yml"
+      - "pyproject.toml"
+      - "poetry.lock"
+      - "requirements/requirements*.txt"
+      - "requirements/requirements-dask.txt"
+
+jobs:
+  test:
+    name: "test Python ${{ matrix.python }} on ${{ matrix.os }} with Dask ${{ matrix.dask }}"
+    runs-on: "${{ matrix.os }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        python: ["3.10", "3.11"]
+        dask: [true, false]
+        include:
+          - os: "macos-latest"
+            python-version: "3.10"
+            dask: false
+    defaults:
+      run:
+        shell: "bash -elo pipefail {0}"
+
+    steps:
+      - uses: "actions/checkout@main"
+      - uses: "actions/setup-python@main"
+        with:
+          python-version: "${{ matrix.python }}"
+      - uses: "actions/cache@main"
+        id: "cache"
+        with:
+          path: "${{ env.pythonLocation }}"
+          key: "test-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml', 'requirements/*') }}"
+      - run: "python --version"
+      - run: "python -mpip install -U pip"
+      - run: "python -mpip --version"
+      - run: "python -mpip install -r requirements/requirements.txt"
+      - name: "Install Dask requirements"
+        if: matrix.dask
+        run: "python -mpip install -r requirements/requirements-dask.txt"
+      - run: "python -mpip install ./"
+      - run: "python -mpip cache info"
+      - run: "python -mpip freeze"
+      - run: "pytest --version"
+      - run: "mypy ./observer"
+        if: "(matrix.os == 'ubuntu-latest') && (matrix.python == '3.11' )"
+      - name: "Run tests with Dask"
+        if: matrix.dask
+        run: "pytest"
+      - name: "Run tests without Dask"
+        if: "!matrix.dask"
+        run: "pytest"

pyproject.toml

@@ -8,6 +8,8 @@ authors = ["tomsail <[email protected]>"]
 python = ">=3.9"
 xarray = "*"
 numpy = "*"
+scipy = "*"
+dask = { version = "*", optional = true }
 
 [tool.poetry.plugins."xarray.backends"]
 selafin = "xarray_selafin_backend.xarray_backend:SelafinBackendEntrypoint"

requirements/requirements-dask.txt

@@ -0,0 +1,36 @@
+argcomplete==3.1.4
+certifi==2024.2.2
+cftime==1.6.3
+click==8.1.7
+cloudpickle==3.0.0
+contourpy==1.2.0
+cycler==0.12.1
+dask==2024.2.1
+fonttools==4.49.0
+fsspec==2024.2.0
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+kiwisolver==1.4.5
+locket==1.0.0
+matplotlib==3.8.3
+netCDF4==1.6.5
+numpy==1.26.4
+packaging==23.2
+pandas==2.2.0
+partd==1.4.1
+pillow==10.2.0
+pipx==1.2.1
+pluggy==1.4.0
+pyparsing==3.1.1
+pytest==8.0.1
+python-dateutil==2.8.2
+pytz==2024.1
+PyYAML==6.0.1
+scipy==1.12.0
+six==1.16.0
+toolz==0.12.1
+tzdata==2024.1
+userpath==1.9.1
+xarray==2024.2.0
+-e git+https://github.com/seareport/xarray-selafin.git@473c1bb40a6849921df313255c2eef405589f590#egg=xarray_selafin_backend
+zipp==3.17.0

requirements/requirements.txt

@@ -0,0 +1,27 @@
+argcomplete==3.1.4
+certifi==2024.2.2
+cftime==1.6.3
+contourpy==1.2.0
+cycler==0.12.1
+fonttools==4.49.0
+iniconfig==2.0.0
+kiwisolver==1.4.5
+matplotlib==3.8.3
+netCDF4==1.6.5
+numpy==1.26.4
+packaging==23.2
+pandas==2.2.0
+pillow==10.2.0
+pipx==1.2.1
+pluggy==1.4.0
+pyparsing==3.1.1
+pytest==8.0.1
+python-dateutil==2.8.2
+pytz==2024.1
+scipy==1.12.0
+six==1.16.0
+tzdata==2024.1
+userpath==1.9.1
+xarray==2024.2.0
+-e git+https://github.com/seareport/xarray-selafin.git@473c1bb40a6849921df313255c2eef405589f590#egg=xarray_selafin_backend
+zipp==3.17.0

tests/io_test.py

@@ -1,6 +1,12 @@
 import pytest
 import xarray as xr
+import numpy as np
 
+try:
+    import dask.array as da
+    DASK_AVAILABLE = True
+except ImportError:
+    DASK_AVAILABLE = False
 
 TIDAL_FLATS = pytest.mark.parametrize(
     "slf_in",
@@ -27,3 +33,30 @@ def test_to_netcdf(tmp_path, slf_in):
     ds_slf.to_netcdf(nc_out)
     ds_nc = xr.open_dataset(nc_out)
     assert ds_nc.equals(ds_slf)
+
+
+@TIDAL_FLATS
+def test_to_selafin(tmp_path, slf_in):
+    ds_slf = xr.open_dataset(slf_in, engine="selafin")
+    slf_out = tmp_path / "test.slf"
+    ds_slf.selafin.write(slf_out)
+    ds_slf2 = xr.open_dataset(slf_out)
+    assert ds_slf2.equals(ds_slf)
+
+
+@TIDAL_FLATS
+def test_slice(tmp_path, slf_in):
+    # simple slice
+    ds_slf = xr.open_dataset(slf_in, engine="selafin")
+    nc_out = tmp_path / "test.nc"
+    ds_slice = ds_slf.isel(time=slice(0, 10))
+    ds_slice.to_netcdf(nc_out)
+    ds_nc = xr.open_dataset(nc_out)
+    assert ds_nc.equals(ds_slice)
+    # multiple slice
+    ds_slf = xr.open_dataset(slf_in, engine="selafin")
+    nc_out = tmp_path / "test2.nc"
+    ds_slice = ds_slf.isel(time=slice(0, 10), plan=0)
+    ds_slice.to_netcdf(nc_out)
+    ds_nc = xr.open_dataset(nc_out)
+    assert ds_nc.equals(ds_slice)

tests/perf_test.py

@@ -0,0 +1,39 @@
+import time
+from xarray_selafin_backend.selafin import Selafin
+from xarray_selafin_backend.telemac_file import TelemacFile
+import pytest
+
+PERF = pytest.mark.parametrize(
+    "f",
+    [
+        pytest.param("tests/data/r3d_tidal_flats.slf", id="3D"),
+        pytest.param("tests/data/r2d_tidal_flats.slf", id="2D"),
+    ],
+)
+
+def timer(func):
+    def wrapper(*args, **kwargs):
+        start = time.time()
+        result = func(*args, **kwargs)
+        end = time.time()
+        print(f"Time taken by {func.__name__}: {end - start} seconds")
+        return result
+    return wrapper
+
+# selafin
+@timer
+@PERF 
+def selafin(f):
+    slf = Selafin(f)
+    slf.get_series([10])
+
+# telemacFile
+@timer
+@PERF 
+def telemac(f):
+    tel = TelemacFile(f)
+    tel.get_data_value(tel.varnames[0], 10)
+
+
+telemac("tests/data/r3d_tidal_flats.slf")
+selafin("tests/data/r3d_tidal_flats.slf")