Replaced manually generated interfaces with auto-converted from Nipype

.github/workflows/pythonpackage.yaml

@@ -36,7 +36,7 @@ jobs:
     - uses: actions/upload-artifact@v3
       with:
         name: converted-nipype
-        path: pydra/tasks/freesurfer/auto
+        path: src/pydra/tasks/freesurfer/auto
 
   devcheck:
     needs: [nipype-conv]
@@ -58,7 +58,10 @@ jobs:
       uses: actions/download-artifact@v3
       with:
         name: converted-nipype
-        path: pydra/tasks/freesurfer/auto
+        path: src/pydra/tasks/freesurfer/auto
+    - name: Strip auto package from gitignore so it is included in package
+      run: |
+        sed -i '/\/src\/pydra\/tasks\/freesurfer\/auto/d' .gitignore
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -68,22 +71,112 @@ jobs:
         python -m pip install --upgrade pip
     - name: Install Pydra
       run: |
+        pushd $HOME
         pip install ${{ matrix.pydra }}
+        popd
         python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
     - name: Install task package
       run: |
-        pip install ${{ matrix.pip-flags }} ".[dev]"
+        pip install "./src-fileformats[dev]" "./src-fileformats-extras[dev]"
+        pip install ${{ matrix.pip-flags }} "./src[dev]"
         python -c "import pydra.tasks.freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
         python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
+        python -c "import fileformats.medimage_freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
+        python -c "import fileformats.extras.medimage_freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
+
 
-  test:
-    needs: [nipype-conv]
+  fileformats-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.11']
+    steps:
+    - uses: actions/checkout@v3
+    - name: Revert version to most recent tag on upstream update
+      if: github.event_name == 'repository_dispatch'
+      run: git checkout $(git tag -l | tail -n 1 | awk -F post '{print $1}')
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install task package
+      run: |
+        pip install "./src-fileformats[test]"
+        python -c "import fileformats.medimage_freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
+    # - name: Test fileformats with pytest
+    #   run: |
+    #     pytest -sv --doctest-modules ./src-fileformats/fileformats/medimage_freesurfer \
+    #         --cov fileformats.medimage_freesurfer --cov-report xml
+
+  fileformats-extras-test:
+    needs: ["fileformats-test"]
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ['3.8', '3.11']
+    steps:
+    - uses: actions/checkout@v3
+    - name: Revert version to most recent tag on upstream update
+      if: github.event_name == 'repository_dispatch'
+      run: git checkout $(git tag -l | tail -n 1 | awk -F post '{print $1}')
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install task package
+      run: |
+        pip install "./src-fileformats" "./src-fileformats-extras[test]"
+        python -c "import fileformats.extras.medimage_freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
+    # - name: Test fileformats extras with pytest
+    #   run: |
+    #     pytest -sv --doctest-modules ./src-fileformats-extras/fileformats/medimage_freesurfer \
+    #         --cov fileformats.extras.medimage_freesurfer --cov-report xml  
 
+  test:
+    needs: [nipype-conv, fileformats-test, fileformats-extras-test]
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.8']  # '3.11'
     steps:
+    - name: Removed unnecessary tools to free space
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"      
+    - name: Get Download cache Key
+      id: cache-key
+      run: echo "::set-output name=key::freesurfer-linux-ubuntu22_amd64-7.4.1"
+    - name: Cache FreeSurfer
+      uses: actions/cache@v2
+      with:
+        path: $HOME/downloads/freesurfer
+        key: ${{ steps.cache-key.outputs.key }}
+        restore-keys: |
+          freesurfer-linux-ubuntu22_amd64-7.4.1
+    - name: Download FreeSurfer
+      if: steps.cache-key.outputs.key != steps.cache-hit.outputs.key
+      run: |
+        mkdir -p $HOME/downloads/freesurfer
+        curl -s -o $HOME/downloads/freesurfer/freesurfer-linux-ubuntu22_amd64-7.4.1.tar.gz https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.4.1/freesurfer-linux-ubuntu22_amd64-7.4.1.tar.gz
+      shell: bash
+    - name: Install Freesurfer
+      env:
+        FREESURFER_LICENCE: ${{ secrets.FREESURFER_LICENCE }}
+      run: |
+        pushd $HOME/downloads/freesurfer
+        tar -zxpf freesurfer-linux-ubuntu22_amd64-7.4.1.tar.gz
+        mv freesurfer $HOME/
+        popd
+        export FREESURFER_HOME=$HOME/freesurfer
+        source $FREESURFER_HOME/SetUpFreeSurfer.sh
+        echo $FREESURFER_LICENCE > $FREESURFER_HOME/license.txt
+        export PATH=$FREESURFER_HOME/bin:$PATH
     - uses: actions/checkout@v3
     - name: Revert version to most recent tag on upstream update
       if: github.event_name == 'repository_dispatch'
@@ -92,11 +185,10 @@ jobs:
       uses: actions/download-artifact@v3
       with:
         name: converted-nipype
-        path: pydra/tasks/freesurfer/auto
-    - name: Check directory structure
+        path: src/pydra/tasks/freesurfer/auto
+    - name: Strip auto package from gitignore so it is included in package
       run: |
-        sudo apt install tree
-        tree .
+        sed -i '/\/src\/pydra\/tasks\/freesurfer\/auto/d' .gitignore
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -106,22 +198,26 @@ jobs:
         python -m pip install --upgrade pip
     - name: Install task package
       run: |
-        pip install ".[test]"
+        pip install "./src-fileformats" "./src-fileformats-extras" "./src[test]"
         python -c "import pydra.tasks.freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
         python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')"
     - name: Test with pytest
       run: |
-        pytest -sv --doctest-modules pydra/tasks/freesurfer \
+        pytest -sv --doctest-modules ./src/pydra/tasks/freesurfer \
             --cov pydra.tasks.freesurfer --cov-report xml
     - uses: codecov/codecov-action@v3
       if: ${{ always() }}
+      with:
+        files: .src/coverage.xml
+        # ,./src-fileformats/coverage.xml,./src-fileformats-extras/coverage.xml
+        name: pydra-freesurfer
 
   deploy:
     needs: [devcheck, test]
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.11']
+        pkg-dir: [".", "./src-fileformats", "./src-fileformats-extras"]
     steps:
     - uses: actions/checkout@v3
       with:
@@ -131,24 +227,27 @@ jobs:
       uses: actions/download-artifact@v3
       with:
         name: converted-nipype
-        path: pydra/tasks/freesurfer/auto
+        path: src/pydra/tasks/freesurfer/auto
     - name: Tag release with a post-release based on Nipype and Nipype2Pydra versions
       if: github.event_name == 'repository_dispatch'
       run: |
         TAG=$(git tag -l | tail -n 1 | awk -F post '{print $1}')
         POST=$(python -c "from pydra.tasks.freesurfer.auto._version import *; print(post_release)")
         git checkout $TAG
-        git add -f pydra/tasks/freesurfer/auto/_version.py
+        git add -f src/pydra/tasks/freesurfer/auto/_version.py
         git commit -am"added auto-generated version to make new tag for package version"
         git tag ${TAG}post${POST}    
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: '3.11'
     - name: Install build tools
       run: python -m pip install build twine
+    - name: Strip auto package from gitignore so it is included in package
+      run: |
+        sed -i '/\/src\/pydra\/tasks\/freesurfer\/auto/d' .gitignore
     - name: Build source and wheel distributions
-      run: python -m build
+      run: python -m build ${{ matrix.pkg-dir }}
     - name: Check distributions
       run: twine check dist/*
     - uses: actions/upload-artifact@v3

.gitignore

@@ -137,5 +137,10 @@ dmypy.json
 # Mac garbarge
 .DS_store
 
-/pydra/tasks/freesurfer/auto
-/pydra/tasks/freesurfer/_version.py
+# Auto-generated version files
+/src/pydra/tasks/freesurfer/_version.py
+/src-fileformats/fileformats/medimage_freesurfer/_version.py
+/src-fileformts-extras/fileformats/extras/medimage_freesurfer/_version.py
+
+# Auto-converted (from Nipype) interfaces directory (needs to be stripped before packaging)
+/src/pydra/tasks/freesurfer/auto

docs/index.rst

@@ -1,5 +1,5 @@
 Welcome to pydra-freesurfer's documentation!
-=========================================
+============================================
 
 .. toctree::
    :maxdepth: 2

nipype-auto-conv/generate

@@ -7,7 +7,7 @@ import shutil
 from importlib import import_module
 import yaml
 import nipype
-import nipype2pydra
+import nipype2pydra.utils
 from nipype2pydra.task import TaskConverter
 
 
@@ -31,7 +31,7 @@ sys.path.insert(0, str(SPECS_DIR))
 
 auto_init = f"# Auto-generated by {__file__}, do not edit as it will be overwritten\n\n"
 
-auto_dir = PKG_ROOT / "pydra" / "tasks" / PKG_NAME / "auto"
+auto_dir = PKG_ROOT / "src" / "pydra" / "tasks" / PKG_NAME / "auto"
 if auto_dir.exists():
     shutil.rmtree(auto_dir)
 
@@ -47,16 +47,18 @@ for fspath in sorted(SPECS_DIR.glob("**/*.yaml")):
 
     callables = import_module(rel_pkg_path + "_callables")
 
+    module_name = nipype2pydra.utils.to_snake_case(spec["task_name"])
+
     converter = TaskConverter(
-        output_module=f"pydra.tasks.{PKG_NAME}.auto.{spec['task_name']}",
+        output_module=f"pydra.tasks.{PKG_NAME}.auto.{module_name}",
         callables_module=callables,  # type: ignore
         **spec,
     )
-    converter.generate(PKG_ROOT)
-    auto_init += f"from .{spec['task_name']} import {converter.task_name}\n"
+    converter.generate(PKG_ROOT / "src")
+    auto_init += f"from .{module_name} import {converter.task_name}\n"
 
 
-with open(PKG_ROOT / "pydra" / "tasks" / PKG_NAME / "auto" / "_version.py", "w") as f:
+with open(auto_dir / "_version.py", "w") as f:
     f.write(
         f"""# Auto-generated by {__file__}, do not edit as it will be overwritten
 
@@ -66,5 +68,5 @@ post_release = (nipype_version + nipype2pydra_version).replace(".", "")
 """
     )
 
-with open(PKG_ROOT / "pydra" / "tasks" / PKG_NAME / "auto" / "__init__.py", "w") as f:
+with open(auto_dir / "__init__.py", "w") as f:
     f.write(auto_init)

src-fileformats-extras/conftest.py

@@ -0,0 +1,37 @@
+import os
+import logging
+from pathlib import Path
+import tempfile
+import pytest
+
+# Set DEBUG logging for unittests
+
+log_level = logging.WARNING
+
+logger = logging.getLogger("fileformats")
+logger.setLevel(log_level)
+
+sch = logging.StreamHandler()
+sch.setLevel(log_level)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+sch.setFormatter(formatter)
+logger.addHandler(sch)
+
+
+# For debugging in IDE's don't catch raised exceptions and let the IDE
+# break at it
+if os.getenv("_PYTEST_RAISE", "0") != "0":
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_exception_interact(call):
+        raise call.excinfo.value
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_internalerror(excinfo):
+        raise excinfo.value
+
+
+@pytest.fixture
+def work_dir():
+    work_dir = tempfile.mkdtemp()
+    return Path(work_dir)

src-fileformats-extras/fileformats/extras/medimage_freesurfer/__init__.py

@@ -0,0 +1 @@
+from ._version import __version__