From 9ba57566b8a2286d4aab63c91687fce0591e31c7 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 22 Mar 2024 15:04:31 +1100 Subject: [PATCH] imported latest ci-cd workflow --- .github/workflows/ci-cd.yaml | 299 ++++++++++++++++++++++++----------- 1 file changed, 206 insertions(+), 93 deletions(-) diff --git a/.github/workflows/ci-cd.yaml b/.github/workflows/ci-cd.yaml index 887e627..d1480dd 100644 --- a/.github/workflows/ci-cd.yaml +++ b/.github/workflows/ci-cd.yaml @@ -9,30 +9,42 @@ name: CI/CD on: push: branches: [ main, develop ] - tags: [ '*' ] pull_request: branches: [ main, develop ] + release: + types: [published] repository_dispatch: - types: [create-release] + types: [create-post-release] + +env: + FREESURFER_VERSION: 7.4.1 + FREESURFER_HOME: $HOME/freesurfer-install jobs: nipype-conv: runs-on: ubuntu-latest steps: + - name: Checkout - uses: actions/checkout@v3 - - name: Revert version to most recent tag on upstream update + uses: actions/checkout@v4 + + - name: Revert version to most recent version tag on upstream update if: github.event_name == 'repository_dispatch' - run: git checkout $(git tag -l | tail -n 1 | awk -F post '{print $1}') - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + run: git checkout $(git tag -l | grep 'v.*' | tail -n 1 | awk -F post '{print $1}') + + - name: Set up Python + uses: actions/setup-python@v5 + - name: Install build dependencies run: python -m pip install --upgrade pip + - name: Install requirements - run: python -m pip install -r ./nipype-auto-conv/requirements.txt + run: python -m pip install ./related-packages/fileformats -r ./nipype-auto-conv/requirements.txt + - name: Run automatic Nipype > Pydra conversion run: ./nipype-auto-conv/generate + - uses: actions/upload-artifact@v3 with: name: converted-nipype @@ -50,163 +62,174 @@ jobs: - '--editable git+https://github.com/nipype/pydra.git#egg=pydra' steps: - name: Checkout - uses: actions/checkout@v3 - - name: Revert version to most recent tag on upstream update + uses: actions/checkout@v4 + + - name: Revert version to most recent version tag on upstream update if: github.event_name == 'repository_dispatch' - run: git checkout $(git tag -l | tail -n 1 | awk -F post '{print $1}') + run: git checkout $(git tag -l | grep 'v.*' | tail -n 1 | awk -F post '{print $1}') + - name: Download tasks converted from Nipype uses: actions/download-artifact@v3 with: name: converted-nipype path: pydra/tasks/freesurfer/auto + - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/freesurfer\/auto/d' .gitignore + sed -i '/^_version.py/d' .gitignore + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install build dependencies run: | python -m pip install --upgrade pip + - name: Install Pydra run: | pushd $HOME pip install ${{ matrix.pydra }} popd python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" + - name: Install task package run: | - pip install "./related-packages/fileformats[dev]" "related-packages/fileformats-extras[dev]" + pip install ${{ matrix.pip-flags }} "./related-packages/fileformats[dev]" + pip install ${{ matrix.pip-flags }} "related-packages/fileformats-extras[dev]" pip install ${{ matrix.pip-flags }} ".[dev]" python -c "import pydra.tasks.freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" python -c "import fileformats.medimage_freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" python -c "import fileformats.extras.medimage_freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" - - fileformats-test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.11'] - steps: - - uses: actions/checkout@v3 - - name: Revert version to most recent tag on upstream update - if: github.event_name == 'repository_dispatch' - run: git checkout $(git tag -l | tail -n 1 | awk -F post '{print $1}') - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - - name: Install task package - run: | - pip install "./related-packages/fileformats[test]" "./related-packages/fileformats-extras[test]" - python -c "import fileformats.medimage_freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" - - name: Test fileformats with pytest - run: | - cd ./related-packages - pytest -sv --cov fileformats.medimage_freesurfer --cov fileformats.extras.medimage_freesurfer --cov-report xml . test: - needs: [nipype-conv, fileformats-test] - runs-on: ubuntu-22.04 + needs: [nipype-conv] + runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8'] # '3.11' + python-version: ['3.8', '3.11'] steps: + - name: Removed unnecessary tools to free space run: | sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: Get Download cache Key - id: cache-key - run: echo "::set-output name=key::freesurfer-linux-ubuntu22_amd64-7.4.1" - - name: Cache FreeSurfer - uses: actions/cache@v2 + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Revert version to most recent version tag on Nipype or Nipype2Pydra update + if: github.event_name == 'repository_dispatch' + run: git checkout $(git tag -l | grep 'v.*' | tail -n 1 | awk -F post '{print $1}') + + - name: Cache Freesurfer Install + id: cache-install + uses: actions/cache@v4 with: - path: $HOME/downloads/freesurfer - key: ${{ steps.cache-key.outputs.key }} - restore-keys: | - freesurfer-linux-ubuntu22_amd64-7.4.1 + path: ${{ env.FREESURFER_HOME }} + key: freesurfer-${{ env.FREESURFER_VERSION }}-${{ runner.os }} + - name: Download FreeSurfer - if: steps.cache-key.outputs.key != steps.cache-hit.outputs.key + if: steps.cache-install.outputs.cache-hit != 'true' run: | mkdir -p $HOME/downloads/freesurfer - curl -s -o $HOME/downloads/freesurfer/freesurfer-linux-ubuntu22_amd64-7.4.1.tar.gz https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.4.1/freesurfer-linux-ubuntu22_amd64-7.4.1.tar.gz + curl -s -o $HOME/downloads/freesurfer/freesurfer-linux-ubuntu22_amd64-${{ env.FREESURFER_VERSION }}.tar.gz https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/${{ env.FREESURFER_VERSION }}/freesurfer-linux-ubuntu22_amd64-${{ env.FREESURFER_VERSION }}.tar.gz shell: bash + - name: Install Freesurfer + if: steps.cache-install.outputs.cache-hit != 'true' env: FREESURFER_LICENCE: ${{ secrets.FREESURFER_LICENCE }} run: | pushd $HOME/downloads/freesurfer - tar -zxpf freesurfer-linux-ubuntu22_amd64-7.4.1.tar.gz + tar -zxpf freesurfer-linux-ubuntu22_amd64-${{ env.FREESURFER_VERSION }}.tar.gz mv freesurfer $HOME/ popd - export FREESURFER_HOME=$HOME/freesurfer - source $FREESURFER_HOME/SetUpFreeSurfer.sh - echo $FREESURFER_LICENCE > $FREESURFER_HOME/license.txt - export PATH=$FREESURFER_HOME/bin:$PATH - - uses: actions/checkout@v3 - - name: Revert version to most recent tag on upstream update - if: github.event_name == 'repository_dispatch' - run: git checkout $(git tag -l | tail -n 1 | awk -F post '{print $1}') + source ${{ env.FREESURFER_HOME }}/SetUpFreeSurfer.sh + echo $FREESURFER_LICENCE > ${{ env.FREESURFER_HOME }}/license.txt + echo "PATH=${{ env.FREESURFER_HOME }}/bin:$PATH" >> $GITHUB_ENV + - name: Download tasks converted from Nipype uses: actions/download-artifact@v3 with: name: converted-nipype path: pydra/tasks/freesurfer/auto + + - name: Show the contents of the auto-generated tasks + run: tree pydra + - name: Strip auto package from gitignore so it is included in package run: | - sed -i '/\/src\/pydra\/tasks\/freesurfer\/auto/d' .gitignore + sed -i '/\/pydra\/tasks\/freesurfer\/auto/d' .gitignore + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install build dependencies run: | python -m pip install --upgrade pip + - name: Install task package run: | pip install "./related-packages/fileformats" "./related-packages/fileformats-extras" ".[test]" python -c "import pydra.tasks.freesurfer as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" + - name: Test with pytest - run: | - pytest -sv --doctest-modules ./pydra/tasks/freesurfer \ - --cov pydra.tasks.freesurfer --cov-report xml - - uses: codecov/codecov-action@v3 + run: >- + pytest -sv + ./pydra/tasks/freesurfer + ./related-packages/fileformats + ./related-packages/fileformats-extras + --cov pydra.tasks.freesurfer + --cov fileformats.medimage_freesurfer + --cov fileformats.extras.medimage_freesurfer + --cov-report xml + + - name: Upload to CodeCov + uses: codecov/codecov-action@v3 if: ${{ always() }} with: - files: coverage.xml,./related-packages/coverage.xml + files: coverage.xml name: pydra-freesurfer + deploy-fileformats: needs: [devcheck, test] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + + - uses: actions/checkout@v4 with: submodules: recursive - fetch-depth: 0 + fetch-depth: 0 + - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' + - name: Install build tools run: python -m pip install build twine + - name: Build source and wheel distributions run: python -m build ./related-packages/fileformats + - name: Check distributions run: twine check ./related-packages/fileformats/dist/* + - name: Check for PyPI token on tag id: deployable - if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) || github.event_name == 'repository_dispatch' + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi + - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 @@ -219,26 +242,33 @@ jobs: needs: [deploy-fileformats] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + + - uses: actions/checkout@v4 with: submodules: recursive - fetch-depth: 0 + fetch-depth: 0 + - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' + - name: Install build tools run: python -m pip install build twine + - name: Build source and wheel distributions run: python -m build ./related-packages/fileformats-extras + - name: Check distributions run: twine check ./related-packages/fileformats-extras/dist/* + - name: Check for PyPI token on tag id: deployable - if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) || github.event_name == 'repository_dispatch' + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_EXTRAS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi + - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 @@ -248,58 +278,141 @@ jobs: packages-dir: ./related-packages/fileformats-extras/dist deploy: - needs: [deploy-fileformats-extras] + needs: [nipype-conv, test, deploy-fileformats, deploy-fileformats-extras] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + + - name: Checkout repository + uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 + + - name: Set up Git user + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + - name: Get latest version tag + id: latest_tag + run: | + git fetch --tags + echo "TAG=$(git tag -l | grep 'v.*' | tail -n 1 | awk -F post '{print $1}')" >> $GITHUB_OUTPUT + + - name: Revert to latest tag + if: github.event_name == 'repository_dispatch' + run: git checkout ${{ steps.latest_tag.outputs.TAG }} + - name: Download tasks converted from Nipype uses: actions/download-artifact@v3 with: name: converted-nipype path: pydra/tasks/freesurfer/auto - - name: Tag release with a post-release based on Nipype and Nipype2Pydra versions - if: github.event_name == 'repository_dispatch' - run: | - TAG=$(git tag -l | tail -n 1 | awk -F post '{print $1}') - POST=$(python -c "from pydra.tasks.freesurfer.auto._version import *; print(post_release)") - git checkout $TAG - git add -f pydra/tasks/freesurfer/auto/_version.py - git commit -am"added auto-generated version to make new tag for package version" - git tag ${TAG}post${POST} + + - name: Show the contents of the auto-generated tasks + run: tree pydra + - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' + - name: Install build tools run: python -m pip install build twine + - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/freesurfer\/auto/d' .gitignore + cat .gitignore + + - name: Install task package to calculate post-release tag + run: | + pip install "./related-packages/fileformats" "./related-packages/fileformats-extras" ".[test]" + + - name: Generate post-release tag based on Nipype and Nipype2Pydra versions + id: post_release_tag + run: | + POST=$(python -c "from pydra.tasks.freesurfer.auto._version import *; print(post_release)") + echo "TAG=${{ steps.latest_tag.outputs.TAG }}post${POST}" >> $GITHUB_OUTPUT + + - name: Add auto directory to git repo + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' + run: | + git add pydra/tasks/freesurfer/auto + git commit -am"added auto-generated version to make new tag for package version" + git status + + - name: Overwrite the tag of release event with latest commit (i.e. including the auto directory) + if: github.event_name == 'release' + run: | + git tag -d ${{ steps.latest_tag.outputs.TAG }}; + git tag ${{ steps.latest_tag.outputs.TAG }}; + + - name: Tag repo with the post-release + if: github.event_name == 'repository_dispatch' + run: git tag ${{ steps.post_release_tag.outputs.TAG }} + - name: Build source and wheel distributions run: python -m build . + - name: Check distributions run: twine check dist/* + - uses: actions/upload-artifact@v3 with: name: distributions path: dist/ + - name: Check for PyPI token on tag id: deployable - if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) || github.event_name == 'repository_dispatch' + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi + - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + password: ${{ secrets.PYPI_API_TOKEN }} + + - name: Create post-release release for releases triggered by nipype2pydra dispatches + if: steps.deployable.outputs.DEPLOY && github.event_name == 'repository_dispatch' + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ steps.post_release_tag.outputs.TAG }} + release_name: Release ${{ steps.post_release_tag.outputs.TAG }} + draft: false + prerelease: false + + report_progress: + needs: [deploy] + runs-on: ubuntu-latest + steps: + + - name: Generate progress report + id: generate-report + run: | + tools/report_progress.py outputs/progress-report.json + echo "progress_report=$(cat outputs/progress-report.json)" >> $GITHUB_OUTPUT + + - name: Report progress to Nipype2Pydra repo + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' + run: >- + curl -XPOST -u "${{ env.POST_RELEASE_PAT }}" -H "Accept: application/vnd.github.everest-preview+json" + "https://api.github.com/repos/nipype/pydra-freesurfer/dispatches" + -d '{ + "event_type": "progress-report", + "client_payload": ${{ steps.generate-report.output.progress_report }} + }' + env: + PAT: ${{ env.PROGRESS_REPORT_PAT }} + # Deploy on tags if PYPI_API_TOKEN is defined in the repository secrets. # Secrets are not accessible in the if: condition [0], so set an output variable [1] # [0] https://github.community/t/16928 -# [1] https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-output-parameter \ No newline at end of file +# [1] https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-output-parameter