diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 3e7fd0864..40790685e 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -9,7 +9,7 @@ jobs: matrix: python-version: [3.9] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Extract black version from setup.py run: | diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml index de9c6eba7..7e1aefbbd 100644 --- a/.github/workflows/codacy.yml +++ b/.github/workflows/codacy.yml @@ -31,4 +31,4 @@ jobs: steps: # Checkout the repository to the GitHub Actions runner - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4a8c4338f..ceab64ca5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dependencies/action.yml b/.github/workflows/dependencies/action.yml new file mode 100644 index 000000000..b3c2f0387 --- /dev/null +++ b/.github/workflows/dependencies/action.yml @@ -0,0 +1,103 @@ +name: 'Install Dependencies' + +description: 'Setup and install dependencies for GaNDLF' + +outputs: + other_modified_files_count: + description: "Which files have changed" + value: ${{ steps.changed-files.outputs.other_modified_files_count }} +# on: +# workflow_call: + +# jobs: +# install_dependencies: +# runs-on: ubuntu-latest +runs: + using: "composite" + steps: + - name: Free space + shell: bash + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "$ANDROID_SDK_ROOT" + df -h + + # - name: Checkout code + # uses: actions/checkout@v4 + + # Use changed-files-specific action to collect file changes. + # The following commented condition applied to a step will run that step only if non-docs files have changed. + # It should be applied to all functionality-related steps. + # if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} + - name: Detect and screen file changes + # shell: bash + id: changed-files-specific + uses: tj-actions/changed-files@v44 + with: + files: | + .github/*.md + .github/ISSUE_TEMPLATE/*.md + .github/workflows/devcontainer.yml + .github/workflows/docker-image.yml + .devcontainer/** + docs/** + mlcube/** + *.md + LICENSE + Dockerfile-* + # run: | + # echo "other_modified_files_count=$(echo ${{ steps.changed-files-specific.outputs.other_modified_files_count }})" >> $GITHUB_OUTPUT + + - name: Summarize docs and non-docs modifications + id: changed-files + shell: bash + run: | + echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" + echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + echo "Count of non-docs files changed: ${{ steps.changed-files-specific.outputs.other_modified_files_count }}" + echo "If only-modified is triggered: ${{ steps.changed-files-specific.outputs.only_modified }}" + echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + echo "$GITHUB_OUTPUT" + + + ## this did NOT work + # - name: Set output + # shell: bash + # run: echo "other_modified_files_count=${{ steps.changed-files-specific.outputs.other_modified_files_count }}" >> $GITHUB_OUTPUT + + ## this did NOT work + # - name: Check saved output + # shell: bash + # run: echo "GITHUB_OUTPUT:${{ GITHUB_OUTPUT }}" + + # This second step is unnecessary but highly recommended because + # It will cache database and saves time re-downloading it if database isn't stale. + - name: Cache pip + # shell: bash + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python 3.9 + # shell: bash + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install dependencies and package + shell: bash + if: ${{steps.changed-files.outputs.other_modified_files_count > 0}} # Run on any non-docs change + run: | + sudo apt-get update + sudo apt-get install libvips libvips-tools -y + python -m pip install --upgrade pip==24.0 + python -m pip install wheel + pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu + pip install -e . diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 730acbff5..4337cd8bd 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -19,7 +19,7 @@ jobs: df -h - name: Checkout (GitHub) - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Use changed-files-specific action to collect file changes. # The following commented condition applied to a step will run that step only if non-docs files have changed. diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 502467131..d872c57f8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: lfs: true submodules: 'recursive' diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index b92a1f625..8b471e9e9 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -10,7 +10,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.x diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba5e79b7a..2c286a20b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/mlcube-test.yml b/.github/workflows/mlcube-test.yml index 23f91b178..7536bb687 100644 --- a/.github/workflows/mlcube-test.yml +++ b/.github/workflows/mlcube-test.yml @@ -12,68 +12,20 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v3 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 - with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + - name: Checkout code + uses: actions/checkout@v4 - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + - name: Call reusable workflow to install dependencies + id: dependencies + uses: ./.github/workflows/dependencies - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - name: Install dependencies and package if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - python -m pip install openvino-dev==2023.0.1 mlcube_docker - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . + python -m pip install mlcube_docker + - name: Run mlcube deploy tests working-directory: ./testing - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | sh test_deploy.sh diff --git a/.github/workflows/openfl-test.yml b/.github/workflows/openfl-test.yml index 00bf95506..d30ee295b 100644 --- a/.github/workflows/openfl-test.yml +++ b/.github/workflows/openfl-test.yml @@ -17,72 +17,21 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v3 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 - with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + - name: Checkout code + uses: actions/checkout@v4 - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + - name: Call reusable workflow to install dependencies + id: dependencies + uses: ./.github/workflows/dependencies - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run generic unit tests to download data and construct CSVs - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" + # openfl tests start here - name: Run OpenFL tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | echo "Removing onnx because of protobuf version conflict" pip uninstall onnx -y diff --git a/.github/workflows/ossar-analysis.yml b/.github/workflows/ossar-analysis.yml index b0323a7b2..804ed0dea 100644 --- a/.github/workflows/ossar-analysis.yml +++ b/.github/workflows/ossar-analysis.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Ensure a compatible version of dotnet is installed. # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201. diff --git a/.github/workflows/publish-nightly.yml b/.github/workflows/publish-nightly.yml index 4d8ff05a3..fb587e476 100644 --- a/.github/workflows/publish-nightly.yml +++ b/.github/workflows/publish-nightly.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master diff --git a/.github/workflows/python-install-check.yml b/.github/workflows/python-install-check.yml index 7ec37528a..7fdb3283c 100644 --- a/.github/workflows/python-install-check.yml +++ b/.github/workflows/python-install-check.yml @@ -9,7 +9,7 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 19b47027f..68e09865d 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -13,101 +13,48 @@ jobs: runs-on: ubuntu-latest steps: - - name: Free space - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf "$ANDROID_SDK_ROOT" - df -h - - name: Checkout - uses: actions/checkout@v3 - - # Use changed-files-specific action to collect file changes. - # The following commented condition applied to a step will run that step only if non-docs files have changed. - # It should be applied to all functionality-related steps. - # if: steps.changed-files-specific.outputs.only_modified == 'false' - - name: Detect and screen file changes - id: changed-files-specific - uses: tj-actions/changed-files@v41 - with: - files: | - .github/*.md - .github/ISSUE_TEMPLATE/*.md - .github/workflows/devcontainer.yml - .github/workflows/docker-image.yml - .devcontainer/** - docs/** - mlcube/** - *.md - LICENSE - Dockerfile-* + - name: Checkout code + uses: actions/checkout@v4 - - name: Summarize docs and non-docs modifications - run: | - echo "List of docs files that have changed: ${{ steps.changed-files-specific.outputs.all_modified_files }}" - echo "Changed non-docs files: ${{ steps.changed-files-specific.outputs.other_modified_files }}" + - name: Call reusable workflow to install dependencies + id: dependencies + uses: ./.github/workflows/dependencies - # This second step is unnecessary but highly recommended because - # It will cache database and saves time re-downloading it if database isn't stale. - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Set up Python 3.9 - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies and package - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change - run: | - sudo apt-get update - sudo apt-get install libvips libvips-tools -y - python -m pip install --upgrade pip==24.0 - python -m pip install wheel - python -m pip install openvino-dev==2023.0.1 mlcube_docker - pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cpu - pip install -e . - name: Run generic unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml -k "generic" - name: Run classification unit tests with histology - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and histology" - name: Run classification unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "classification and not histology" - name: Run segmentation unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "segmentation and not transunet" - name: Run regression unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "regression" - name: Run transunet unit tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "transunet" - name: Run entrypoints tests - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "entrypoints" - name: Run test for update_version - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change run: | pytest --cov=. --cov-report=xml --cov-append -k "update_version" - - name: Upload coverage to CodeCov - if: steps.changed-files-specific.outputs.only_modified == 'false' # Run on any non-docs change + - name: Upload coverage + if: ${{steps.dependencies.outputs.other_modified_files_count > 0}} # Run on any non-docs change uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }}