Skip to content

Release

Release #2465

Workflow file for this run

name: Release
on:
push:
branches:
- "release/*"
schedule:
# beijing: 2 am.
# utc: 6 pm.
- cron: "0 18 * * *"
workflow_dispatch:
inputs:
placeholder:
description: "update .github/workflows/release.yml to config your build"
required: false
workflow_call:
inputs:
is_priv:
required: true
type: boolean
branch:
required: false
type: string
default: "main"
upload_override_branch:
required: false
type: string
cuda_cmake_cache:
required: false
type: string
secrets:
ONEFLOW_PRIV_ORG:
required: true
ONEFLOW_PRIV_GH_TOKEN:
required: true
ONEFLOW_PRIV_OSS_BUCKET:
required: true
OSS_ACCESS_KEY_ID:
required: true
OSS_ACCESS_KEY_SECRET:
required: true
ONEFLOW_CI_HTTP_PROXY:
required: false
concurrency:
group: release-${{ github.ref }}-${{ inputs.branch }}
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
env:
ONEFLOW_SRC: .
jobs:
generate-build-matrix:
name: "Generate build matrix"
runs-on: ubuntu-latest
env:
ONEFLOW_SRC: .
outputs:
matrix: ${{ steps.find-cache.outputs.matrix }}
formatted_date: ${{ steps.date.outputs.formatted_date }}
steps:
- name: Checkout Oneflow-Inc/oneflow
uses: actions/checkout@v2
if: ${{ !inputs.is_priv }}
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: Checkout oneflow
uses: actions/checkout@v2
if: ${{ inputs.is_priv }}
with:
ref: ${{ inputs.branch }}
repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow
token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }}
- uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@whl-skip-nccl
name: Find build cache
id: find-cache
timeout-minutes: 5
with:
delete-cache: ${{ contains(github.event.pull_request.labels.*.name, 'need-clean-ccache') }}
runner-labels: |
self-hosted
linux
release
oneflow-src: ${{ env.ONEFLOW_SRC }}
entries: |
cu122
cu121
cu118
cpu
- name: Get current date
id: date
run: echo "formatted_date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
staging_release:
env:
MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/release/${{ matrix.entry }}
WHEELHOUSE_DIR: manylinux_wheelhouse
OSS_DIR: branch/${{ github.ref_name }}/${{ matrix.entry }}/${{ github.sha }}
GITHUB_REF_NAME: ${{ github.ref_name }}
GITHUB_SHA: ${{ github.sha }}
ONEFLOW_OSS_BUCKET: oneflow-staging
https_proxy: ${{ secrets.ONEFLOW_CI_HTTP_PROXY }}
needs: [generate-build-matrix]
name: Staging Release
timeout-minutes: 240
runs-on: [self-hosted, linux, release]
if: github.repository == 'Oneflow-Inc/oneflow' || inputs.is_priv
strategy:
fail-fast: false
max-parallel: 6
matrix: ${{ fromJson(needs.generate-build-matrix.outputs.matrix) }}
steps:
- name: Fix permissions
run: |
docker run --rm -v $PWD:/p -w /p busybox rm -rf *
- name: Install dependencies
run: |
python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
python3 -m pip install -U setuptools wheel --user
python3 -m pip install oss2 --user
- name: Checkout Oneflow-Inc/oneflow
uses: actions/checkout@v2
if: ${{ !inputs.is_priv }}
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: Checkout private oneflow
uses: actions/checkout@v2
if: ${{ inputs.is_priv }}
with:
ref: ${{ inputs.branch }}
repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow
token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }}
- name: Checkout cutlass_extension
uses: actions/checkout@v2
if: ${{ inputs.is_priv }}
with:
repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/cutlass-extension
token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }}
path: cutlass-extension
- name: Set Private env
if: ${{ inputs.is_priv }}
run: |
GITHUB_SHA=$(git rev-parse HEAD)
echo "OSS_DIR=branch/${{ inputs.upload_override_branch || inputs.branch }}/${{ matrix.entry }}/${GITHUB_SHA}" >> $GITHUB_ENV
echo "GITHUB_REF_NAME=${{ inputs.upload_override_branch || inputs.branch }}" >> $GITHUB_ENV
echo "GITHUB_SHA=${GITHUB_SHA}" >> $GITHUB_ENV
echo "ONEFLOW_OSS_BUCKET=${{ secrets.ONEFLOW_PRIV_OSS_BUCKET }}" >> $GITHUB_ENV
- name: Print env
if: ${{ inputs.is_priv }}
run: |
env
- uses: Oneflow-Inc/get-oneflow@whl-skip-nccl
name: Build ${{ matrix.entry }}
if: ${{ matrix.entry =='cu118' || startsWith(matrix.entry, 'cu12') }}
with:
cmake-init-cache: ${{ env.ONEFLOW_SRC }}/${{ inputs.cuda_cmake_cache || 'cmake/caches/ci/release/cu118.cmake' }}
build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc9.sh
oneflow-src: ${{ env.ONEFLOW_SRC }}
oneflow-build-env: manylinux
wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
clear-wheelhouse-dir: true
self-hosted: true
compute-platform: ${{ matrix.entry }}
manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
docker-run-use-system-http-proxy: false
docker-run-use-lld: false
retry-failed-build: true
clean-ccache: true
nightly: ${{ inputs.is_priv || github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}}
nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }}
use-nvidia-wheels: ${{ matrix.entry !='cu112' }}
python-versions: |
3.12
3.11
3.10
3.9
3.8
- uses: Oneflow-Inc/get-oneflow@whl-skip-nccl
name: Build ${{ matrix.entry }}
if: ${{ startsWith(matrix.entry, 'cu') && matrix.entry !='cu118' && !startsWith(matrix.entry, 'cu12') }}
with:
cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cuda.cmake
build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc9.sh
oneflow-src: ${{ env.ONEFLOW_SRC }}
oneflow-build-env: manylinux
wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
clear-wheelhouse-dir: true
self-hosted: true
compute-platform: ${{ matrix.entry }}
manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
docker-run-use-system-http-proxy: false
docker-run-use-lld: false
retry-failed-build: true
clean-ccache: true
nightly: ${{ inputs.is_priv || github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}}
nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }}
use-nvidia-wheels: ${{ matrix.entry !='cu112' }}
python-versions: |
3.12
3.11
3.10
3.9
3.8
- uses: Oneflow-Inc/get-oneflow@whl-skip-nccl
name: Build ${{ matrix.entry }}
if: ${{ matrix.entry =='cpu' }}
with:
cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cpu.cmake
build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build.sh
oneflow-src: ${{ env.ONEFLOW_SRC }}
oneflow-build-env: manylinux
wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
clear-wheelhouse-dir: true
self-hosted: true
compute-platform: ${{ matrix.entry }}
manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
docker-run-use-system-http-proxy: false
docker-run-use-lld: false
retry-failed-build: true
clean-ccache: false
nightly: ${{ inputs.is_priv || github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}}
nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }}
python-versions: |
3.12
3.11
3.10
3.9
3.8
- name: Upload wheel
uses: ./.github/actions/upload_oss
with:
src_path: ${{ env.WHEELHOUSE_DIR }}
oss_dst_path: oss://${{ env.ONEFLOW_OSS_BUCKET }}/${{ env.OSS_DIR }}
oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
- name: Update pip index
env:
OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
run: |
python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
python3 -m pip install oss2 beautifulsoup4 --user
python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b ${{ env.ONEFLOW_OSS_BUCKET }} \
--index_key=branch/${{ env.GITHUB_REF_NAME }}/${{ matrix.entry }}/index.html \
--index_key=branch/${{ env.GITHUB_REF_NAME }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \
--index_key=${{ env.OSS_DIR }}/index.html \
--index_key=commit/${{ env.GITHUB_SHA }}/${{ matrix.entry }}/index.html
- name: Update API docs
if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu' && !inputs.is_priv
env:
READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }}
run: |
curl -X POST -d "branches=master" -d "token=${READTHEDOCS_TOKEN}" https://readthedocs.org/api/v2/webhook/oneflow/135376/