From a55e6e7bb760a100fd8cf2d63b6985de94f983a4 Mon Sep 17 00:00:00 2001 From: v4hn Date: Sun, 13 Oct 2024 14:50:33 +0200 Subject: [PATCH] [WIP] refactor pipeline preparation --- .github/actions/aggregate-worker/action.yaml | 7 +- .../prepare-worker-pipeline/action.yaml | 57 ++++++++++++++++ .github/workflows/aggregate-debs.yaml | 68 +++++++------------ .github/workflows/build.yaml | 59 +++------------- .github/workflows/worker.yaml | 11 +-- scripts/import-workspace.sh | 2 +- 6 files changed, 100 insertions(+), 104 deletions(-) create mode 100644 .github/actions/prepare-worker-pipeline/action.yaml diff --git a/.github/actions/aggregate-worker/action.yaml b/.github/actions/aggregate-worker/action.yaml index 22ecf94438..8960968d6e 100644 --- a/.github/actions/aggregate-worker/action.yaml +++ b/.github/actions/aggregate-worker/action.yaml @@ -1,9 +1,6 @@ name: 'Aggregate Worker' description: 'Aggregate files from a worker' inputs: - stage: - type: string - required: true worker: type: string required: true @@ -16,9 +13,9 @@ runs: uses: actions/cache/restore@v4 with: path: ${{ env.REPO }} - key: apt-repo-${{ inputs.stage }}-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + key: apt-repo-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - apt-repo-${{ inputs.stage }}-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }} + apt-repo-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }} - name: Aggregate files shell: bash if: steps.restore.outputs.cache-matched-key != '' diff --git a/.github/actions/prepare-worker-pipeline/action.yaml b/.github/actions/prepare-worker-pipeline/action.yaml new file mode 100644 index 0000000000..2c390b4f5a --- /dev/null +++ b/.github/actions/prepare-worker-pipeline/action.yaml @@ -0,0 +1,57 @@ +name: 'Prepare Worker Pipeline' +description: 'Initializes staged build pipeline with job scheduling' + +outputs: + workers: + description: 'colon-separated list of workers used in the specification' + value: ${{ steps.worker.outputs.workers }} + +runs: + using: "composite" + steps: + - name: Check out the repo + uses: actions/checkout@v4 + - name: Prepare environment + run: | + echo 'Acquire::Retries "20";' | sudo tee -a /etc/apt/apt.conf.d/80-retries + echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries + echo 'Debug::Acquire::Retries "true";' | sudo tee -a /etc/apt/apt.conf.d/80-retries + sudo apt update + DEBIAN_FRONTEND=noninteractive sudo apt install -y software-properties-common retry + sudo retry -d 50,10,30,300 -t 12 add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros + sudo apt update + DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin + - name: Clone sources + run: | + echo 'Importing sources...' + ./scripts/import-workspace.sh ./workspace + - name: Prepare rosdep keys + run: | + cat rosdep.yaml | tee -a local.yaml + echo >> local.yaml # ensure trailing newline + for PKG in $(catkin_topological_order --only-names workspace); do + printf "%s:\n %s:\n - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml + done + - name: Prepare jobs + id: worker + run: | + # TODO: implement check for currently existing package versions and reduced rebuild + ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }} + echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT + - name: Store jobs cache + uses: actions/cache/save@v4 + with: + path: ${{ env.JOBS_YAML }} + key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Prepare meta data cache + run: | + mkdir -p ${{ env.AGG }} + mv local.yaml ${{ env.AGG }}/local.yaml + cp sources.repos ${{ env.AGG }}/sources_specified.repos + mkdir -p ${{ env.AGG }}/.github/workflows + cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml + - name: Store meta data cache + uses: actions/cache/save@v4 + with: + path: ${{ env.AGG }} + key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/aggregate-debs.yaml b/.github/workflows/aggregate-debs.yaml index b8d379e34c..b0dd739fc9 100644 --- a/.github/workflows/aggregate-debs.yaml +++ b/.github/workflows/aggregate-debs.yaml @@ -5,14 +5,16 @@ on: type: number required: true +env: + REPO: /home/runner/apt_repo + AGG: /home/runner/apt_repo_dependencies + jobs: aggregate: runs-on: ubuntu-24.04 env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 - REPO: /home/runner/apt_repo - AGG: /home/runner/apt_repo_dependencies stage: stage${{ inputs.stage }} + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 steps: - name: Check out the repo uses: actions/checkout@v4 @@ -30,103 +32,83 @@ jobs: - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker0 + worker: ${{ env.stage }}-worker0 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker1 + worker: ${{ env.stage }}-worker1 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker2 + worker: ${{ env.stage }}-worker2 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker3 + worker: ${{ env.stage }}-worker3 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker4 + worker: ${{ env.stage }}-worker4 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker5 + worker: ${{ env.stage }}-worker5 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker6 + worker: ${{ env.stage }}-worker6 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker7 + worker: ${{ env.stage }}-worker7 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker8 + worker: ${{ env.stage }}-worker8 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker9 + worker: ${{ env.stage }}-worker9 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker10 + worker: ${{ env.stage }}-worker10 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker11 + worker: ${{ env.stage }}-worker11 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker12 + worker: ${{ env.stage }}-worker12 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker13 + worker: ${{ env.stage }}-worker13 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker14 + worker: ${{ env.stage }}-worker14 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker15 + worker: ${{ env.stage }}-worker15 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker16 + worker: ${{ env.stage }}-worker16 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker17 + worker: ${{ env.stage }}-worker17 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker18 + worker: ${{ env.stage }}-worker18 - name: Aggregate worker uses: ./.github/actions/aggregate-worker with: - stage: ${{ env.stage }} - worker: worker19 + worker: ${{ env.stage }}-worker19 - name: Store apt repo uses: actions/cache/save@v4 if: always() diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e1fefd88bd..52a25a456a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,59 +7,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} env: - AGG: /home/runner/apt_repo_dependencies DISTRIBUTION: ubuntu + DEB_DISTRO: jammy + BRANCH: jammy-one-experimental + JOBS_YAML: /home/runner/jobs.yaml + AGG: /home/runner/apt_repo_dependencies jobs: stage-1: runs-on: ubuntu-24.04 outputs: - workers: ${{ steps.worker.outputs.workers }} - env: - JOBS_YAML: /home/runner/jobs.yaml + workers: ${{ steps.prepare.outputs.workers }} steps: - - name: Check out the repo - uses: actions/checkout@v4 - - name: Clone sources - run: | - echo 'Acquire::Retries "20";' | sudo tee -a /etc/apt/apt.conf.d/80-retries - echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries - echo 'Debug::Acquire::Retries "true";' | sudo tee -a /etc/apt/apt.conf.d/80-retries - sudo add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros - sudo apt update - DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin - mkdir workspace - echo 'Importing sources...' - vcs import -w 5 --recursive --shallow --input sources.repos workspace - - name: Prepare rosdep keys - run: | - cp rosdep.yaml local.yaml - echo >> local.yaml # ensure trailing newline - for PKG in $(catkin_topological_order --only-names workspace); do - printf "%s:\n %s:\n - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml - done - - name: Prepare Jobs - id: worker - run: | - ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }} - echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT - - name: Store jobs cache - uses: actions/cache/save@v4 - with: - path: ${{ env.JOBS_YAML }} - key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} - - name: Prepare meta data cache - run: | - mkdir -p ${{ env.AGG }} - mv local.yaml ${{ env.AGG }}/local.yaml - cp sources.repos ${{ env.AGG }}/sources_specified.repos - mkdir -p ${{ env.AGG }}/.github/workflows - cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml - - name: Store meta data cache - uses: actions/cache/save@v4 - with: - path: ${{ env.AGG }} - key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Prepare Pipeline + id: prepare + uses: ./.github/actions/prepare-worker-pipeline stage0-worker0: uses: ./.github/workflows/worker.yaml @@ -1465,9 +1427,6 @@ jobs: needs: stage18 if: always() && !cancelled() runs-on: ubuntu-24.04 - env: - ROS_DISTRO: one - DEB_DISTRO: jammy steps: - name: get apt packages from last job uses: actions/cache/restore@v4 @@ -1481,6 +1440,6 @@ jobs: mv ${{ env.AGG }} /home/runner/apt_repo - uses: v4hn/ros-deb-builder-action/deploy@roso-noble with: - BRANCH: ${{ env.DEB_DISTRO }}-${{ env.ROS_DISTRO }}-unstable + BRANCH: ${{ env.BRANCH }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SQUASH_HISTORY: true diff --git a/.github/workflows/worker.yaml b/.github/workflows/worker.yaml index 5d704b5f5b..13846e7942 100644 --- a/.github/workflows/worker.yaml +++ b/.github/workflows/worker.yaml @@ -7,14 +7,15 @@ on: type: string required: true +env: + DEB_DISTRO: jammy + JOBS_YAML: /home/runner/jobs.yaml + jobs: build: runs-on: ubuntu-24.04 env: - ROS_DISTRO: one - DEB_DISTRO: jammy SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 - JOBS_YAML: /home/runner/jobs.yaml steps: - name: Check out the repo uses: actions/checkout@v4 @@ -32,7 +33,7 @@ jobs: echo "Cache not found" exit 1 fi - - name: Use apt packages ${{ inputs.depends }} + - name: Use dependencies ${{ inputs.depends }} id: fetch-dependencies if: ${{ inputs.depends }} uses: actions/cache/restore@v4 @@ -69,8 +70,8 @@ jobs: uses: v4hn/ros-deb-builder-action/build@roso-noble timeout-minutes: 330 with: - ROS_DISTRO: ${{ env.ROS_DISTRO }} DEB_DISTRO: ${{ env.DEB_DISTRO }} + ROS_DISTRO: one SBUILD_CONF: "$extra_packages = ['/home/runner/apt_repo_dependencies']; $enable_network = 1; ${{ env.extra_sbuild_conf }}" ROSDEP_SOURCE: rosdep.yaml REPOS_FILE: worker-jobs.repos diff --git a/scripts/import-workspace.sh b/scripts/import-workspace.sh index 07588f242a..6aee6ccb85 100755 --- a/scripts/import-workspace.sh +++ b/scripts/import-workspace.sh @@ -11,5 +11,5 @@ fi for f in $REPOS; do echo importing $f - vcs import --shallow --recursive $WORKSPACE --input "$f" >/dev/null + vcs import -w 5 --shallow --recursive $WORKSPACE --input "$f" >/dev/null done