[WIP] refactor pipeline preparation

.github/actions/aggregate-worker/action.yaml

@@ -1,9 +1,6 @@
 name: 'Aggregate Worker'
 description: 'Aggregate files from a worker'
 inputs:
- stage:
- type: string
- required: true
  worker:
  type: string
  required: true
@@ -16,9 +13,9 @@ runs:
  uses: actions/cache/restore@v4
  with:
  path: ${{ env.REPO }}
- key: apt-repo-${{ inputs.stage }}-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+ key: apt-repo-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
  restore-keys: |
- apt-repo-${{ inputs.stage }}-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}
+ apt-repo-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}
  - name: Aggregate files
  shell: bash
  if: steps.restore.outputs.cache-matched-key != ''

.github/actions/prepare-worker-pipeline/action.yaml

@@ -0,0 +1,57 @@
+name: 'Prepare Worker Pipeline'
+description: 'Initializes staged build pipeline with job scheduling'
+
+outputs:
+ workers:
+ description: 'colon-separated list of workers used in the specification'
+ value: ${{ steps.worker.outputs.workers }}
+
+runs:
+ using: "composite"
+ steps:
+ - name: Check out the repo
+ uses: actions/checkout@v4
+ - name: Prepare environment
+ run: |
+ echo 'Acquire::Retries "20";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
+ echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
+ echo 'Debug::Acquire::Retries "true";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
+ sudo apt update
+ DEBIAN_FRONTEND=noninteractive sudo apt install -y software-properties-common retry
+ sudo retry -d 50,10,30,300 -t 12 add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros
+ sudo apt update
+ DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin
+ - name: Clone sources
+ run: |
+ echo 'Importing sources...'
+ ./scripts/import-workspace.sh ./workspace
+ - name: Prepare rosdep keys
+ run: |
+ cat rosdep.yaml | tee -a local.yaml
+ echo >> local.yaml # ensure trailing newline
+ for PKG in $(catkin_topological_order --only-names workspace); do
+ printf "%s:\n %s:\n - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml
+ done
+ - name: Prepare jobs
+ id: worker
+ run: |
+ # TODO: implement check for currently existing package versions and reduced rebuild
+ ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }}
+ echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT
+ - name: Store jobs cache
+ uses: actions/cache/save@v4
+ with:
+ path: ${{ env.JOBS_YAML }}
+ key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+ - name: Prepare meta data cache
+ run: |
+ mkdir -p ${{ env.AGG }}
+ mv local.yaml ${{ env.AGG }}/local.yaml
+ cp sources.repos ${{ env.AGG }}/sources_specified.repos
+ mkdir -p ${{ env.AGG }}/.github/workflows
+ cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml
+ - name: Store meta data cache
+ uses: actions/cache/save@v4
+ with:
+ path: ${{ env.AGG }}
+ key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/aggregate-debs.yaml

@@ -5,14 +5,16 @@ on:
  type: number
  required: true
 
+env:
+ REPO: /home/runner/apt_repo
+ AGG: /home/runner/apt_repo_dependencies
+
 jobs:
  aggregate:
  runs-on: ubuntu-24.04
  env:
- SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
- REPO: /home/runner/apt_repo
- AGG: /home/runner/apt_repo_dependencies
  stage: stage${{ inputs.stage }}
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
  steps:
  - name: Check out the repo
  uses: actions/checkout@v4
@@ -30,103 +32,83 @@ jobs:
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker0
+ worker: ${{ env.stage }}-worker0
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker1
+ worker: ${{ env.stage }}-worker1
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker2
+ worker: ${{ env.stage }}-worker2
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker3
+ worker: ${{ env.stage }}-worker3
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker4
+ worker: ${{ env.stage }}-worker4
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker5
+ worker: ${{ env.stage }}-worker5
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker6
+ worker: ${{ env.stage }}-worker6
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker7
+ worker: ${{ env.stage }}-worker7
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker8
+ worker: ${{ env.stage }}-worker8
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker9
+ worker: ${{ env.stage }}-worker9
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker10
+ worker: ${{ env.stage }}-worker10
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker11
+ worker: ${{ env.stage }}-worker11
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker12
+ worker: ${{ env.stage }}-worker12
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker13
+ worker: ${{ env.stage }}-worker13
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker14
+ worker: ${{ env.stage }}-worker14
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker15
+ worker: ${{ env.stage }}-worker15
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker16
+ worker: ${{ env.stage }}-worker16
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker17
+ worker: ${{ env.stage }}-worker17
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker18
+ worker: ${{ env.stage }}-worker18
  - name: Aggregate worker
  uses: ./.github/actions/aggregate-worker
  with:
- stage: ${{ env.stage }}
- worker: worker19
+ worker: ${{ env.stage }}-worker19
  - name: Store apt repo
  uses: actions/cache/save@v4
  if: always()

.github/workflows/build.yaml

@@ -7,59 +7,21 @@ concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
 
 env:
- AGG: /home/runner/apt_repo_dependencies
  DISTRIBUTION: ubuntu
+ DEB_DISTRO: jammy
+ BRANCH: jammy-one-experimental
+ JOBS_YAML: /home/runner/jobs.yaml
+ AGG: /home/runner/apt_repo_dependencies
 
 jobs:
  stage-1:
  runs-on: ubuntu-24.04
  outputs:
- workers: ${{ steps.worker.outputs.workers }}
- env:
- JOBS_YAML: /home/runner/jobs.yaml
+ workers: ${{ steps.prepare.outputs.workers }}
  steps:
- - name: Check out the repo
- uses: actions/checkout@v4
- - name: Clone sources
- run: |
- echo 'Acquire::Retries "20";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
- echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
- echo 'Debug::Acquire::Retries "true";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
- sudo add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros
- sudo apt update
- DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin
- mkdir workspace
- echo 'Importing sources...'
- vcs import -w 5 --recursive --shallow --input sources.repos workspace
- - name: Prepare rosdep keys
- run: |
- cp rosdep.yaml local.yaml
- echo >> local.yaml # ensure trailing newline
- for PKG in $(catkin_topological_order --only-names workspace); do
- printf "%s:\n %s:\n - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml
- done
- - name: Prepare Jobs
- id: worker
- run: |
- ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }}
- echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT
- - name: Store jobs cache
- uses: actions/cache/save@v4
- with:
- path: ${{ env.JOBS_YAML }}
- key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
- - name: Prepare meta data cache
- run: |
- mkdir -p ${{ env.AGG }}
- mv local.yaml ${{ env.AGG }}/local.yaml
- cp sources.repos ${{ env.AGG }}/sources_specified.repos
- mkdir -p ${{ env.AGG }}/.github/workflows
- cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml
- - name: Store meta data cache
- uses: actions/cache/save@v4
- with:
- path: ${{ env.AGG }}
- key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+ - name: Prepare Pipeline
+ id: prepare
+ uses: ./.github/actions/prepare-worker-pipeline
 
  stage0-worker0:
  uses: ./.github/workflows/worker.yaml
@@ -1465,9 +1427,6 @@ jobs:
  needs: stage18
  if: always() && !cancelled()
  runs-on: ubuntu-24.04
- env:
- ROS_DISTRO: one
- DEB_DISTRO: jammy
  steps:
  - name: get apt packages from last job
  uses: actions/cache/restore@v4
@@ -1481,6 +1440,6 @@ jobs:
  mv ${{ env.AGG }} /home/runner/apt_repo
  - uses: v4hn/ros-deb-builder-action/deploy@roso-noble
  with:
- BRANCH: ${{ env.DEB_DISTRO }}-${{ env.ROS_DISTRO }}-unstable
+ BRANCH: ${{ env.BRANCH }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  SQUASH_HISTORY: true

.github/workflows/worker.yaml

@@ -7,14 +7,15 @@ on:
  type: string
  required: true
 
+env:
+ DEB_DISTRO: jammy
+ JOBS_YAML: /home/runner/jobs.yaml
+
 jobs:
  build:
  runs-on: ubuntu-24.04
  env:
- ROS_DISTRO: one
- DEB_DISTRO: jammy
  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
- JOBS_YAML: /home/runner/jobs.yaml
  steps:
  - name: Check out the repo
  uses: actions/checkout@v4
@@ -32,7 +33,7 @@ jobs:
  echo "Cache not found"
  exit 1
  fi
- - name: Use apt packages ${{ inputs.depends }}
+ - name: Use dependencies ${{ inputs.depends }}
  id: fetch-dependencies
  if: ${{ inputs.depends }}
  uses: actions/cache/restore@v4
@@ -69,8 +70,8 @@ jobs:
  uses: v4hn/ros-deb-builder-action/build@roso-noble
  timeout-minutes: 330
  with:
- ROS_DISTRO: ${{ env.ROS_DISTRO }}
  DEB_DISTRO: ${{ env.DEB_DISTRO }}
+ ROS_DISTRO: one
  SBUILD_CONF: "$extra_packages = ['/home/runner/apt_repo_dependencies']; $enable_network = 1; ${{ env.extra_sbuild_conf }}"
  ROSDEP_SOURCE: rosdep.yaml
  REPOS_FILE: worker-jobs.repos

scripts/import-workspace.sh

@@ -11,5 +11,5 @@ fi
 
 for f in $REPOS; do
  echo importing $f
- vcs import --shallow --recursive $WORKSPACE --input "$f" >/dev/null
+ vcs import -w 5 --shallow --recursive $WORKSPACE --input "$f" >/dev/null
 done