refactor pipeline preparation

v4hn · Oct 14, 2024 · 37bf994 · 37bf994
1 parent 5952097
commit 37bf994
Show file tree

Hide file tree

Showing 7 changed files with 132 additions and 152 deletions.
diff --git a/.github/actions/aggregate-worker/action.yaml b/.github/actions/aggregate-worker/action.yaml
@@ -1,9 +1,6 @@
 name: 'Aggregate Worker'
 description: 'Aggregate files from a worker'
 inputs:
-  stage:
-    type: string
-    required: true
   worker:
     type: string
     required: true
@@ -16,19 +13,19 @@ runs:
       uses: actions/cache/restore@v4
       with:
         path: ${{ env.REPO }}
-        key: apt-repo-${{ inputs.stage }}-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+        key: apt-repo-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
         restore-keys: |
-          apt-repo-${{ inputs.stage }}-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}
+          apt-repo-${{ inputs.worker }}-${{ github.sha }}-${{ github.run_id }}
     - name: Aggregate files
       shell: bash
       if: steps.restore.outputs.cache-matched-key != ''
       run: |
         cat ${{ env.REPO }}/pkg_build_status.csv >> ${{ env.AGG }}/pkg_build_status.csv
-        cat ${{ env.REPO }}/sources.repos |
-          if [ -f ${{ env.AGG }}/sources.repos ]; then
+        cat ${{ env.REPO }}/sources_exact.repos |
+          if [ -f ${{ env.AGG }}/sources_exact.repos ]; then
             tail -n+2 # skip "repositories:"
           else
             cat
-          fi | tee -a ${{ env.AGG }}/sources.repos
-        rm -f ${{ env.REPO }}/pkg_build_status.csv ${{ env.REPO }}/sources.repos ${{ env.REPO }}/local.yaml
+          fi | tee -a ${{ env.AGG }}/sources_exact.repos
+        rm -f ${{ env.REPO }}/pkg_build_status.csv ${{ env.REPO }}/sources_exact.repos ${{ env.REPO }}/local.yaml
         mv ${{ env.REPO }}/* ${{ env.AGG }}
diff --git a/.github/actions/prepare-worker-pipeline/action.yaml b/.github/actions/prepare-worker-pipeline/action.yaml
@@ -0,0 +1,62 @@
+name: 'Prepare Worker Pipeline'
+description: 'Initializes staged build pipeline with job scheduling'
+
+outputs:
+  workers:
+    description: 'colon-separated list of workers used in the specification'
+    value: ${{ steps.worker.outputs.workers }}
+
+runs:
+  using: "composite"
+  steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Prepare environment
+        shell: bash
+        run: |
+          echo 'Acquire::Retries "20";'                  | sudo tee -a /etc/apt/apt.conf.d/80-retries
+          echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
+          echo 'Debug::Acquire::Retries "true";'         | sudo tee -a /etc/apt/apt.conf.d/80-retries
+          sudo apt update
+          DEBIAN_FRONTEND=noninteractive sudo apt install -y software-properties-common retry
+          sudo retry -d 50,10,30,300 -t 12 add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros
+          sudo apt update
+          DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin
+      - name: Clone sources
+        shell: bash
+        run: |
+          echo 'Importing sources...'
+          ./scripts/import-workspace.sh
+      - name: Prepare rosdep keys
+        shell: bash
+        run: |
+          cat rosdep.yaml | tee -a local.yaml
+          echo >> local.yaml # ensure trailing newline
+          for PKG in $(catkin_topological_order --only-names workspace); do
+            printf "%s:\n  %s:\n  - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml
+          done
+      - name: Prepare jobs
+        shell: bash
+        id: worker
+        run: |
+          # TODO: implement check for currently existing package versions and reduced rebuild
+          ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }}
+          echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT
+      - name: Store jobs cache
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.JOBS_YAML }}
+          key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+      - name: Prepare meta data cache
+        shell: bash
+        run: |
+          mkdir -p ${{ env.AGG }}
+          mv local.yaml ${{ env.AGG }}/local.yaml
+          cp sources.repos ${{ env.AGG }}/sources_specified.repos
+          mkdir -p ${{ env.AGG }}/.github/workflows
+          cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml
+      - name: Store meta data cache
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.AGG }}
+          key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/aggregate-debs.yaml b/.github/workflows/aggregate-debs.yaml
@@ -5,14 +5,16 @@ on:
         type: number
         required: true
 
+env:
+  REPO: /home/runner/apt_repo
+  AGG: /home/runner/apt_repo_dependencies
+
 jobs:
   aggregate:
     runs-on: ubuntu-24.04
     env:
-      SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
-      REPO: /home/runner/apt_repo
-      AGG: /home/runner/apt_repo_dependencies
       stage: stage${{ inputs.stage }}
+      SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -30,103 +32,83 @@ jobs:
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker0
+          worker: ${{ env.stage }}-worker0
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker1
+          worker: ${{ env.stage }}-worker1
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker2
+          worker: ${{ env.stage }}-worker2
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker3
+          worker: ${{ env.stage }}-worker3
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker4
+          worker: ${{ env.stage }}-worker4
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker5
+          worker: ${{ env.stage }}-worker5
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker6
+          worker: ${{ env.stage }}-worker6
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker7
+          worker: ${{ env.stage }}-worker7
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker8
+          worker: ${{ env.stage }}-worker8
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker9
+          worker: ${{ env.stage }}-worker9
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker10
+          worker: ${{ env.stage }}-worker10
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker11
+          worker: ${{ env.stage }}-worker11
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker12
+          worker: ${{ env.stage }}-worker12
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker13
+          worker: ${{ env.stage }}-worker13
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker14
+          worker: ${{ env.stage }}-worker14
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker15
+          worker: ${{ env.stage }}-worker15
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker16
+          worker: ${{ env.stage }}-worker16
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker17
+          worker: ${{ env.stage }}-worker17
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker18
+          worker: ${{ env.stage }}-worker18
       - name: Aggregate worker
         uses: ./.github/actions/aggregate-worker
         with:
-          stage: ${{ env.stage }}
-          worker: worker19
+          worker: ${{ env.stage }}-worker19
       - name: Store apt repo
         uses: actions/cache/save@v4
         if: always()

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -10,59 +10,31 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
 
 env:
-  AGG: /home/runner/apt_repo_dependencies
   DISTRIBUTION: ubuntu
+  DEB_DISTRO: jammy
+  BRANCH: jammy-one-unstable
+  JOBS_YAML: /home/runner/jobs.yaml
+  AGG: /home/runner/apt_repo_dependencies
 
 jobs:
   stage-1:
     runs-on: ubuntu-24.04
     outputs:
-      workers: ${{ steps.worker.outputs.workers }}
-    env:
-      JOBS_YAML: /home/runner/jobs.yaml
+      workers: ${{ steps.prepare.outputs.workers }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
-      - name: Clone sources
+      - name: Prepare Pipeline
+        id: prepare
+        uses: ./.github/actions/prepare-worker-pipeline
+      - name: rosdep keys
+        shell: bash
         run: |
-          echo 'Acquire::Retries "20";'                  | sudo tee -a /etc/apt/apt.conf.d/80-retries
-          echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
-          echo 'Debug::Acquire::Retries "true";'         | sudo tee -a /etc/apt/apt.conf.d/80-retries
-          sudo add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros
-          sudo apt update
-          DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin
-          mkdir workspace
-          echo 'Importing sources...'
-          vcs import -w 5 --recursive --shallow --input sources.repos workspace
-      - name: Prepare rosdep keys
+          cat ${{ env.AGG }}/local.yaml
+      - name: jobs
+        shell: bash
         run: |
-          cp rosdep.yaml local.yaml
-          echo >> local.yaml # ensure trailing newline
-          for PKG in $(catkin_topological_order --only-names workspace); do
-            printf "%s:\n  %s:\n  - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml
-          done
-      - name: Prepare Jobs
-        id: worker
-        run: |
-          ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }}
-          echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT
-      - name: Store jobs cache
-        uses: actions/cache/save@v4
-        with:
-          path: ${{ env.JOBS_YAML }}
-          key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
-      - name: Prepare meta data cache
-        run: |
-          mkdir -p ${{ env.AGG }}
-          mv local.yaml ${{ env.AGG }}/local.yaml
-          cp sources.repos ${{ env.AGG }}/sources_specified.repos
-          mkdir -p ${{ env.AGG }}/.github/workflows
-          cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml
-      - name: Store meta data cache
-        uses: actions/cache/save@v4
-        with:
-          path: ${{ env.AGG }}
-          key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          cat ${{ env.JOBS_YAML }}
 
   stage0-worker0:
     uses: ./.github/workflows/worker.yaml
@@ -1468,9 +1440,6 @@ jobs:
     needs: stage18
     if: always() && !cancelled()
     runs-on: ubuntu-24.04
-    env:
-      ROS_DISTRO: one
-      DEB_DISTRO: jammy
     steps:
       - name: get apt packages from last job
         uses: actions/cache/restore@v4
@@ -1484,6 +1453,6 @@ jobs:
           mv ${{ env.AGG }} /home/runner/apt_repo
       - uses: v4hn/ros-deb-builder-action/deploy@roso-noble
         with:
-          BRANCH: ${{ env.DEB_DISTRO }}-${{ env.ROS_DISTRO }}-unstable
+          BRANCH: ${{ env.BRANCH }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SQUASH_HISTORY: true
diff --git a/.github/workflows/worker.yaml b/.github/workflows/worker.yaml
@@ -7,14 +7,15 @@ on:
         type: string
         required: true
 
+env:
+  DEB_DISTRO: jammy
+  JOBS_YAML: /home/runner/jobs.yaml
+
 jobs:
   build:
     runs-on: ubuntu-24.04
     env:
-      ROS_DISTRO: one
-      DEB_DISTRO: jammy
       SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
-      JOBS_YAML: /home/runner/jobs.yaml
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -32,7 +33,7 @@ jobs:
             echo "Cache not found"
             exit 1
           fi
-      - name: Use apt packages ${{ inputs.depends }}
+      - name: Use dependencies ${{ inputs.depends }}
         id: fetch-dependencies
         if: ${{ inputs.depends }}
         uses: actions/cache/restore@v4
@@ -69,8 +70,8 @@ jobs:
         uses: v4hn/ros-deb-builder-action/build@roso-noble
         timeout-minutes: 330
         with:
-          ROS_DISTRO: ${{ env.ROS_DISTRO }}
           DEB_DISTRO: ${{ env.DEB_DISTRO }}
+          ROS_DISTRO: one
           SBUILD_CONF: "$extra_packages = ['/home/runner/apt_repo_dependencies']; $enable_network = 1; ${{ env.extra_sbuild_conf }}"
           ROSDEP_SOURCE: rosdep.yaml
           REPOS_FILE: worker-jobs.repos