[WIP] refactor pipeline preparation

v4hn · Oct 13, 2024 · 2bfa546 · 2bfa546
1 parent ca31636
commit 2bfa546
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 59 deletions.
diff --git a/.github/workflows/aggregate-debs.yaml b/.github/workflows/aggregate-debs.yaml
@@ -9,10 +9,10 @@ jobs:
   aggregate:
     runs-on: ubuntu-24.04
     env:
-      SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
+      stage: stage${{ inputs.stage }}
       REPO: /home/runner/apt_repo
       AGG: /home/runner/apt_repo_dependencies
-      stage: stage${{ inputs.stage }}
+      SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -7,59 +7,21 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
 
 env:
-  AGG: /home/runner/apt_repo_dependencies
   DISTRIBUTION: ubuntu
+  DEB_DISTRO: jammy
+  BRANCH: ${{ format('{0}-one-experimental', env.DEB_DISTRO) }}
+  JOBS_YAML: /home/runner/jobs.yaml
+  AGG: /home/runner/apt_repo_dependencies
 
 jobs:
   stage-1:
-    runs-on: ubuntu-24.04
-    outputs:
-      workers: ${{ steps.worker.outputs.workers }}
-    env:
-      JOBS_YAML: /home/runner/jobs.yaml
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-      - name: Clone sources
-        run: |
-          echo 'Acquire::Retries "20";'                  | sudo tee -a /etc/apt/apt.conf.d/80-retries
-          echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
-          echo 'Debug::Acquire::Retries "true";'         | sudo tee -a /etc/apt/apt.conf.d/80-retries
-          sudo add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros
-          sudo apt update
-          DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin
-          mkdir workspace
-          echo 'Importing sources...'
-          vcs import -w 5 --recursive --shallow --input sources.repos workspace
-      - name: Prepare rosdep keys
-        run: |
-          cp rosdep.yaml local.yaml
-          echo >> local.yaml # ensure trailing newline
-          for PKG in $(catkin_topological_order --only-names workspace); do
-            printf "%s:\n  %s:\n  - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml
-          done
-      - name: Prepare Jobs
-        id: worker
-        run: |
-          ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }}
-          echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT
-      - name: Store jobs cache
-        uses: actions/cache/save@v4
-        with:
-          path: ${{ env.JOBS_YAML }}
-          key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
-      - name: Prepare meta data cache
-        run: |
-          mkdir -p ${{ env.AGG }}
-          mv local.yaml ${{ env.AGG }}/local.yaml
-          cp sources.repos ${{ env.AGG }}/sources_specified.repos
-          mkdir -p ${{ env.AGG }}/.github/workflows
-          cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml
-      - name: Store meta data cache
-        uses: actions/cache/save@v4
-        with:
-          path: ${{ env.AGG }}
-          key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+    uses: ./.github/workflows/prepare-worker-pipeline.yaml
+    if: always() && !cancelled()
+    with:
+      distribution: ${{ env.DISTRIBUTION }}
+      repository-branch: ${{ env.BRANCH }}
+      jobs_yaml: ${{ env.JOBS_YAML }}
+      agg: ${{ env.AGG }}
 
   stage0-worker0:
     uses: ./.github/workflows/worker.yaml
@@ -1465,9 +1427,6 @@ jobs:
     needs: stage18
     if: always() && !cancelled()
     runs-on: ubuntu-24.04
-    env:
-      ROS_DISTRO: one
-      DEB_DISTRO: jammy
     steps:
       - name: get apt packages from last job
         uses: actions/cache/restore@v4
@@ -1481,6 +1440,6 @@ jobs:
           mv ${{ env.AGG }} /home/runner/apt_repo
       - uses: v4hn/ros-deb-builder-action/deploy@roso-noble
         with:
-          BRANCH: ${{ env.DEB_DISTRO }}-${{ env.ROS_DISTRO }}-unstable
+          BRANCH: ${{ env.BRANCH }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SQUASH_HISTORY: true
diff --git a/.github/workflows/prepare-worker-pipeline.yaml b/.github/workflows/prepare-worker-pipeline.yaml
@@ -0,0 +1,73 @@
+on:
+  workflow_call:
+    inputs:
+      distribution:
+        type: string
+        description: "distribution used in rosdep (usually 'debian' or 'ubuntu')"
+        required: true
+      # TODO: implement this check
+      repository-branch:
+        description: "branch containing the latest debs - used to determine which packages need to be rebuilt"
+        type: string
+        default: ""
+      jobs_yaml:
+        description: "where to save the jobs yaml"
+        type: string
+        required: true
+      agg:
+        description: "where are build artifacts aggregated"
+        type: string
+        required: true
+    outputs:
+      workers:
+        description: "colon-separated list of workers used in the specification"
+        value: ${{ jobs.prepare-pipeline.outputs.workers }}
+
+jobs:
+  prepare-pipeline:
+    runs-on: ubuntu-24.04
+    outputs:
+      workers: ${{ steps.worker.outputs.workers }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Clone sources
+        run: |
+          echo 'Acquire::Retries "20";'                  | sudo tee -a /etc/apt/apt.conf.d/80-retries
+          echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries
+          echo 'Debug::Acquire::Retries "true";'         | sudo tee -a /etc/apt/apt.conf.d/80-retries
+          sudo add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros
+          sudo apt update
+          DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin
+          mkdir workspace
+          echo 'Importing sources...'
+          vcs import -w 5 --recursive --shallow --input sources.repos workspace
+      - name: Prepare rosdep keys
+        run: |
+          cp rosdep.yaml local.yaml
+          echo >> local.yaml # ensure trailing newline
+          for PKG in $(catkin_topological_order --only-names workspace); do
+            printf "%s:\n  %s:\n  - %s\n" "$PKG" "${{ inputs.distribution }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml
+          done
+      - name: Prepare Jobs
+        id: worker
+        run: |
+          ./scripts/generate-jobs.py workspace | tee ${{ inputs.jobs_yaml }}
+          echo "workers=$(cat ${{ inputs.jobs_yaml }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT
+      - name: Store jobs cache
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ inputs.jobs_yaml }}
+          key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+      - name: Prepare meta data cache
+        run: |
+          mkdir -p ${{ inputs.agg }}
+          mv local.yaml ${{ inputs.agg }}/local.yaml
+          cp sources.repos ${{ inputs.agg }}/sources_specified.repos
+          mkdir -p ${{ inputs.agg }}/.github/workflows
+          cp .github/workflows/sync-unstable.yaml ${{ inputs.agg }}/.github/workflows/sync-unstable.yaml
+      - name: Store meta data cache
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ inputs.agg }}
+          key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/worker.yaml b/.github/workflows/worker.yaml
@@ -11,10 +11,9 @@ jobs:
   build:
     runs-on: ubuntu-24.04
     env:
-      ROS_DISTRO: one
       DEB_DISTRO: jammy
-      SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
       JOBS_YAML: /home/runner/jobs.yaml
+      SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -32,7 +31,7 @@ jobs:
             echo "Cache not found"
             exit 1
           fi
-      - name: Use apt packages ${{ inputs.depends }}
+      - name: Use dependencies ${{ inputs.depends }}
         id: fetch-dependencies
         if: ${{ inputs.depends }}
         uses: actions/cache/restore@v4
@@ -69,8 +68,8 @@ jobs:
         uses: v4hn/ros-deb-builder-action/build@roso-noble
         timeout-minutes: 330
         with:
-          ROS_DISTRO: ${{ env.ROS_DISTRO }}
           DEB_DISTRO: ${{ env.DEB_DISTRO }}
+          ROS_DISTRO: one
           SBUILD_CONF: "$extra_packages = ['/home/runner/apt_repo_dependencies']; $enable_network = 1; ${{ env.extra_sbuild_conf }}"
           ROSDEP_SOURCE: rosdep.yaml
           REPOS_FILE: worker-jobs.repos