From 2bfa546d7b16c1e7d79224bd561a94493eac0402 Mon Sep 17 00:00:00 2001 From: v4hn Date: Sun, 13 Oct 2024 14:50:33 +0200 Subject: [PATCH] [WIP] refactor pipeline preparation --- .github/workflows/aggregate-debs.yaml | 4 +- .github/workflows/build.yaml | 65 +++-------------- .../workflows/prepare-worker-pipeline.yaml | 73 +++++++++++++++++++ .github/workflows/worker.yaml | 7 +- 4 files changed, 90 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/prepare-worker-pipeline.yaml diff --git a/.github/workflows/aggregate-debs.yaml b/.github/workflows/aggregate-debs.yaml index b8d379e34c..d7b62d246b 100644 --- a/.github/workflows/aggregate-debs.yaml +++ b/.github/workflows/aggregate-debs.yaml @@ -9,10 +9,10 @@ jobs: aggregate: runs-on: ubuntu-24.04 env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 + stage: stage${{ inputs.stage }} REPO: /home/runner/apt_repo AGG: /home/runner/apt_repo_dependencies - stage: stage${{ inputs.stage }} + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 steps: - name: Check out the repo uses: actions/checkout@v4 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e1fefd88bd..db85c5f9ef 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,59 +7,21 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} env: - AGG: /home/runner/apt_repo_dependencies DISTRIBUTION: ubuntu + DEB_DISTRO: jammy + BRANCH: ${{ format('{0}-one-experimental', env.DEB_DISTRO) }} + JOBS_YAML: /home/runner/jobs.yaml + AGG: /home/runner/apt_repo_dependencies jobs: stage-1: - runs-on: ubuntu-24.04 - outputs: - workers: ${{ steps.worker.outputs.workers }} - env: - JOBS_YAML: /home/runner/jobs.yaml - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - name: Clone sources - run: | - echo 'Acquire::Retries "20";' | sudo tee -a /etc/apt/apt.conf.d/80-retries - echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries - echo 'Debug::Acquire::Retries "true";' | sudo tee -a /etc/apt/apt.conf.d/80-retries - sudo add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros - sudo apt update - DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin - mkdir workspace - echo 'Importing sources...' - vcs import -w 5 --recursive --shallow --input sources.repos workspace - - name: Prepare rosdep keys - run: | - cp rosdep.yaml local.yaml - echo >> local.yaml # ensure trailing newline - for PKG in $(catkin_topological_order --only-names workspace); do - printf "%s:\n %s:\n - %s\n" "$PKG" "${{ env.DISTRIBUTION }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml - done - - name: Prepare Jobs - id: worker - run: | - ./scripts/generate-jobs.py workspace | tee ${{ env.JOBS_YAML }} - echo "workers=$(cat ${{ env.JOBS_YAML }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT - - name: Store jobs cache - uses: actions/cache/save@v4 - with: - path: ${{ env.JOBS_YAML }} - key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} - - name: Prepare meta data cache - run: | - mkdir -p ${{ env.AGG }} - mv local.yaml ${{ env.AGG }}/local.yaml - cp sources.repos ${{ env.AGG }}/sources_specified.repos - mkdir -p ${{ env.AGG }}/.github/workflows - cp .github/workflows/sync-unstable.yaml ${{ env.AGG }}/.github/workflows/sync-unstable.yaml - - name: Store meta data cache - uses: actions/cache/save@v4 - with: - path: ${{ env.AGG }} - key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + uses: ./.github/workflows/prepare-worker-pipeline.yaml + if: always() && !cancelled() + with: + distribution: ${{ env.DISTRIBUTION }} + repository-branch: ${{ env.BRANCH }} + jobs_yaml: ${{ env.JOBS_YAML }} + agg: ${{ env.AGG }} stage0-worker0: uses: ./.github/workflows/worker.yaml @@ -1465,9 +1427,6 @@ jobs: needs: stage18 if: always() && !cancelled() runs-on: ubuntu-24.04 - env: - ROS_DISTRO: one - DEB_DISTRO: jammy steps: - name: get apt packages from last job uses: actions/cache/restore@v4 @@ -1481,6 +1440,6 @@ jobs: mv ${{ env.AGG }} /home/runner/apt_repo - uses: v4hn/ros-deb-builder-action/deploy@roso-noble with: - BRANCH: ${{ env.DEB_DISTRO }}-${{ env.ROS_DISTRO }}-unstable + BRANCH: ${{ env.BRANCH }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SQUASH_HISTORY: true diff --git a/.github/workflows/prepare-worker-pipeline.yaml b/.github/workflows/prepare-worker-pipeline.yaml new file mode 100644 index 0000000000..5e2878aa13 --- /dev/null +++ b/.github/workflows/prepare-worker-pipeline.yaml @@ -0,0 +1,73 @@ +on: + workflow_call: + inputs: + distribution: + type: string + description: "distribution used in rosdep (usually 'debian' or 'ubuntu')" + required: true + # TODO: implement this check + repository-branch: + description: "branch containing the latest debs - used to determine which packages need to be rebuilt" + type: string + default: "" + jobs_yaml: + description: "where to save the jobs yaml" + type: string + required: true + agg: + description: "where are build artifacts aggregated" + type: string + required: true + outputs: + workers: + description: "colon-separated list of workers used in the specification" + value: ${{ jobs.prepare-pipeline.outputs.workers }} + +jobs: + prepare-pipeline: + runs-on: ubuntu-24.04 + outputs: + workers: ${{ steps.worker.outputs.workers }} + steps: + - name: Check out the repo + uses: actions/checkout@v4 + - name: Clone sources + run: | + echo 'Acquire::Retries "20";' | sudo tee -a /etc/apt/apt.conf.d/80-retries + echo 'Acquire::Retries::Delay::Maximum "300";' | sudo tee -a /etc/apt/apt.conf.d/80-retries + echo 'Debug::Acquire::Retries "true";' | sudo tee -a /etc/apt/apt.conf.d/80-retries + sudo add-apt-repository -y ppa:v-launchpad-jochen-sprickerhof-de/ros + sudo apt update + DEBIAN_FRONTEND=noninteractive sudo apt install -y vcstool catkin + mkdir workspace + echo 'Importing sources...' + vcs import -w 5 --recursive --shallow --input sources.repos workspace + - name: Prepare rosdep keys + run: | + cp rosdep.yaml local.yaml + echo >> local.yaml # ensure trailing newline + for PKG in $(catkin_topological_order --only-names workspace); do + printf "%s:\n %s:\n - %s\n" "$PKG" "${{ inputs.distribution }}" "ros-one-$(printf '%s' "$PKG" | tr '_' '-')" | tee -a local.yaml + done + - name: Prepare Jobs + id: worker + run: | + ./scripts/generate-jobs.py workspace | tee ${{ inputs.jobs_yaml }} + echo "workers=$(cat ${{ inputs.jobs_yaml }} | sed -n '/^stage.*:$/ p' | tr -d '\n')" >> $GITHUB_OUTPUT + - name: Store jobs cache + uses: actions/cache/save@v4 + with: + path: ${{ inputs.jobs_yaml }} + key: jobs-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Prepare meta data cache + run: | + mkdir -p ${{ inputs.agg }} + mv local.yaml ${{ inputs.agg }}/local.yaml + cp sources.repos ${{ inputs.agg }}/sources_specified.repos + mkdir -p ${{ inputs.agg }}/.github/workflows + cp .github/workflows/sync-unstable.yaml ${{ inputs.agg }}/.github/workflows/sync-unstable.yaml + - name: Store meta data cache + uses: actions/cache/save@v4 + with: + path: ${{ inputs.agg }} + key: apt-repo-stage-1-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/worker.yaml b/.github/workflows/worker.yaml index 5d704b5f5b..faf191adfe 100644 --- a/.github/workflows/worker.yaml +++ b/.github/workflows/worker.yaml @@ -11,10 +11,9 @@ jobs: build: runs-on: ubuntu-24.04 env: - ROS_DISTRO: one DEB_DISTRO: jammy - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 JOBS_YAML: /home/runner/jobs.yaml + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 steps: - name: Check out the repo uses: actions/checkout@v4 @@ -32,7 +31,7 @@ jobs: echo "Cache not found" exit 1 fi - - name: Use apt packages ${{ inputs.depends }} + - name: Use dependencies ${{ inputs.depends }} id: fetch-dependencies if: ${{ inputs.depends }} uses: actions/cache/restore@v4 @@ -69,8 +68,8 @@ jobs: uses: v4hn/ros-deb-builder-action/build@roso-noble timeout-minutes: 330 with: - ROS_DISTRO: ${{ env.ROS_DISTRO }} DEB_DISTRO: ${{ env.DEB_DISTRO }} + ROS_DISTRO: one SBUILD_CONF: "$extra_packages = ['/home/runner/apt_repo_dependencies']; $enable_network = 1; ${{ env.extra_sbuild_conf }}" ROSDEP_SOURCE: rosdep.yaml REPOS_FILE: worker-jobs.repos