From 685b047ae7a402c84e64f07cdf7024ad74d1bd64 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 12 Mar 2024 13:48:33 +0600 Subject: [PATCH 1/4] chore: add workflow to check package dependencies This commit adds a new reusable workflow which can be run on Pull Requests to see if the packages listed in "essential" of a slice are the true dependencies of the slice package. This workflow will add a PR comment if it finds any errors. --- .github/workflows/pkg-deps.yaml | 102 ++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/pkg-deps.yaml diff --git a/.github/workflows/pkg-deps.yaml b/.github/workflows/pkg-deps.yaml new file mode 100644 index 000000000..3e0d109f5 --- /dev/null +++ b/.github/workflows/pkg-deps.yaml @@ -0,0 +1,102 @@ +name: Package dependencies + +on: + workflow_call: + +jobs: + check-dependency: + name: Check dependency + runs-on: ubuntu-latest + if: | + github.event_name == 'pull_request' && + startswith(github.base_ref, 'ubuntu-') + env: + branch: ${{ github.base_ref }} + msg_file: dependencies.message + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v4 + + - name: Check changed paths + id: changed-paths + uses: dorny/paths-filter@v3 + with: + # ref: https://github.com/marketplace/actions/paths-changes-filter + filters: | + slices: + - added|modified: 'slices/**/*.yaml' + # Space delimited list usable as command-line argument list in + # Linux shell. If needed, it uses single or double quotes to + # wrap filename with unsafe characters. + list-files: shell + + - name: Check dependencies + id: check-deps + if: steps.changed-paths.outputs.slices == 'true' + run: | + set -e + + export LC_COLLATE=C + + version=$(echo "$branch" | grep -Eo '[0-9.]+') + docker run -i -d --rm --name ubuntu ubuntu:"$version" + docker exec ubuntu apt-get update + + cleanup() { + docker stop ubuntu + } + trap cleanup EXIT + + echo "Slice package dependency issues:" > "$msg_file" + for f in ${{ steps.changed-paths.outputs.slices_files }}; do + echo "Processing $f.." >&2 + pkg=$(yq '.package' "$f") + + mapfile -t deps < <( + docker exec ubuntu apt depends \ + --no-recommends --no-suggests --no-conflicts \ + --no-breaks --no-replaces --no-enhances \ + "$pkg" 2>/dev/null | \ + sed -nr 's/.*Depends:\s(\S*).*/\1/p' | \ + sed 's///; s/:any//' | \ + sort | uniq + ) + + mapfile -t listed < <( + yq '.slices.[].essential[]' "$f" | \ + sed "s/_.*//; /^$pkg$/d" | sort | uniq + ) + + errs=() + for p in "${listed[@]}"; do + if printf '%s\0' "${deps[@]}" | grep -Fxqz -- "$p"; then + continue + fi + errs+=("$p") + done + + if (( "${#errs[@]}")); then + echo "
" >> "$msg_file" + echo -e "$f\n" >> "$msg_file" + for e in "${errs[@]}"; do + echo " - \`$e\` is listed in \`essential\`, but is not a dependency." >> "$msg_file" + done + echo -e "\n
" >> "$msg_file" + fi + done + echo -e "\n---" >> "$msg_file" + cat "$msg_file" + + - name: Craft default message if no errors + run: | + set -e + if [[ ! -f "$msg_file" ]]; then + echo "Slice package dependency issues:" > "$msg_file" + echo -e "\tNone found." >> "$msg_file" + fi + + - name: Post messages to PR + uses: mshick/add-pr-comment@v2 + with: + message-path: ${{ env.msg_file }} From f5df44a4e1d4fdae6e3e5bf7dc2cfc915eb704e5 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 13 Mar 2024 12:56:05 +0600 Subject: [PATCH 2/4] chore: move pkg-deps script to a standalone file --- .github/scripts/pkg-deps/pkg-deps | 71 +++++++++++++++++++++++++++++ .github/workflows/pkg-deps.yaml | 76 ++++++------------------------- 2 files changed, 84 insertions(+), 63 deletions(-) create mode 100755 .github/scripts/pkg-deps/pkg-deps diff --git a/.github/scripts/pkg-deps/pkg-deps b/.github/scripts/pkg-deps/pkg-deps new file mode 100755 index 000000000..5e9489c04 --- /dev/null +++ b/.github/scripts/pkg-deps/pkg-deps @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +export LC_COLLATE=C + +if [[ -z "$branch" ]]; then + echo "error: no branch specified" >&2 + exit 1 +fi + +version=$(echo "$branch" | grep -Eo '[0-9.]+') +docker run -i -d --rm --name ubuntu ubuntu:"$version" >&2 + +cleanup() { + docker rm -f ubuntu >&2 +} +trap cleanup EXIT + +docker exec ubuntu apt-get update >&2 + +msg_file="${msg_file:-$(mktemp)}" +echo "Writing dependencies diff to $msg_file" >&2 +if [[ -n "$GITHUB_OUTPUT" ]]; then + echo "msg_file=$msg_file" >> $GITHUB_OUTPUT +fi + +echo "Diff of dependencies:" > "$msg_file" +for f in $@; do + echo "Processing $f.." >&2 + pkg=$(yq '.package' "$f") + + mapfile -t deps < <( + docker exec ubuntu apt depends \ + --no-recommends --no-suggests --no-conflicts \ + --no-breaks --no-replaces --no-enhances \ + "$pkg" 2>/dev/null | \ + sed -nr 's/.*Depends:\s(\S*).*/\1/p' | \ + sed 's///; s/:any//' | \ + sort | uniq + ) + + mapfile -t listed < <( + yq '.slices.[].essential[]' "$f" | \ + sed "s/_.*//; /^$pkg$/d" | sort | uniq + ) + + diffs=() + for p in "${listed[@]}"; do + if printf '%s\0' "${deps[@]}" | grep -Fxqz -- "$p"; then + continue + fi + diffs+=("$p") + done + + if (( "${#diffs[@]}")); then + echo "
" >> "$msg_file" + echo -e "$f\n" >> "$msg_file" + for e in "${diffs[@]}"; do + echo " - \`$e\` is listed in \`essential\`, but is not a dependency." >> "$msg_file" + done + echo -e "\n
" >> "$msg_file" + fi +done + +if ! grep "" "$msg_file"; then + echo -e "\tNone found." >> "$msg_file" +fi + +echo -e "\n---" >> "$msg_file" +cat "$msg_file" diff --git a/.github/workflows/pkg-deps.yaml b/.github/workflows/pkg-deps.yaml index 3e0d109f5..c5f04e795 100644 --- a/.github/workflows/pkg-deps.yaml +++ b/.github/workflows/pkg-deps.yaml @@ -12,7 +12,7 @@ jobs: startswith(github.base_ref, 'ubuntu-') env: branch: ${{ github.base_ref }} - msg_file: dependencies.message + main-branch-path: files-from-main permissions: pull-requests: write steps: @@ -31,72 +31,22 @@ jobs: # wrap filename with unsafe characters. list-files: shell + - name: Checkout main branch + uses: actions/checkout@v4 + with: + ref: main + path: ${{ env.main-branch-path }} + - name: Check dependencies id: check-deps - if: steps.changed-paths.outputs.slices == 'true' - run: | - set -e - - export LC_COLLATE=C - - version=$(echo "$branch" | grep -Eo '[0-9.]+') - docker run -i -d --rm --name ubuntu ubuntu:"$version" - docker exec ubuntu apt-get update - - cleanup() { - docker stop ubuntu - } - trap cleanup EXIT - - echo "Slice package dependency issues:" > "$msg_file" - for f in ${{ steps.changed-paths.outputs.slices_files }}; do - echo "Processing $f.." >&2 - pkg=$(yq '.package' "$f") - - mapfile -t deps < <( - docker exec ubuntu apt depends \ - --no-recommends --no-suggests --no-conflicts \ - --no-breaks --no-replaces --no-enhances \ - "$pkg" 2>/dev/null | \ - sed -nr 's/.*Depends:\s(\S*).*/\1/p' | \ - sed 's///; s/:any//' | \ - sort | uniq - ) - - mapfile -t listed < <( - yq '.slices.[].essential[]' "$f" | \ - sed "s/_.*//; /^$pkg$/d" | sort | uniq - ) - - errs=() - for p in "${listed[@]}"; do - if printf '%s\0' "${deps[@]}" | grep -Fxqz -- "$p"; then - continue - fi - errs+=("$p") - done - - if (( "${#errs[@]}")); then - echo "
" >> "$msg_file" - echo -e "$f\n" >> "$msg_file" - for e in "${errs[@]}"; do - echo " - \`$e\` is listed in \`essential\`, but is not a dependency." >> "$msg_file" - done - echo -e "\n
" >> "$msg_file" - fi - done - echo -e "\n---" >> "$msg_file" - cat "$msg_file" - - - name: Craft default message if no errors + env: + script-dir: "${{ env.main-branch-path }}/.github/scripts/install_slices" run: | - set -e - if [[ ! -f "$msg_file" ]]; then - echo "Slice package dependency issues:" > "$msg_file" - echo -e "\tNone found." >> "$msg_file" - fi + set -ex + ./${{ env.script-dir }}/pkg-deps \ + ${{ steps.changed-paths.outputs.slices_files }} - name: Post messages to PR uses: mshick/add-pr-comment@v2 with: - message-path: ${{ env.msg_file }} + message-path: ${{ steps.check-deps.outputs.msg_file }} From bc4b2e1ae925c0c196c17471f891f881d54f533b Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 13 Mar 2024 13:07:43 +0600 Subject: [PATCH 3/4] chore: fix path of env variable --- .github/workflows/pkg-deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pkg-deps.yaml b/.github/workflows/pkg-deps.yaml index c5f04e795..781bff330 100644 --- a/.github/workflows/pkg-deps.yaml +++ b/.github/workflows/pkg-deps.yaml @@ -40,7 +40,7 @@ jobs: - name: Check dependencies id: check-deps env: - script-dir: "${{ env.main-branch-path }}/.github/scripts/install_slices" + script-dir: "${{ env.main-branch-path }}/.github/scripts/pkg-deps" run: | set -ex ./${{ env.script-dir }}/pkg-deps \ From a011766c35308ef500f0fad78316e6a6af8ab217 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 13 Mar 2024 16:03:55 +0600 Subject: [PATCH 4/4] chore: show git style diff --- .github/scripts/pkg-deps/pkg-deps | 47 +++++++++++++------------------ 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/.github/scripts/pkg-deps/pkg-deps b/.github/scripts/pkg-deps/pkg-deps index 5e9489c04..c4cff2310 100755 --- a/.github/scripts/pkg-deps/pkg-deps +++ b/.github/scripts/pkg-deps/pkg-deps @@ -25,40 +25,31 @@ if [[ -n "$GITHUB_OUTPUT" ]]; then echo "msg_file=$msg_file" >> $GITHUB_OUTPUT fi -echo "Diff of dependencies:" > "$msg_file" +echo -e "Diff of dependencies:\n" > "$msg_file" for f in $@; do echo "Processing $f.." >&2 pkg=$(yq '.package' "$f") - mapfile -t deps < <( - docker exec ubuntu apt depends \ - --no-recommends --no-suggests --no-conflicts \ - --no-breaks --no-replaces --no-enhances \ - "$pkg" 2>/dev/null | \ - sed -nr 's/.*Depends:\s(\S*).*/\1/p' | \ - sed 's///; s/:any//' | \ - sort | uniq - ) - - mapfile -t listed < <( - yq '.slices.[].essential[]' "$f" | \ - sed "s/_.*//; /^$pkg$/d" | sort | uniq - ) - - diffs=() - for p in "${listed[@]}"; do - if printf '%s\0' "${deps[@]}" | grep -Fxqz -- "$p"; then - continue - fi - diffs+=("$p") - done - - if (( "${#diffs[@]}")); then + fupstream="$(mktemp)" + docker exec ubuntu apt depends \ + --no-recommends --no-suggests --no-conflicts \ + --no-breaks --no-replaces --no-enhances \ + "$pkg" 2>/dev/null | \ + sed -nr 's/.*Depends:\s(\S*).*/\1/p' | \ + sed 's///; s/:any//' | \ + sort | uniq > "$fupstream" + + flocal="$(mktemp)" + yq '.slices.[].essential[]' "$f" | \ + sed "s/_.*//; /^$pkg$/d" | sort | uniq > "$flocal" + + fdiff="$(mktemp)" + if ! diff -u "$fupstream" "$flocal" > "$fdiff"; then echo "
" >> "$msg_file" echo -e "$f\n" >> "$msg_file" - for e in "${diffs[@]}"; do - echo " - \`$e\` is listed in \`essential\`, but is not a dependency." >> "$msg_file" - done + echo "\`\`\`diff" >> "$msg_file" + cat "$fdiff" | tail -n +3 >> "$msg_file" + echo "\`\`\`" >> "$msg_file" echo -e "\n
" >> "$msg_file" fi done