perf.yml

name: Perf

on:
  push:
    branches: [main]
  pull_request:

jobs:
  perf:
    runs-on: macos-latest
    strategy:
      fail-fast: false
    env:
      DFX_VERSION: 0.16.1
      IC_REPL_VERSION: 0.6.2
      MOC_VERSION: 0.10.4
      IC_WASM_VERSION: 0.7.0
      RUSTC_VERSION: 1.75.0
      MOC_ARTIFACT: 1242507845
    steps:
      - uses: actions/checkout@v3
      - name: Checkout out gh-pages report
        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'build_base')
        uses: actions/checkout@v3
        with:
          ref: gh-pages
          path: main/_out
      - name: Checkout out base branch
        if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build_base')
        uses: actions/checkout@v3
        with:
          ref: ${{ github.base_ref }}
          path: main/
      - uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: ${{ env.RUSTC_VERSION }}
          override: true
          target: wasm32-unknown-unknown
      - name: Cache cargo build
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: cargo-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
      - uses: actions/setup-python@v4
        if: github.event_name == 'pull_request'
        with:
          python-version: "3.10"
      - name: Install Python dependencies
        if: github.event_name == 'pull_request'
        run: |
          python -m pip install --upgrade pip
          pip install pandas markdown lxml html5lib bs4 tabulate scipy
      - uses: actions/setup-node@v3
        with:
          node-version: 18
      - name: Install ic-repl, mops, dfx, and moc
        run: |
          echo y | DFX_VERSION=$DFX_VERSION bash -ci "$(curl -fsSL https://smartcontracts.org/install.sh)"
          wget https://github.com/chenyan2002/ic-repl/releases/download/$IC_REPL_VERSION/ic-repl-macos
          cp ./ic-repl-macos /usr/local/bin/ic-repl
          chmod a+x /usr/local/bin/ic-repl
          npm i -g ic-mops
          dfx cache install
          wget https://nightly.link/dfinity/motoko/actions/artifacts/$MOC_ARTIFACT.zip
          unzip $MOC_ARTIFACT.zip
          chmod a+x bin/moc
          cp -rf bin/moc $(dfx cache show)
          wget https://github.com/dfinity/ic-wasm/releases/download/$IC_WASM_VERSION/ic-wasm-linux64
          cp ./ic-wasm-linux64 /usr/local/bin/ic-wasm
          cd $(dfx cache show)
          wget https://github.com/dfinity/motoko/releases/download/$MOC_VERSION/motoko-Darwin-x86_64-$MOC_VERSION.tar.gz
          tar zxvf motoko-Darwin-x86_64-$MOC_VERSION.tar.gz
          wget https://github.com/dfinity/ic-wasm/releases/download/$IC_WASM_VERSION/ic-wasm-macos
          cp ./ic-wasm-macos /usr/local/bin/ic-wasm
          chmod a+x /usr/local/bin/ic-wasm
      - name: Setup system subnet and start dfx
        run: |
          mkdir -p $HOME/.config/dfx
          cp networks.json $HOME/.config/dfx/
          dfx start --background
      - name: Run perf for base branch
        if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build_base')
        run: |
          cd main
          make
          dfx stop
          dfx start --clean --background
      - name: Run perf
        run: |
          make
          make emit_version
      - name: Generate table
        if: github.event_name == 'pull_request'
        env:
          PR_NUM: ${{ github.event.number }}
        run: |
          set -eu
          echo $PR_NUM > pr_num.txt
          cat > TABLE.md << EOF
          <!-- perf comment -->
          > **Note**
          > The flamegraph link only works after you merge.
          > Unchanged benchmarks are omitted.

          EOF
          cat > DIFF.md << EOF
          <!-- diff comment -->
          > **Note**
          > Diffing the performance result against the published result from main branch.
          > Unchanged benchmarks are omitted.

          EOF
          for d in _out/*; do
            if [[ -d "$d" ]]; then  # check if $d is a directory
              echo "Processing $d"
              if [[ -e "main/$d/README.md" ]]; then  # check if main/$d/README.md exists
                readme_diff=$(diff "$d/README.md" "main/$d/README.md" || true)  # compare README.md files
                if [[ -n "$readme_diff" ]]; then  # if files are different
                  cat $d/README.md >> TABLE.md
                  python .github/workflows/diff.py "$d/README.md" "main/$d/README.md" >> DIFF.md
                else
                  echo "$d/README.md is the same as main branch, skipping."
                fi
              else
                cat $d/README.md >> TABLE.md
                python .github/workflows/diff.py "$d/README.md" "main/$d/README.md" >> DIFF.md
              fi
            fi
          done
          python .github/workflows/diff.py final >> DIFF.md
      - name: Read table
        if: github.event_name == 'pull_request'
        id: perf
        uses: juliangruber/read-file-action@v1
        with:
          path: ./TABLE.md
      - name: Read diff
        if: github.event_name == 'pull_request'
        id: diff
        uses: juliangruber/read-file-action@v1
        with:
          path: ./DIFF.md
      - name: Find diff comment
        if: github.event_name == 'pull_request'
        uses: peter-evans/find-comment@v2
        id: fc_diff
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '<!-- diff comment -->'
      - name: Create or update diff comment
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
        uses: peter-evans/create-or-update-comment@v2
        with:
          comment-id: ${{ steps.fc_diff.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ${{ steps.diff.outputs.content }}
          edit-mode: replace
      - name: Find performance comment
        if: github.event_name == 'pull_request'
        uses: peter-evans/find-comment@v2
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '<!-- perf comment -->'
      - name: Create or update performance comment
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
        uses: peter-evans/create-or-update-comment@v2
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ${{ steps.perf.outputs.content }}
          edit-mode: replace
      - name: Save tables as artifact for forked PR
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
        uses: actions/upload-artifact@v3
        with:
          name: reports
          path: |
            ./DIFF.md
            ./TABLE.md
            ./pr_num.txt
          retention-days: 5
      - name: Upload report
        if: github.ref == 'refs/heads/main'
        uses: JamesIves/github-pages-deploy-action@releases/v3
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BRANCH: gh-pages
          FOLDER: _out/