Chained benchmarks #135
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: [ main ] | |
| push: | |
| branches: [ main ] | |
| tags: [ v* ] | |
| jobs: | |
| ci: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '21' | |
| cache: 'maven' | |
| - name: Test | |
| run: mvn --batch-mode --update-snapshots verify | |
| - name: Prepare release notes | |
| uses: release-drafter/release-drafter@v5 | |
| with: | |
| config-name: release-drafter.yml | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| benchmark: | |
| needs: [ ci ] | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '21' | |
| cache: 'maven' | |
| - name: Run java benchmarks # java benchmarks, excluding the chained tests | |
| id: benchmark_java | |
| run: | | |
| mvn package -DskipTests=true | |
| java -Djmh.executor=VIRTUAL -jar bench/bench-java/target/benchmarks.jar -i 5 -wi 5 -f 1 -to 1100ms -r 1000ms -w 1000ms -rff jmh-result-java.json -e ".*Chained.*" | tee out.txt | |
| echo 'output<<EOF' >> $GITHUB_OUTPUT | |
| cat out.txt >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| - name: Run java benchmarks (chained) # on CI, running chained tests only with up to 100 channels | |
| id: benchmark_java_chained | |
| run: | | |
| mvn package -DskipTests=true | |
| java -Djmh.executor=VIRTUAL -jar bench/bench-java/target/benchmarks.jar -i 5 -wi 5 -f 1 -to 1100ms -r 1000ms -w 1000ms -rff jmh-result-java-chained.json -p channelCount=100 ".*Chained.*" | tee out.txt | |
| echo 'output<<EOF' >> $GITHUB_OUTPUT | |
| cat out.txt >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| - name: Run kotlin benchmarks # kotlin benchmarks, excluding the chained tests | |
| id: benchmark_kotlin | |
| run: | | |
| java -jar bench/bench-kotlin/target/benchmarks.jar -i 5 -wi 5 -f 1 -to 1100ms -r 1000ms -w 1000ms -rf json -rff jmh-result-kotlin.json -e ".*Chained.*" | tee out.txt | |
| echo 'output<<EOF' >> $GITHUB_OUTPUT | |
| cat out.txt >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| - name: Run kotlin benchmarks (chained) # on CI, running chained tests only with up to 100 channels | |
| id: benchmark_kotlin_chained | |
| run: | | |
| java -jar bench/bench-kotlin/target/benchmarks.jar -i 5 -wi 5 -f 1 -to 1100ms -r 1000ms -w 1000ms -rf json -rff jmh-result-kotlin-chained.json -p channelCount=100 ".*Chained.*" | tee out.txt | |
| echo 'output<<EOF' >> $GITHUB_OUTPUT | |
| cat out.txt >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| - name: Merge java and kotlin benchmark results | |
| run: jq -s '.[0] + .[1] + .[2] + .[3]' jmh-result-java.json jmh-result-java-chained.json jmh-result-kotlin.json jmh-result-kotlin-chained.json > jmh-result-all.json | |
| - name: Extract branch name | |
| shell: bash | |
| run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT | |
| id: extract_branch | |
| - name: Check & store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| # Only store updated benchmark results on main branch | |
| if: steps.extract_branch.outputs.branch == 'main' | |
| with: | |
| tool: 'jmh' | |
| output-file-path: jmh-result-all.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| # Show alert with commit comment on detecting possible performance regression | |
| alert-threshold: '120%' | |
| # Enable alert commit comment | |
| comment-on-alert: true | |
| fail-on-alert: true | |
| alert-comment-cc-users: '@adamw' | |
| # Store benchmark results in the docs folder of the main branch | |
| gh-pages-branch: ${{ steps.extract_branch.outputs.branch }} | |
| benchmark-data-dir-path: 'docs/bench' | |
| skip-fetch-gh-pages: true | |
| - name: Comment on PR with results | |
| uses: actions/github-script@v7 | |
| env: | |
| JAVA_OUTPUT: ${{ steps.benchmark_java.outputs.output }} | |
| JAVA_OUTPUT_CHAINED: ${{ steps.benchmark_java_chained.outputs.output }} | |
| KOTLIN_OUTPUT: ${{ steps.benchmark_kotlin.outputs.output }} | |
| KOTLIN_OUTPUT_CHAINED: ${{ steps.benchmark_kotlin_chained.outputs.output }} | |
| with: | |
| script: | | |
| function extractContent(str) { | |
| const regex = /^Benchmark([\s\S]*?)(?=^Benchmark result)/m; | |
| const match = str.match(regex); | |
| return match ? match[1].trim() : null; | |
| } | |
| // Only if there's a PR within which we run | |
| if (context.issue.number) { | |
| let javaResults = extractContent(`${process.env.JAVA_OUTPUT}`); | |
| let javaChainedResults = extractContent(`${process.env.JAVA_OUTPUT_CHAINED}`); | |
| let kotlinResults = extractContent(`${process.env.KOTLIN_OUTPUT}`); | |
| let kotlinChainedResults = extractContent(`${process.env.KOTLIN_OUTPUT_CHAINED}`); | |
| let comment = `<details><summary>Benchmark results</summary> | |
| \`\`\` | |
| Java: | |
| ${javaResults} | |
| Java chained: | |
| ${javaChainedResults} | |
| Kotlin: | |
| ${kotlinResults} | |
| Kotlin chained: | |
| ${kotlinChainedResults} | |
| \`\`\` | |
| </details>`; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); | |
| } | |
| publishReleaseNotes: | |
| name: Publish release notes | |
| needs: [ ci ] | |
| runs-on: ubuntu-22.04 | |
| if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v')) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '21' | |
| - name: Extract version from tag name | |
| run: | | |
| version=${GITHUB_REF/refs\/tags\/v/} | |
| echo "VERSION=$version" >> $GITHUB_ENV | |
| - name: Publish release notes | |
| uses: release-drafter/release-drafter@v5 | |
| with: | |
| config-name: release-drafter.yml | |
| publish: true | |
| name: "v${{ env.VERSION }}" | |
| tag: "v${{ env.VERSION }}" | |
| version: "v${{ env.VERSION }}" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |