Build and run long ksmt tests #118
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and run long ksmt tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| tests: | |
| description: > | |
| Comma separated list of tests to run from ksmt-test module. | |
| For example: Z3BenchmarksBasedTest.testConverter, Z3BenchmarksBasedTest.testSolver | |
| required: true | |
| type: string | |
| chunkSize: | |
| description: Maximal size of a single test data chunk | |
| required: true | |
| type: number | |
| default: 10000 | |
| testDataSize: | |
| description: Size of test dataset | |
| required: true | |
| type: number | |
| default: 170000 | |
| env: | |
| TEST_DATA_REVISION: 0.2.1 | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix-tests: ${{ steps.set-tests.outputs.tests }} | |
| matrix-chunks: ${{ steps.set-chunks.outputs.chunks }} | |
| steps: | |
| - id: set-chunks | |
| name: Calculate chunk indices based on size of the dataset | |
| run: | | |
| LIMIT="${{ inputs.testDataSize }}" | |
| SIZE="${{ inputs.chunkSize }}" | |
| CHUNKS_AMOUNT=$((LIMIT / SIZE - 1)) | |
| CHUNKS="[0" | |
| for (( i=1; i<=$CHUNKS_AMOUNT; i+=1 )); do | |
| CHUNKS+=",$i"; | |
| done | |
| CHUNKS+="]" | |
| echo "chunks=$CHUNKS" >> $GITHUB_OUTPUT | |
| - id: set-tests | |
| name: > | |
| Convert comma separated list to JSON array: 'a, b' -> '["a","b"]' | |
| run: | | |
| STR="${{ inputs.tests }}" | |
| IFS=',' read -r -a array <<< "$STR" | |
| ARRAY_SIZE="${#array[@]}" | |
| TEST_ARRAY="[" | |
| if test $ARRAY_SIZE -gt 0; then | |
| TRIM="$(echo -e "${array[0]}" | tr -d '[:space:]')" | |
| TEST_ARRAY+="\"$TRIM\"" | |
| for (( i=1; i<$ARRAY_SIZE; i+=1 )); do | |
| TRIM="$(echo -e "${array[i]}" | tr -d '[:space:]')" | |
| TEST_ARRAY+=",\"$TRIM\"" | |
| done | |
| fi | |
| TEST_ARRAY+="]" | |
| echo "tests=$TEST_ARRAY" >> $GITHUB_OUTPUT | |
| prepare_test_data: | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest ] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Prepare test data (cache) | |
| id: test-data-cache | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-test-data | |
| with: | |
| key: test-data-${{ env.TEST_DATA_REVISION }}-${{ matrix.os }} | |
| path: ksmt-test/testData/testData.zip | |
| - name: Set up JDK 1.8 | |
| if: steps.test-data-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 8 | |
| distribution: 'zulu' | |
| cache: gradle | |
| - name: Setup gradle | |
| if: steps.test-data-cache.outputs.cache-hit != 'true' | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Prepare test data (download) | |
| if: steps.test-data-cache.outputs.cache-hit != 'true' | |
| run: > | |
| ./gradlew | |
| :ksmt-test:downloadPreparedSmtLibBenchmarkTestData | |
| --no-daemon | |
| '-PtestDataRevision=${{ env.TEST_DATA_REVISION }}' | |
| run_tests: | |
| needs: [setup, prepare_test_data] | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest ] | |
| test: ${{ fromJSON(needs.setup.outputs.matrix-tests) }} | |
| chunk: ${{ fromJSON(needs.setup.outputs.matrix-chunks) }} | |
| name: Run ${{ matrix.test }}[${{ matrix.chunk }}] on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Prepare test data (cache) | |
| id: test-data-cache | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-test-data | |
| with: | |
| key: test-data-${{ env.TEST_DATA_REVISION }}-${{ matrix.os }} | |
| path: ksmt-test/testData/testData.zip | |
| - name: Check test data downloaded | |
| if: steps.test-data-cache.outputs.cache-hit != 'true' | |
| run: | | |
| echo "Test data is not available" | |
| exit 1 | |
| - name: Set up JDK 1.8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 8 | |
| distribution: 'zulu' | |
| cache: gradle | |
| - name: Setup gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| # Since ksmt-test/testData/testData.zip exists task will not download it again | |
| - name: Prepare test data (unpack) | |
| run: > | |
| ./gradlew | |
| :ksmt-test:downloadPreparedSmtLibBenchmarkTestData | |
| --no-daemon | |
| '-PtestDataRevision=${{ env.TEST_DATA_REVISION }}' | |
| # We don't want to run basic KSMT tests on each chunk | |
| - name: Build project | |
| if: ${{ matrix.chunk == '0' }} | |
| run: ./gradlew build --no-daemon | |
| - name: Run ${{ matrix.test }} | |
| run: > | |
| ./gradlew | |
| :ksmt-test:test --tests ${{ format('"io.ksmt.test.benchmarks.{0}"', matrix.test) }} | |
| --no-daemon | |
| --continue | |
| -PrunBenchmarksBasedTests=true | |
| -PbenchmarkChunkSize=${{ inputs.chunkSize }} | |
| -PbenchmarkChunk=${{ matrix.chunk }} | |
| - name: Upload ksmt test reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: ${{ format('ksmt-test-report-{0}-{1}-{2}', matrix.os, matrix.test, matrix.chunk) }} | |
| path: ksmt-test/build/test-results/test/binary | |
| merge_test_report: | |
| name: Merge chunked test reports into a single one | |
| needs: [ run_tests ] | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: reports | |
| - name: Set up JDK 1.8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 8 | |
| distribution: 'zulu' | |
| - name: Setup gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Merge test report [ubuntu-latest] | |
| run: > | |
| ./gradlew | |
| :ksmt-test:mergeTestReports | |
| --no-daemon | |
| -PtestReportMergePrefix=ksmt-test-report-ubuntu-latest | |
| - name: Merge test report [windows-latest] | |
| run: > | |
| ./gradlew | |
| :ksmt-test:mergeTestReports | |
| --no-daemon | |
| -PtestReportMergePrefix=ksmt-test-report-windows-latest | |
| - name: Upload merged test reports | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ksmt-test-report | |
| path: ksmt-test-report-* |