run-long-tests.yml

name: Build and run long ksmt tests

on:
  workflow_dispatch:
    inputs:
      tests:
        description: >
          Comma separated list of tests to run from ksmt-test module. 
          For example: Z3BenchmarksBasedTest.testConverter, Z3BenchmarksBasedTest.testSolver
        required: true
        type: string
      chunkSize:
        description: Maximal size of a single test data chunk
        required: true
        type: number
        default: 10000
      testDataSize:
        description: Size of test dataset
        required: true
        type: number
        default: 170000

env:
  TEST_DATA_REVISION: 0.2.1

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      matrix-tests: ${{ steps.set-tests.outputs.tests }}
      matrix-chunks: ${{ steps.set-chunks.outputs.chunks }}
    steps:
      - id: set-chunks
        name: Calculate chunk indices based on size of the dataset
        run: |
          LIMIT="${{ inputs.testDataSize }}"
          SIZE="${{ inputs.chunkSize }}"
          CHUNKS_AMOUNT=$((LIMIT / SIZE - 1))
          CHUNKS="[0"
          for (( i=1; i<=$CHUNKS_AMOUNT; i+=1 )); do
            CHUNKS+=",$i";
          done
          CHUNKS+="]"
          echo "chunks=$CHUNKS" >> $GITHUB_OUTPUT
      - id: set-tests
        name: >
          Convert comma separated list to JSON array: 'a, b' -> '["a","b"]'
        run: |
          STR="${{ inputs.tests }}"
          IFS=',' read -r -a array <<< "$STR"
          ARRAY_SIZE="${#array[@]}"
          TEST_ARRAY="["
          if test $ARRAY_SIZE -gt 0; then
            TRIM="$(echo -e "${array[0]}" | tr -d '[:space:]')"
            TEST_ARRAY+="\"$TRIM\""
            for (( i=1; i<$ARRAY_SIZE; i+=1 )); do
              TRIM="$(echo -e "${array[i]}" | tr -d '[:space:]')"
              TEST_ARRAY+=",\"$TRIM\""
            done
          fi
          TEST_ARRAY+="]"
          echo "tests=$TEST_ARRAY" >> $GITHUB_OUTPUT

  prepare_test_data:
    strategy:
      matrix:
        os: [ ubuntu-latest, windows-latest ]

    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v4

      - name: Prepare test data (cache)
        id: test-data-cache
        uses: actions/cache@v4
        env:
          cache-name: cache-test-data
        with:
          key: test-data-${{ env.TEST_DATA_REVISION }}-${{ matrix.os }}
          path: ksmt-test/testData/testData.zip

      - name: Set up JDK 1.8
        if: steps.test-data-cache.outputs.cache-hit != 'true'
        uses: actions/setup-java@v4
        with:
          java-version: 8
          distribution: 'zulu'
          cache: gradle

      - name: Setup gradle
        if: steps.test-data-cache.outputs.cache-hit != 'true'
        uses: gradle/actions/setup-gradle@v3

      - name: Prepare test data (download)
        if: steps.test-data-cache.outputs.cache-hit != 'true'
        run: >
            ./gradlew
            :ksmt-test:downloadPreparedSmtLibBenchmarkTestData
            --no-daemon
            '-PtestDataRevision=${{ env.TEST_DATA_REVISION }}'

  run_tests:
    needs: [setup, prepare_test_data]
    continue-on-error: true
    strategy:
      matrix:
        os: [ ubuntu-latest, windows-latest ]
        test: ${{ fromJSON(needs.setup.outputs.matrix-tests) }}
        chunk: ${{ fromJSON(needs.setup.outputs.matrix-chunks) }}

    name: Run ${{ matrix.test }}[${{ matrix.chunk }}] on ${{ matrix.os }}

    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v4

      - name: Prepare test data (cache)
        id: test-data-cache
        uses: actions/cache@v4
        env:
          cache-name: cache-test-data
        with:
          key: test-data-${{ env.TEST_DATA_REVISION }}-${{ matrix.os }}
          path: ksmt-test/testData/testData.zip

      - name: Check test data downloaded
        if: steps.test-data-cache.outputs.cache-hit != 'true'
        run: |
          echo "Test data is not available"
          exit 1

      - name: Set up JDK 1.8
        uses: actions/setup-java@v4
        with:
          java-version: 8
          distribution: 'zulu'
          cache: gradle

      - name: Setup gradle
        uses: gradle/actions/setup-gradle@v3

      # Since ksmt-test/testData/testData.zip exists task will not download it again
      - name: Prepare test data (unpack)
        run: >
            ./gradlew
            :ksmt-test:downloadPreparedSmtLibBenchmarkTestData
            --no-daemon
            '-PtestDataRevision=${{ env.TEST_DATA_REVISION }}'

      # We don't want to run basic KSMT tests on each chunk
      - name: Build project
        if: ${{ matrix.chunk == '0' }}
        run: ./gradlew build --no-daemon

      - name: Run ${{ matrix.test }}
        run: >
            ./gradlew
            :ksmt-test:test --tests  ${{ format('"io.ksmt.test.benchmarks.{0}"', matrix.test) }}
            --no-daemon
            --continue
            -PrunBenchmarksBasedTests=true
            -PbenchmarkChunkSize=${{ inputs.chunkSize }}
            -PbenchmarkChunk=${{ matrix.chunk }}

      - name: Upload ksmt test reports
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: ${{ format('ksmt-test-report-{0}-{1}-{2}', matrix.os, matrix.test, matrix.chunk) }}
          path: ksmt-test/build/test-results/test/binary

  merge_test_report:
    name: Merge chunked test reports into a single one

    needs: [ run_tests ]
    runs-on: ubuntu-latest
    if: ${{ always() }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          path: reports

      - name: Set up JDK 1.8
        uses: actions/setup-java@v4
        with:
          java-version: 8
          distribution: 'zulu'

      - name: Setup gradle
        uses: gradle/actions/setup-gradle@v3

      - name: Merge test report [ubuntu-latest]
        run: >
            ./gradlew
            :ksmt-test:mergeTestReports
            --no-daemon
            -PtestReportMergePrefix=ksmt-test-report-ubuntu-latest

      - name: Merge test report [windows-latest]
        run: >
            ./gradlew
            :ksmt-test:mergeTestReports
            --no-daemon
            -PtestReportMergePrefix=ksmt-test-report-windows-latest
            
      - name: Upload merged test reports
        uses: actions/upload-artifact@v4
        with:
          name: ksmt-test-report
          path: ksmt-test-report-*