From 1a97e95f9a5d22a8c182cfd3dbb14c39968953ee Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 6 Aug 2024 14:17:01 +0100 Subject: [PATCH] Perf test under CTest conf to exclude from default test set (#6422) Co-authored-by: Amaury Chamayou Co-authored-by: Amaury Chamayou --- .azure-pipelines-templates/daily-matrix.yml | 12 ++++++------ .azure-pipelines-templates/deploy_aci.yml | 2 +- .github/workflows/bencher.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/long-test.yml | 4 ++-- .github/workflows/release.yml | 2 +- CMakeLists.txt | 1 + cmake/common.cmake | 1 + doc/overview/performance.rst | 2 +- 9 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml index 844e0b8ee4ee..5dafafda7eba 100644 --- a/.azure-pipelines-templates/daily-matrix.yml +++ b/.azure-pipelines-templates/daily-matrix.yml @@ -69,7 +69,7 @@ jobs: cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.ASAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" suffix: "ASAN" artifact_name: "Virtual_ASAN" - ctest_filter: '-LE "benchmark|perf"' + ctest_filter: '-LE "benchmark"' ctest_timeout: "1600" depends_on: configure installExtendedTestingTools: true @@ -82,7 +82,7 @@ jobs: cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.TSAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}" suffix: "TSAN" artifact_name: "Virtual_TSAN" - ctest_filter: '-LE "benchmark|perf"' + ctest_filter: '-LE "benchmark"' ctest_timeout: "1600" depends_on: configure installExtendedTestingTools: true @@ -104,7 +104,7 @@ jobs: cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.SGX.cmake_args }} -DCLIENT_PROTOCOLS_TEST=ON -DSHUFFLE_SUITE=ON" suffix: "Release" artifact_name: "SGX_Release" - ctest_filter: '-LE "benchmark|perf|rotation"' + ctest_filter: '-LE "benchmark|rotation"' depends_on: configure installExtendedTestingTools: true @@ -116,7 +116,7 @@ jobs: cmake_env: "${{ parameters.build.SNPCC.cmake_env }}" suffix: "Release" artifact_name: "SNPCC_Release" - ctest_filter: '-LE "benchmark|perf|rotation"' + ctest_filter: '-LE "benchmark|rotation"' depends_on: configure installExtendedTestingTools: true @@ -127,7 +127,7 @@ jobs: cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.Virtual.cmake_args }} -DCLIENT_PROTOCOLS_TEST=ON -DSHUFFLE_SUITE=ON" suffix: "Release" artifact_name: "Virtual_Release" - ctest_filter: '-LE "benchmark|perf|rotation"' + ctest_filter: '-LE "benchmark|rotation"' depends_on: configure installExtendedTestingTools: true @@ -138,6 +138,6 @@ jobs: cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.SGX.cmake_args }} ${{ parameters.build.unsafe.cmake_args }}" suffix: "Unsafe" artifact_name: "SGX_Unsafe" - ctest_filter: '-LE "benchmark|perf|rotation"' + ctest_filter: '-LE "benchmark|rotation"' depends_on: configure installExtendedTestingTools: true diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml index 1784a466021e..4795ceb8718b 100644 --- a/.azure-pipelines-templates/deploy_aci.yml +++ b/.azure-pipelines-templates/deploy_aci.yml @@ -101,7 +101,7 @@ jobs: set -ex cd /CCF/build npm config set cache /ccfci/workspace_$(Build.BuildNumber)/.npm - WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|perf|tlstest|suite|snp_flaky" -E "lts_compatibility" + WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|tlstest|suite|snp_flaky" -E "lts_compatibility" # Remove irrelevant and bulky data from workspace before uploading find /ccfci/workspace_$(Build.BuildNumber) -type f -name cchost -delete find /ccfci/workspace_$(Build.BuildNumber) -type f -name "*.so" -delete diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index 5cfc20a46e9e..4c7b39e9135e 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -27,7 +27,7 @@ jobs: # Microbenchmarks ./tests.sh -VV -L benchmark # End to end performance tests - ./tests.sh -VV -L perf + ./tests.sh -VV -L perf -C perf # Convert microbenchmark output to bencher json source env/bin/activate PYTHONPATH=../tests python convert_pico_to_bencher.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bd99193da04..f415bc259d38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: # Unit tests ./tests.sh --output-on-failure -L unit -j$(nproc --all) # All other acceptably fast tests, mostly end-to-end - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|suite|unit" + ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|protocolstest|suite|unit" # Partitions tests ./tests.sh --timeout 240 --output-on-failure -L partitions -C partitions shell: bash diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 9642da08e851..b21639624a93 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -52,7 +52,7 @@ jobs: run: | set +x cd build - ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark|perf" + ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark" - name: "Upload logs" if: success() || failure() @@ -88,7 +88,7 @@ jobs: run: | set +x cd build - ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark|perf" + ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark" - name: "Upload logs" if: success() || failure() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec975eae95b0..8708e3475bfd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -123,7 +123,7 @@ jobs: export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15) # Unit tests ./tests.sh --output-on-failure -L unit -j$(nproc --all) - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|unit" + ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|unit" shell: bash if: "${{ matrix.platform.name != 'snp' }}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 40717b16a8af..3137d23d47c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1561,6 +1561,7 @@ if(BUILD_TESTS) NAME historical_query_perf_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_perf.py LABEL perf PERF_LABEL "Historical Queries" + CONFIGURATIONS perf ) add_e2e_test( diff --git a/cmake/common.cmake b/cmake/common.cmake index 54ffffca9570..f786365d1f0f 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -306,6 +306,7 @@ function(add_piccolo_test) --label ${TEST_NAME} --perf-label ${PARSED_ARGS_PERF_LABEL} --snapshot-tx-interval 10000 ${PARSED_ARGS_ADDITIONAL_ARGS} -e ${ENCLAVE_TYPE} -t ${ENCLAVE_PLATFORM} ${NODES} + CONFIGURATIONS perf ) # Make python test client framework importable diff --git a/doc/overview/performance.rst b/doc/overview/performance.rst index 48c3be827473..c200d2047868 100644 --- a/doc/overview/performance.rst +++ b/doc/overview/performance.rst @@ -97,7 +97,7 @@ The end-to-end service performance tests can also be from the CCF build director .. code-block:: bash - ./tests.sh -VV -L "perf" + ./tests.sh -VV -L "perf" -C "perf" Each of these tests creates a temporary CCF service on the local machine, then sends a high volume of transactions to measure peak and average throughput. The python test wrappers will print summary statistics including a transaction rate histogram when the test completes. These statistics can be retrieved from any CCF service via the ``getMetrics`` RPC.