Perf test under CTest conf to exclude from default test set (microsof…

…t#6422) Co-authored-by: Amaury Chamayou <[email protected]> Co-authored-by: Amaury Chamayou <[email protected]>
maxtropets · Aug 6, 2024 · 1a97e95 · 1a97e95
1 parent 20965eb
commit 1a97e95
Show file tree

Hide file tree

Showing 9 changed files with 15 additions and 13 deletions.
diff --git a/.azure-pipelines-templates/daily-matrix.yml b/.azure-pipelines-templates/daily-matrix.yml
@@ -69,7 +69,7 @@ jobs:
       cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.ASAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}"
       suffix: "ASAN"
       artifact_name: "Virtual_ASAN"
-      ctest_filter: '-LE "benchmark|perf"'
+      ctest_filter: '-LE "benchmark"'
       ctest_timeout: "1600"
       depends_on: configure
       installExtendedTestingTools: true
@@ -82,7 +82,7 @@ jobs:
       cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.TSAN.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}"
       suffix: "TSAN"
       artifact_name: "Virtual_TSAN"
-      ctest_filter: '-LE "benchmark|perf"'
+      ctest_filter: '-LE "benchmark"'
       ctest_timeout: "1600"
       depends_on: configure
       installExtendedTestingTools: true
@@ -104,7 +104,7 @@ jobs:
       cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.SGX.cmake_args }} -DCLIENT_PROTOCOLS_TEST=ON -DSHUFFLE_SUITE=ON"
       suffix: "Release"
       artifact_name: "SGX_Release"
-      ctest_filter: '-LE "benchmark|perf|rotation"'
+      ctest_filter: '-LE "benchmark|rotation"'
       depends_on: configure
       installExtendedTestingTools: true
 
@@ -116,7 +116,7 @@ jobs:
       cmake_env: "${{ parameters.build.SNPCC.cmake_env }}"
       suffix: "Release"
       artifact_name: "SNPCC_Release"
-      ctest_filter: '-LE "benchmark|perf|rotation"'
+      ctest_filter: '-LE "benchmark|rotation"'
       depends_on: configure
       installExtendedTestingTools: true
 
@@ -127,7 +127,7 @@ jobs:
       cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.Virtual.cmake_args }} -DCLIENT_PROTOCOLS_TEST=ON -DSHUFFLE_SUITE=ON"
       suffix: "Release"
       artifact_name: "Virtual_Release"
-      ctest_filter: '-LE "benchmark|perf|rotation"'
+      ctest_filter: '-LE "benchmark|rotation"'
       depends_on: configure
       installExtendedTestingTools: true
 
@@ -138,6 +138,6 @@ jobs:
       cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.SGX.cmake_args }} ${{ parameters.build.unsafe.cmake_args }}"
       suffix: "Unsafe"
       artifact_name: "SGX_Unsafe"
-      ctest_filter: '-LE "benchmark|perf|rotation"'
+      ctest_filter: '-LE "benchmark|rotation"'
       depends_on: configure
       installExtendedTestingTools: true
diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml
@@ -101,7 +101,7 @@ jobs:
           set -ex
           cd /CCF/build
           npm config set cache /ccfci/workspace_$(Build.BuildNumber)/.npm
-          WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|perf|tlstest|suite|snp_flaky" -E "lts_compatibility"
+          WORKSPACE=/ccfci/workspace_$(Build.BuildNumber) ELECTION_TIMEOUT_MS=10000 ./tests.sh -VV -T Test -LE "benchmark|tlstest|suite|snp_flaky" -E "lts_compatibility"
           # Remove irrelevant and bulky data from workspace before uploading
           find /ccfci/workspace_$(Build.BuildNumber) -type f -name cchost -delete
           find /ccfci/workspace_$(Build.BuildNumber) -type f -name "*.so" -delete

diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml
@@ -27,7 +27,7 @@ jobs:
           # Microbenchmarks
           ./tests.sh -VV -L benchmark
           # End to end performance tests
-          ./tests.sh -VV -L perf
+          ./tests.sh -VV -L perf -C perf
           # Convert microbenchmark output to bencher json
           source env/bin/activate
           PYTHONPATH=../tests python convert_pico_to_bencher.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -90,7 +90,7 @@ jobs:
           # Unit tests
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
           # All other acceptably fast tests, mostly end-to-end
-          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|suite|unit"
+          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|protocolstest|suite|unit"
           # Partitions tests
           ./tests.sh --timeout 240 --output-on-failure -L partitions -C partitions
         shell: bash

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           set +x
           cd build
-          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark|perf"
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
 
       - name: "Upload logs"
         if: success() || failure()
@@ -88,7 +88,7 @@ jobs:
         run: |
           set +x
           cd build
-          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark|perf"
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
 
       - name: "Upload logs"
         if: success() || failure()

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -123,7 +123,7 @@ jobs:
           export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
           # Unit tests
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
-          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|unit"
+          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|unit"
         shell: bash
         if: "${{ matrix.platform.name != 'snp' }}"
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1561,6 +1561,7 @@ if(BUILD_TESTS)
       NAME historical_query_perf_test
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_perf.py
       LABEL perf PERF_LABEL "Historical Queries"
+      CONFIGURATIONS perf
     )
 
     add_e2e_test(

diff --git a/cmake/common.cmake b/cmake/common.cmake
@@ -306,6 +306,7 @@ function(add_piccolo_test)
       --label ${TEST_NAME} --perf-label ${PARSED_ARGS_PERF_LABEL}
       --snapshot-tx-interval 10000 ${PARSED_ARGS_ADDITIONAL_ARGS} -e
       ${ENCLAVE_TYPE} -t ${ENCLAVE_PLATFORM} ${NODES}
+    CONFIGURATIONS perf
   )
 
   # Make python test client framework importable

diff --git a/doc/overview/performance.rst b/doc/overview/performance.rst
@@ -97,7 +97,7 @@ The end-to-end service performance tests can also be from the CCF build director
 
 .. code-block:: bash
 
-    ./tests.sh -VV -L "perf"
+    ./tests.sh -VV -L "perf" -C "perf"
 
 Each of these tests creates a temporary CCF service on the local machine, then sends a high volume of transactions to measure peak and average throughput. The python test wrappers will print summary statistics including a transaction rate histogram when the test completes. These statistics can be retrieved from any CCF service via the ``getMetrics`` RPC.