diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c02b969a..002fdbe8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -359,6 +359,30 @@ jobs: run: | dpkg -i ${{ steps.package_names.outputs.DEB_PACKAGE_PATH }} + - name: Run Integration Tests (AnghaBench 50) + shell: bash + working-directory: ${{ steps.build_paths.outputs.REL_SOURCE }}/anvill + run: | + python3 -m pip install -r libraries/lifting-tools-ci/requirements.txt + scripts/test-angha-50.sh \ + --ghidra-install-dir $GHIDRA_INSTALL_DIR \ + --decompile-cmd "anvill-decompile-spec" \ + --jobs 8 + + - name: Tar and Compress logs + if: failure() + run: | + shopt -s globstar + tar -cf test-errs.tar.xz ${{ steps.build_paths.outputs.REL_SOURCE }}/anvill/angha-test-50/**/std* + shell: bash + + - name: Upload stderr/stdout logs on error + if: failure() + uses: actions/upload-artifact@v3 + with: + name: AnghaBench 50 logs + path: test-errs.tar.xz + - name: Store the DEB package uses: actions/upload-artifact@v3 with: diff --git a/ci/angha_1k_test_settings.json b/ci/angha_50_test_settings.json similarity index 100% rename from ci/angha_1k_test_settings.json rename to ci/angha_50_test_settings.json diff --git a/libraries/lifting-tools-ci b/libraries/lifting-tools-ci index 95b0aa262..7f02cf509 160000 --- a/libraries/lifting-tools-ci +++ b/libraries/lifting-tools-ci @@ -1 +1 @@ -Subproject commit 95b0aa2621908df4982a18b02ef50fcb94d1044c +Subproject commit 7f02cf50924fe99751ead9952885ce4231e93cdc diff --git a/scripts/test-angha-1k.sh b/scripts/test-angha-50.sh similarity index 85% rename from scripts/test-angha-1k.sh rename to scripts/test-angha-50.sh index 2b7a9b54f..c88a81a0c 100755 --- a/scripts/test-angha-1k.sh +++ b/scripts/test-angha-50.sh @@ -5,11 +5,12 @@ GHIDRA_INSTALL_DIR="~/ghidra_10.1.5_PUBLIC/" ANVILL_DECOMPILE="anvill-decompile-spec" function Help { - echo "Run Anvill on AnghaBech-1K" + echo "Run Anvill on AnghaBench-50" echo "" echo "Options:" echo " --ghidra-install-dir The ghidra install dir. Default ${GHIDRA_INSTALL_DIR}" echo " --decompile-cmd The anvill decompile command to invoke. Default ${ANVILL_DECOMPILE}" + echo " --jobs The number of jobs that can run concurrently. Defaults to system's CPU count" echo " -h --help Print help." } @@ -69,6 +70,12 @@ while [[ $# -gt 0 ]] ; do --decompile-cmd) ANVILL_DECOMPILE=${2} shift # past argument + ;; + + # How many concurrent jobs + --jobs) + NUM_JOBS=${2} + shift # past argument ;; *) @@ -90,17 +97,17 @@ fi if ! ${ANVILL_DECOMPILE} --version &>/dev/null; -then +then echo "[!] Could not execute anvill decompile cmd: ${ANVILL_DECOMPILE}" exit 1 fi # create a working directory -mkdir -p angha-test-1k -pushd angha-test-1k +mkdir -p angha-test-50 +pushd angha-test-50 -# fetch the test set: 1K binaries (per arch) -${SRC_DIR}/libraries/lifting-tools-ci/datasets/fetch_anghabench.sh --run-size 1k --binaries +# fetch the test set: 50 binaries (per arch) +${SRC_DIR}/libraries/lifting-tools-ci/datasets/fetch_anghabench.sh --run-size 50 --binaries # extract it for tarfile in *.tar.xz do @@ -111,14 +118,20 @@ FAILED="no" for arch in $(ls -1 binaries/) do echo "[+] Testing architecture ${arch}" - ${SRC_DIR}/libraries/lifting-tools-ci/tool_run_scripts/anvill.py \ + args=( --ghidra-install-dir "${GHIDRA_INSTALL_DIR}" \ --anvill-decompile "${ANVILL_DECOMPILE}" \ --input-dir "$(pwd)/binaries/${arch}" \ --output-dir "$(pwd)/results/${arch}" \ --run-name "anvill-live-ci-${arch}" \ - --test-options "${SRC_DIR}/ci/angha_1k_test_settings.json" \ + --test-options "${SRC_DIR}/ci/angha_50_test_settings.json" \ --dump-stats + ) + if [[ -v NUM_JOBS ]]; then + args+=(--jobs "${NUM_JOBS}") + fi + ${SRC_DIR}/libraries/lifting-tools-ci/tool_run_scripts/anvill.py "${args[@]}" + if ! check_test "$(pwd)/results/${arch}/python/stats.json"