Merge branch 'master' into clang-cfg-interfaces

.github/workflows/prerelease.yml

@@ -304,6 +304,128 @@ jobs:
         with:
           name: sv-comp-results
           path: ./sv-comp-results.md
+
+#
+#     Linux kernel benchmark
+#
+  run_linux_kernel_bench:
+    name: "Run Linux kernel test suite"
+    needs: build
+    strategy:
+      matrix:
+        llvm-version: [18]
+        image-version: [22.04]
+        vast-target: ['hl']
+        disable-unsup: [true, false]
+
+    runs-on: ubuntu-${{ matrix.image-version }}
+    timeout-minutes: 360
+    container:
+      image:
+        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest
+
+    steps:
+      - name: Fetch VAST artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: VAST
+
+      - name: Unpack VAST
+        run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1
+
+      - name: Export vast binaries
+        run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH
+
+      - name: Install benchmark dependencies
+        run: |
+          apt-get update
+          apt-get -y install git fakeroot build-essential \
+            ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison \
+            bear \
+            lld-${{ matrix.llvm-version }}
+
+      - name: Clone and build the Linux kernel to create a compilation database
+        run: |
+          git clone --depth=1 https://github.com/torvalds/linux.git linux
+          cd linux
+          bear -- make defconfig
+          bear -- make LLVM=-${{ matrix.llvm-version }} -j $(nproc)
+          cd ../
+
+      - name: Clone vast benchmark directory
+        uses: actions/checkout@v4
+        with:
+          repository: trailofbits/vast-benchmarks
+          sparse-checkout: benchmarks/linux_kernel
+          ref: main
+          path: vast-benchmarks/
+          fetch-depth: 1
+
+      - name: Setup result suffix and vast arguments
+        run: |
+          if [ "${{ matrix.disable-unsup }}" = "false" ]; then
+            echo "VAST_RESULTS_SUFFIX=with_unsup" >> $GITHUB_ENV
+          else
+            echo "VAST_RESULTS_SUFFIX=without_unsup" >> $GITHUB_ENV
+            echo "VAST_DISABLE_UNSUPPORTED=-vast-disable-unsupported" >> $GITHUB_ENV
+          fi
+
+      - name: Run benchmarks
+        run: >
+          python3 ${PWD}/vast-benchmarks/benchmarks/linux_kernel/run_vast_benchmark.py
+          vast-front
+          ${PWD}/linux/compile_commands.json
+          --num_processes=$(nproc)
+          --vast_option="-xc"
+          --vast_option="-vast-emit-mlir=${{ matrix.vast-target }}"
+          --vast_option="${{ env.VAST_DISABLE_UNSUPPORTED }}"
+          --print_errors
+          > vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv
+          path: ./vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv
+
+  convert_linux_kernel_bench:
+    name: "Convert Linux Kernel benchmark results to Markdown"
+    needs: run_linux_kernel_bench
+    strategy:
+      matrix:
+        image-version: [22.04]
+    runs-on: ubuntu-${{ matrix.image-version }}
+    steps:
+      - name: Fetch result artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: vast_linux_kernel_times_*
+          merge-multiple: true
+
+      - name: Install converter dependencies
+        run: pip3 install pandas scipy tabulate
+
+      - name: Clone vast benchmark directory
+        uses: actions/checkout@v4
+        with:
+          repository: trailofbits/vast-benchmarks
+          sparse-checkout: utils/
+          ref: main
+          path: vast-benchmarks/
+          fetch-depth: 1
+
+      - name: Generate the results
+        run: >
+          python3 vast-benchmarks/utils/to_markdown.py \
+          '{ "HighLevel with unsupported": "vast_linux_kernel_times_hl_with_unsup.tsv", "HighLevel" : "vast_linux_kernel_times_hl_without_unsup.tsv" }'
+          --output_filepath linux_kernel_times.md
+
+      - name: Post results as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: linux_kernel_times.md
+          path: ./linux_kernel_times.md
+
 #
 #     Webpage build
 #
@@ -314,7 +436,7 @@ jobs:
         image-version: [22.04]
     name: "Build VAST doc"
     runs-on: ubuntu-${{ matrix.image-version }}
-    needs: [eval_llvm_ts, eval_svcomp]
+    needs: [eval_llvm_ts, eval_svcomp, convert_linux_kernel_bench]
     timeout-minutes: 60
     container:
       image:
@@ -350,6 +472,12 @@ jobs:
           name: sv-comp-results
           path: sv-comp-results
 
+      - name: Fetch Linux kernel benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          name: linux_kernel_times.md
+          path: linux_kernel_times
+
       - name: Build Pages
         run: |
           sh ./www/setup.sh _site ./builds/ci/

include/vast/CodeGen/ClangVisitorBase.hpp

@@ -51,14 +51,33 @@ namespace vast::cg {
             };
         }
 
+        template< typename yield_type >
+        auto mk_stmt_builder_with_args(const clang_stmt *stmt, const mlir_type &types...) {
+            return [&, this, stmt] (auto &state, auto loc) {
+                state.getBlock()->getParent()->addArgument(types, loc);
+                self.visit(stmt);
+                auto &op = state.getBlock()->back();
+                VAST_ASSERT(op.getNumResults() == 1);
+                bld.create< yield_type >(loc, op.getResult(0));
+            };
+        }
+
         auto mk_value_builder(const clang_stmt *stmt) {
             return mk_stmt_builder< hl::ValueYieldOp >(stmt);
         }
 
+        auto mk_value_builder_with_args(const clang_stmt *stmt, const mlir_type &types...) {
+            return mk_stmt_builder_with_args< hl::ValueYieldOp >(stmt, types);
+        }
+
         auto mk_cond_builder(const clang_stmt *stmt) {
             return mk_stmt_builder< hl::CondYieldOp >(stmt);
         }
 
+        auto mk_cond_builder_with_args(const clang_stmt *stmt, const mlir_type &types...) {
+            return mk_stmt_builder_with_args< hl::CondYieldOp >(stmt, types);
+        }
+
         auto mk_true_yielder() {
             return [this] (auto &, auto loc) {
                bld.create< hl::CondYieldOp >(loc, bld.true_value(loc));

include/vast/CodeGen/DefaultStmtVisitor.hpp

@@ -20,6 +20,10 @@ namespace vast::cg {
         using base = stmt_visitor_base< default_stmt_visitor >;
         using base::base;
 
+        default_stmt_visitor(mcontext_t &mctx, acontext_t &actx, codegen_builder &bld, visitor_view self, scope_context &scope)
+            : base(mctx, bld, self, scope), actx(actx)
+        {}
+
         operation visit(const clang_stmt *stmt) {return Visit(stmt); }
 
         using base::Visit;
@@ -192,6 +196,7 @@ namespace vast::cg {
 
         operation VisitMemberExpr(const clang::MemberExpr *expr);
         operation VisitConditionalOperator(const clang::ConditionalOperator *op);
+        operation VisitBinaryConditionalOperator(const clang::BinaryConditionalOperator *op);
         operation VisitAddrLabelExpr(const clang::AddrLabelExpr *expr);
         operation VisitConstantExpr(const clang::ConstantExpr *expr);
         operation VisitArraySubscriptExpr(const clang::ArraySubscriptExpr *expr);
@@ -231,7 +236,7 @@ namespace vast::cg {
         // operation VisitCXXOperatorCallExpr(const clang::CXXOperatorCallExpr *expr)
 
         operation VisitOffsetOfExpr(const clang::OffsetOfExpr *expr);
-        // operation VisitOpaqueValueExpr(const clang::OpaqueValueExpr *expr)
+        operation VisitOpaqueValueExpr(const clang::OpaqueValueExpr *expr);
         // operation VisitOverloadExpr(const clang::OverloadExpr *expr)
 
         operation VisitParenExpr(const clang::ParenExpr *expr);
@@ -273,6 +278,8 @@ namespace vast::cg {
 
         operation VisitInitListExpr(const clang::InitListExpr *expr);
         operation VisitImplicitValueInitExpr(const clang::ImplicitValueInitExpr *expr);
+        protected:
+            acontext_t &actx;
     };
 
     template< typename Op >

include/vast/CodeGen/DefaultVisitor.hpp

@@ -18,13 +18,15 @@ namespace vast::cg
         default_visitor(
               visitor_base &head
             , mcontext_t &mctx
+            , acontext_t &actx
             , codegen_builder &bld
             , std::shared_ptr< meta_generator > mg
             , std::shared_ptr< symbol_generator > sg
             , bool emit_strict_function_return
             , missing_return_policy policy
         )
             : mctx(mctx)
+            , actx(actx)
             , bld(bld)
             , self(head)
             , mg(std::move(mg))
@@ -50,6 +52,7 @@ namespace vast::cg
         std::optional< symbol_name > symbol(const clang_decl_ref_expr *decl) override;
 
         mcontext_t &mctx;
+        acontext_t &actx;
         codegen_builder &bld;
         visitor_view self;
 

include/vast/Dialect/HighLevel/HighLevelCF.td

@@ -19,7 +19,7 @@ class ControlFlowOp< string mnemonic, list< Trait > traits = [] >
 
 def HighLevel_CondYieldOp : HighLevel_Op< "cond.yield", [
   // TODO(Heno): add ReturnLike trait
-  Terminator, ParentOneOf<["IfOp", "WhileOp", "ForOp", "DoOp", "CondOp"]>
+  Terminator, ParentOneOf<["IfOp", "WhileOp", "ForOp", "DoOp", "CondOp", "BinaryCondOp"]>
 ] > {
   let summary = "condition yield operation";
   let description = [{
@@ -157,11 +157,49 @@ def HighLevel_CondOp : ControlFlowOp< "cond" >
     )>
   ];
 
-  let extraClassDeclaration = [{
-    bool typesMatch(mlir::Type lhs, mlir::Type rhs);
+  let assemblyFormat = [{ attr-dict `:` type(results) $condRegion `?` $thenRegion `:` $elseRegion }];
+}
+
+def HighLevel_BinaryCondOp
+  : HighLevel_Op<
+        "binary_cond",
+        [NoTerminator, DeclareOpInterfaceMethods<RegionKindInterface>]
+    >
+    , Results<(outs AnyType:$result)>
+{
+  let summary = "VAST binary conditional statement";
+  let description = [{
+    Binary Conditional operator defined by the GNU extension of C.
+    This operation holds in total 4 regions:
+    1) The `common` region holds the common subexpression of the condition and result
+    2) The `cond` region takes an argument, which should be the result of `common`,
+       and yields the condition result.
+    2) The `then` region takes an argument, which should be the result of `common`,
+       and yields the result (it may potentially contain implicit casts).
+    3) The `else` region contains the right hand side expression and yields the result.
   }];
 
-  let assemblyFormat = [{ attr-dict `:` type(results) $condRegion `?` $thenRegion `:` $elseRegion }];
+  let regions = (region
+    ValueRegion:$commonRegion,
+    CondRegion:$condRegion,
+    ValueRegion:$thenRegion,
+    ValueRegion:$elseRegion
+  );
+
+  let hasRegionVerifier = 1;
+
+  let skipDefaultBuilders = 1;
+  let builders = [
+    OpBuilder<(ins
+      "Type":$type,
+      "builder_callback_ref":$commonBuilder,
+      "builder_callback_ref":$condBuilder,
+      "builder_callback_ref":$thenBuilder,
+      "builder_callback_ref":$elseBuilder
+    )>
+  ];
+
+  let assemblyFormat = [{ attr-dict `:` type(results) $commonRegion `,` $condRegion `?` $thenRegion `:` $elseRegion }];
 }
 
 def HighLevel_WhileOp : ControlFlowOp< "while", [NoTerminator] >

include/vast/Dialect/HighLevel/HighLevelOps.td

@@ -1471,4 +1471,20 @@ def VAArgExpr
   let assemblyFormat = "$arg_list `:` functional-type(operands, $result) attr-dict";
 }
 
+def OpaqueValueExpr
+  : HighLevel_Op< "opaque_expr" >
+  , Arguments< (ins Variadic< AnyType >: $arg) >
+  , Results< (outs AnyType: $result) >
+{
+  let summary = "Opaque value expression from clang AST.";
+  let description = [{
+    This op takes an argument and returns it. According to clang documentation
+    it is "usually" a copy operation with no special semantings.
+    The current use in VAST is to have the opaque value computed in some region and
+    then using this op to extract the computed value from region argument (see BinaryCondOp).
+  }];
+
+  let assemblyFormat = "$arg `:` functional-type(operands, $result) attr-dict";
+}
+
 #endif // VAST_DIALECT_HIGHLEVEL_IR_HIGHLEVELOPS

include/vast/Interfaces/AST/CMakeLists.txt

@@ -5,4 +5,4 @@ add_vast_op_interface(DeclInterface)
 add_vast_op_interface(ContextInterface)
 add_vast_op_interface(ExprInterface)
 add_vast_op_interface(StmtInterface)
-add_vast_type_interface(TypeInterface)
+add_vast_type_interface(TypeInterface)

include/vast/Tower/Handle.hpp

@@ -0,0 +1,22 @@
+// Copyright (c) 2022-present, Trail of Bits, Inc.
+
+#pragma once
+
+#include "vast/Util/Common.hpp"
+
+VAST_RELAX_WARNINGS
+VAST_UNRELAX_WARNINGS
+
+namespace vast::tw {
+    using conversion_path_t             = std::vector< std::string >;
+    using conversion_path_fingerprint_t = std::string;
+
+    using handle_id_t = std::size_t;
+
+    struct handle_t
+    {
+        handle_id_t id;
+        vast_module mod;
+    };
+
+} // namespace vast::tw