Enable -flto and -fsanitize=cfi in clang

.github/scripts/get_system_info.sh

@@ -27,6 +27,8 @@ function system_info {
 	echo "**********system_info**********"
 	cat /etc/os-release | grep -oP "PRETTY_NAME=\K.*"
 	cat /proc/version
+	echo "**********Binaries**********"
+	find build $(dirname "$(readlink -f "$0")")/../../build -type f | xargs md5sum
 	echo "**********SYCL-LS**********"
 	source /opt/intel/oneapi/setvars.sh
 	sycl-ls

.github/workflows/build-hw-reusable.yml

@@ -52,7 +52,7 @@ jobs:
           static_adapter: "${{inputs.static_loader}}"
         }]
         build_type: [Debug, Release]
-        compiler: [{c: gcc, cxx: g++}, {c: clang, cxx: clang++}]
+        compiler: [{c: clang, cxx: clang++}]
         # TODO: The latest L0 loader segfaults when built with clang.
         exclude:
          - adapter: {name: L0, platform: ""}

.github/workflows/cmake.yml

@@ -135,198 +135,27 @@ jobs:
     with:
       test_label: "fuzz-short"
 
-  level-zero:
-    name: Level Zero
-    uses: ./.github/workflows/build-hw-reusable.yml
-    with:
-      adapter_name: L0
-      runner_name: L0
-
-  level-zero-v2:
-    name: Level Zero V2
-    uses: ./.github/workflows/build-hw-reusable.yml
-    with:
-      adapter_name: L0_V2
-      runner_name: L0
-
-  level-zero-static:
-    name: Level Zero static
-    uses: ./.github/workflows/build-hw-reusable.yml
-    with:
-      adapter_name: L0
-      runner_name: L0
-      static_loader: ON
-      static_adapter: ON
-
-  opencl:
-    name: OpenCL
-    uses: ./.github/workflows/build-hw-reusable.yml
-    with:
-      adapter_name: OPENCL
-      runner_name: OPENCL
-      platform: "Intel(R) OpenCL"
-
   cuda:
     name: CUDA
     uses: ./.github/workflows/build-hw-reusable.yml
     with:
       adapter_name: CUDA
       runner_name: CUDA
-
-  hip:
-    name: HIP
-    uses: ./.github/workflows/build-hw-reusable.yml
-    with:
-      adapter_name: HIP
-      runner_name: HIP
-
-  native-cpu:
-    name: Native CPU
+  ncpu-at-cuda-1:
+    name: N. Cpu @ Cuda 1
     uses: ./.github/workflows/build-hw-reusable.yml
     with:
       adapter_name: NATIVE_CPU
-      runner_name: NATIVE_CPU
-
-  # Native CPU jobs are here to force the loader to be used (UR will not use the loader if there is only one target)
-  combined-opencl-native-cpu:
-    name: OpenCL + Native CPU (Loader)
+      runner_name: CUDA
+  ncpu-at-cuda-2:
+    name: N. Cpu @ Cuda 2
     uses: ./.github/workflows/build-hw-reusable.yml
     with:
-      adapter_name: OPENCL
-      other_adapter_name: NATIVE_CPU
-      runner_name: OPENCL
-      platform: "OPENCL:Intel(R) OpenCL"
-
-  combined-level-zero-native-cpu:
-    name: Level Zero + Native CPU (Loader)
+      adapter_name: NATIVE_CPU
+      runner_name: CUDA
+  ncpu-at-cuda-3:
+    name: N. Cpu @ Cuda 3
     uses: ./.github/workflows/build-hw-reusable.yml
     with:
-      adapter_name: L0
-      other_adapter_name: NATIVE_CPU
-      runner_name: L0
-
-  e2e-level-zero:
-    name: E2E L0
-    permissions:
-      contents: read
-      pull-requests: write
-    needs: [ubuntu-build, level-zero]
-    uses: ./.github/workflows/e2e_level_zero.yml
-
-  e2e-opencl:
-    name: E2E OpenCL
-    permissions:
-      contents: read
-      pull-requests: write
-    needs: [ubuntu-build, opencl]
-    uses: ./.github/workflows/e2e_opencl.yml
-
-  e2e-cuda:
-    name: E2E CUDA
-    permissions:
-      contents: read
-      pull-requests: write
-    needs: [ubuntu-build, cuda]
-    uses: ./.github/workflows/e2e_cuda.yml
-
-  windows-build:
-    name: Build - Windows
-    strategy:
-      matrix:
-        os: ['windows-2019', 'windows-2022']
-        adapter: [
-          {name: None, var: ''}, {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON'},
-          {name: None, var: ''}, {name: L0_V2, var: '-DUR_BUILD_ADAPTER_L0_V2=ON'},
-          {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
-        ]
-
-        # TODO: building level zero loader on windows-2019 and clang-cl is currently broken
-        exclude:
-         - os: 'windows-2019'
-           adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON'}
-         - os: 'windows-2019'
-           adapter: {name: L0_V2, var: '-DUR_BUILD_ADAPTER_L0_V2=ON'}
-         - os: 'windows-2019'
-           adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
-         - adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON'}
-           compiler: {c: clang-cl, cxx: clang-cl}
-         - adapter: {name: L0_V2, var: '-DUR_BUILD_ADAPTER_L0_V2=ON'}
-           compiler: {c: clang-cl, cxx: clang-cl}
-         - adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
-           compiler: {c: clang-cl, cxx: clang-cl}
-
-        build_type: [Debug, Release]
-        compiler: [{c: cl, cxx: cl}, {c: clang-cl, cxx: clang-cl}]
-        include:
-          - compiler: {c: clang-cl, cxx: clang-cl}
-            toolset: "-T ClangCL"
-          - os: 'windows-2022'
-            adapter: {name: L0, var: '-DUR_BUILD_ADAPTER_L0=ON -DUR_STATIC_ADAPTER_L0=ON'}
-            build_type: 'Release'
-            compiler: {c: cl, cxx: cl}
-
-    runs-on: ${{matrix.os}}
-
-    steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
-    - name: Install hwloc
-      run: vcpkg install hwloc:x64-windows
-
-    - name: Configure CMake
-      env:
-        VCPKG_PATH: "C:/vcpkg/packages/hwloc_x64-windows"
-      run: >
-        cmake
-        -B${{github.workspace}}/build
-        ${{matrix.toolset}}
-        -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}"
-        -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
-        -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
-        -DCMAKE_POLICY_DEFAULT_CMP0094=NEW
-        -DUR_ENABLE_TRACING=ON
-        -DUR_DEVELOPER_MODE=ON
-        -DUR_BUILD_TESTS=ON
-        -DUR_FORMAT_CPP_STYLE=OFF
-        -DUR_CONFORMANCE_TEST_LOADER=OFF
-        ${{matrix.adapter.var}}
-
-    - name: Build all
-      run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j $Env:NUMBER_OF_PROCESSORS
-
-    - name: Test
-      working-directory: ${{github.workspace}}/build
-      run: ctest -C ${{matrix.build_type}} --output-on-failure -L "umf|loader|validation|tracing|unit|urtrace"
-
-  macos-build:
-    name: Build - MacOS
-    strategy:
-        matrix:
-          os: ['macos-13']
-    runs-on: ${{matrix.os}}
-
-    steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
-      with:
-        python-version: 3.9
-
-    - name: Install prerequisites
-      run: python3 -m pip install -r third_party/requirements.txt
-
-    - name: Install hwloc
-      run: brew install hwloc
-
-    - name: Configure CMake
-      run: >
-        cmake
-        -B${{github.workspace}}/build
-        -DUR_ENABLE_TRACING=ON
-        -DUR_DEVELOPER_MODE=ON
-        -DCMAKE_BUILD_TYPE=Release
-        -DUR_BUILD_TESTS=ON
-        -DUR_FORMAT_CPP_STYLE=ON
-        -DUMF_ENABLE_POOL_TRACKING=ON
-    - name: Build
-      run: cmake --build ${{github.workspace}}/build -j $(sysctl -n hw.logicalcpu)
+      adapter_name: NATIVE_CPU
+      runner_name: CUDA

cmake/helpers.cmake

@@ -63,6 +63,11 @@ if(CMAKE_SYSTEM_NAME STREQUAL Linux)
     check_cxx_compiler_flag("-fstack-clash-protection" CXX_HAS_FSTACK_CLASH_PROTECTION)
 endif()
 
+set(SAVED_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+set(CMAKE_REQUIRED_FLAGS "-flto -fvisibility=hidden")
+check_cxx_compiler_flag("-fsanitize=cfi" CXX_HAS_CFI_SANITIZE)
+set(CMAKE_REQUIRED_FLAGS ${SAVED_CMAKE_REQUIRED_FLAGS})
+
 function(add_ur_target_compile_options name)
     if(NOT MSVC)
         target_compile_definitions(${name} PRIVATE -D_FORTIFY_SOURCE=2)
@@ -78,11 +83,10 @@ function(add_ur_target_compile_options name)
             # Hardening options
             -fPIC
             -fstack-protector-strong
-            -fvisibility=hidden # Required for -fsanitize=cfi
-            # -fsanitize=cfi requires -flto, which breaks a lot of things
-            # See: https://github.com/oneapi-src/unified-runtime/issues/2120
-            # -flto
-            # $<$<CXX_COMPILER_ID:Clang,AppleClang>:-fsanitize=cfi>
+            -fvisibility=hidden
+            # cfi-icall requires called functions in shared libraries to also be built with cfi-icall, which we can't
+            # guarantee. -fsanitize=cfi depends on -flto
+            $<$<BOOL:${CXX_HAS_CFI_SANITIZE}>:-flto -fsanitize=cfi -fno-sanitize=cfi-icall -fdebug-default-version=4>
             $<$<BOOL:${CXX_HAS_FCF_PROTECTION_FULL}>:-fcf-protection=full>
             $<$<BOOL:${CXX_HAS_FSTACK_CLASH_PROTECTION}>:-fstack-clash-protection>
 
@@ -126,7 +130,10 @@ endfunction()
 function(add_ur_target_link_options name)
     if(NOT MSVC)
         if (NOT APPLE)
-            target_link_options(${name} PRIVATE "LINKER:-z,relro,-z,now,-z,noexecstack")
+            target_link_options(${name} PRIVATE
+                $<$<BOOL:${CXX_HAS_CFI_SANITIZE}>:-v -flto -fsanitize=cfi -fno-sanitize=cfi-icall -fdebug-default-version=4>
+                "LINKER:-z,relro,-z,now,-z,noexecstack"
+            )
             if (UR_DEVELOPER_MODE)
                 target_link_options(${name} PRIVATE -Werror -Wextra)
             endif()

test/conformance/enqueue/enqueue_adapter_opencl.match

@@ -0,0 +1,3 @@
+# Note: This file is only for use with cts_exe.py
+# Fails when -fsanitize=cfi
+{{OPT}}urEnqueueEventsWaitMultiDeviceMTTest.EnqueueWaitOnAllQueues/MultiThread

test/conformance/exp_command_buffer/exp_command_buffer_adapter_cuda.match

@@ -0,0 +1,11 @@
+# Note: This file is only for use with cts_exe.py
+# These cause SIGILL when built with -fsanitize=cfi on Nvidia
+{{OPT}}urCommandBufferKernelHandleUpdateTest.Success/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.UpdateAgain/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.RestoreOriginalKernel/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.KernelAlternativeNotRegistered/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.RegisterInvalidKernelAlternative/*
+{{OPT}}urCommandBufferValidUpdateParametersTest.UpdateDimensionsWithoutUpdatingKernel/*
+{{OPT}}urCommandBufferValidUpdateParametersTest.UpdateOnlyLocalWorkSize/*
+{{OPT}}urCommandBufferValidUpdateParametersTest.SuccessNullptrHandle/*
+{{OPT}}KernelCommandEventSyncUpdateTest.TwoWaitEvents/*

test/conformance/exp_command_buffer/exp_command_buffer_adapter_hip.match

@@ -0,0 +1,10 @@
+# Note: This file is only for use with cts_exe.py
+# These cause SIGILL when built with -fsanitize=cfi on AMD
+{{OPT}}urCommandBufferKernelHandleUpdateTest.Success/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.UpdateAgain/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.RestoreOriginalKernel/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.KernelAlternativeNotRegistered/*
+{{OPT}}urCommandBufferKernelHandleUpdateTest.RegisterInvalidKernelAlternative/*
+{{OPT}}urCommandBufferValidUpdateParametersTest.UpdateDimensionsWithoutUpdatingKernel/*
+{{OPT}}urCommandBufferValidUpdateParametersTest.UpdateOnlyLocalWorkSize/*
+{{OPT}}urCommandBufferValidUpdateParametersTest.SuccessNullptrHandle/*

test/fuzz/CMakeLists.txt

@@ -51,7 +51,9 @@ target_link_libraries(fuzztest-base
     ${PROJECT_NAME}::headers
     ${PROJECT_NAME}::common
     -fsanitize=fuzzer)
-target_compile_options(fuzztest-base PRIVATE -g -fsanitize=fuzzer)
+# When built with -g and -flto (which is required by some hardening flags), this causes a segfault in (upstream)
+# LLVM 14-15 while linking when CMAKE_BUILD_TYPE is Release
+target_compile_options(fuzztest-base PRIVATE -fsanitize=fuzzer)
 target_compile_definitions(fuzztest-base PRIVATE -DKERNEL_IL_PATH="${UR_CONFORMANCE_DEVICE_BINARIES_DIR}/fill/spir64.bin.0")
 target_include_directories(fuzztest-base PRIVATE ${UR_CONFORMANCE_DEVICE_BINARIES_DIR})
 add_dependencies(fuzztest-base generate_device_binaries)