Merge branch 'main' into dhanus/openrng-build

.ci/env/apt.sh

@@ -23,21 +23,19 @@ function update {
 }
 
 function add_repo {
-    wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-    sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-    rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-    echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
-    sudo add-apt-repository -y "deb https://apt.repos.intel.com/oneapi all main"
-    sudo apt-get update
+    wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | \
+        gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
+        | sudo tee /etc/apt/sources.list.d/oneAPI.list
+    sudo apt update
 }
 
 function install_dpcpp {
-    sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-2024.2
-    sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd'
+    sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp intel-oneapi-runtime-libs
 }
 
 function install_mkl {
-    sudo apt-get install intel-oneapi-mkl-devel
+    sudo apt-get install -y intel-oneapi-mkl-devel=2024.2.1-103
 }
 
 function install_clang-format {

.ci/env/environment.yml

@@ -1,6 +1,5 @@
 name: ci-env
 channels:
   - conda-forge
-  - defaults
 dependencies:
   - impi-devel=2021.12.0

.ci/pipeline/ci.yml

@@ -28,6 +28,9 @@ variables:
   TBB_VERSION : 'v2021.10.0'
   VM_IMAGE : 'ubuntu-22.04'
   SYSROOT_OS: 'jammy'
+  WIN_BASEKIT_VERSION: '2024.2.1.101'
+  WINDOWS_BASEKIT_URL: 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d91caaa0-7306-46ea-a519-79a0423e1903/w_BaseKit_p_$(WIN_BASEKIT_VERSION)_offline.exe'
+  WINDOWS_DPCPP_COMPONENTS: 'intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel'
 
 jobs:
 - job: 'FormatterChecks'
@@ -58,6 +61,9 @@ jobs:
   - script: |
       .ci/env/apt.sh dev-base
     displayName: 'apt-get and conda install'
+  - script: |
+      .ci/env/apt.sh mkl
+    displayName: 'mkl installation'
   - script: |
       .ci/scripts/describe_system.sh
     displayName: 'System info'
@@ -415,6 +421,9 @@ jobs:
   - script: |
       .ci/env/apt.sh dpcpp
     displayName: 'dpcpp installation'
+  - script: |
+      .ci/env/apt.sh mkl
+    displayName: 'mkl installation'
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
       .ci/scripts/describe_system.sh
@@ -435,14 +444,17 @@ jobs:
     continueOnError: true
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      source /opt/intel/oneapi/mkl/latest/env/vars.sh
       .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface daal/cpp --build-system cmake
     displayName: 'daal/cpp examples'
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      source /opt/intel/oneapi/mkl/latest/env/vars.sh
       .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface oneapi/cpp --build-system cmake
     displayName: 'oneapi/cpp examples'
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
+      source /opt/intel/oneapi/mkl/latest/env/vars.sh
       .ci/scripts/test.sh --test-kind samples --build-dir $(release.dir) --compiler gnu --interface daal/cpp/mpi --conda-env ci-env --build-system cmake
     displayName: 'daal/cpp/mpi samples'
   - task: PublishPipelineArtifact@1
@@ -520,12 +532,15 @@ jobs:
                  --test_thread_mode=par
     displayName: 'cpp-examples-thread-release-static'
 
-  - script: |
-      export DALROOT=`pwd`/bazel-bin/release/daal/latest
-      bazel test //examples/oneapi/cpp:all \
-                 --test_link_mode=release_dynamic \
-                 --test_thread_mode=par
-    displayName: 'cpp-examples-thread-release-dynamic'
+  # The issue that bazel doesnt link MKL libs via -Wl, --start-group..
+  # oneDAL make build pass this test
+  # TODO: add cycle linking in bazel
+  # - script: |
+  #     export DALROOT=`pwd`/bazel-bin/release/daal/latest
+      # bazel test //examples/oneapi/cpp:all \
+      #            --test_link_mode=release_dynamic \
+      #            --test_thread_mode=par
+  #   displayName: 'cpp-examples-thread-release-dynamic'
 
   - script: |
       bazel test //cpp/daal:tests
@@ -620,7 +635,7 @@ jobs:
       conda activate CB
       source $(Pipeline.Workspace)/daal/latest/env/vars.sh
       ./sklearnex/conda-recipe/run_test.sh
-    timeoutInMinutes: 15
+    timeoutInMinutes: 20
     displayName: sklearnex test
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
@@ -645,57 +660,6 @@ jobs:
     displayName: 'Upload conformance tests artifacts'
     continueOnError: true
 
-- job: 'macOSMakeClang'
-  timeoutInMinutes: 0
-  variables:
-    release.dir: '__release_mac_clang'
-    platform.type : 'mac32e'
-  pool:
-    vmImage:  'macos-12'
-  steps:
-  - script: |
-      brew install dos2unix tree
-      conda create -n ci-env -q -y -c conda-forge python=3.10
-      source /usr/local/miniconda/etc/profile.d/conda.sh
-      conda activate ci-env
-      pip install -q cpufeature
-    displayName: 'brew and conda install'
-  - script: |
-      source /usr/local/miniconda/etc/profile.d/conda.sh
-      conda activate ci-env
-      .ci/scripts/describe_system.sh
-    displayName: 'System info'
-  - script: |
-      .ci/scripts/build.sh --compiler clang --target daal --optimizations "sse2 avx2" --conda-env ci-env
-    displayName: 'make daal'
-  - script: |
-      .ci/scripts/build.sh --compiler clang --target onedal_c --optimizations "sse2 avx2"
-    displayName: 'make onedal_c'
-  - task: PublishPipelineArtifact@1
-    inputs:
-      artifactName: '$(platform.type) build'
-      targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
-    displayName: 'Upload build artifacts'
-    continueOnError: true
-  - script: |
-      .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface daal/cpp --build-system cmake
-    displayName: 'daal/cpp examples'
-  - script: |
-      .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler clang --interface oneapi/cpp --build-system cmake
-    displayName: 'oneapi/cpp examples'
-  - script: |
-      deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes
-      tree -h -I include __nuget/inteldal*/
-      ls -lh __nuget/inteldal*.nupkg
-    displayName: 'nuget pkg'
-  - task: PublishPipelineArtifact@1
-    inputs:
-      artifactName: '$(platform.type) fail'
-      targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
-    displayName: 'Uploading on fail'
-    condition: failed()
-    continueOnError: true
-
 - job: 'WindowsMakeVC'
   timeoutInMinutes: 0
   variables:
@@ -704,6 +668,8 @@ jobs:
   pool:
     vmImage: 'windows-2022'
   steps:
+  - script: .ci/scripts/install_windows.bat $(WINDOWS_BASEKIT_URL) $(WINDOWS_DPCPP_COMPONENTS)
+    displayName: Install oneAPI Base Toolkit
   - script: |
       set PATH=C:\msys64\usr\bin;%PATH%
       pip install cpufeature
@@ -714,9 +680,13 @@ jobs:
       bash .ci/scripts/describe_system.sh
     displayName: 'System info'
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
+      set MKL_FPK_GPU_VERSION_LINE=2024.0.0
       .\.ci\scripts\build.bat daal vc avx2
     displayName: 'make daal'
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
+      set MKL_FPK_GPU_VERSION_LINE=2024.0.0
       .\.ci\scripts\build.bat onedal_c vc avx2
     displayName: 'make onedal_c'
   - task: PublishPipelineArtifact@1
@@ -726,10 +696,12 @@ jobs:
     displayName: 'Upload build artifacts'
     continueOnError: true
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
       .\.ci\scripts\test.bat daal\cpp lib msvs cmake
       .\.ci\scripts\test.bat daal\cpp dll msvs cmake
     displayName: 'daal/cpp examples'
   - script: |
+      call C:\temp\oneapi\setvars.bat --force
       .\.ci\scripts\test.bat oneapi\cpp lib msvs cmake
       .\.ci\scripts\test.bat oneapi\cpp dll msvs cmake
     displayName: 'oneapi/cpp examples'

.ci/scripts/build.bat

@@ -28,14 +28,18 @@ set PATH=C:\msys64\usr\bin;%PATH%
 echo pacman -S --noconfirm msys/make msys/dos2unix
 pacman -S --noconfirm msys/make msys/dos2unix
 
-echo call .ci\env\tbb.bat
-if "%TBBROOT%"=="" if not exist .\__deps\tbb\win\tbb call .ci\env\tbb.bat || set errorcode=1
-
-echo call .\dev\download_micromkl.bat
-if "%MKLGPUFPKROOT%"=="" if not exist .\__deps\mklgpufpk\win call .\dev\download_micromkl.bat || set errorcode=1
-
-echo call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
-if "%VISUALSTUDIOVERSION%"=="" call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64 || set errorcode=1
+IF "%VS_VER%"=="2017_build_tools" (
+    @call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    echo "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+) ELSE (
+    IF "%VS_VER%"=="2019_build_tools" (
+        @call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+        echo "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    ) ELSE (
+          @call "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+          echo "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+    )
+)
 
 echo make %1 -j%NUMBER_OF_PROCESSORS% COMPILER=%2 PLAT=win32e REQCPU=%3
 make %1 -j%NUMBER_OF_PROCESSORS% COMPILER=%2 PLAT=win32e REQCPU=%3 || set errorcode=1

.ci/scripts/build.sh

@@ -164,8 +164,7 @@ fi
 #main actions
 echo "Call env scripts"
 if [ "${backend_config}" == "mkl" ]; then
-    echo "Sourcing MKL env"
-    "${ONEDAL_DIR}"/dev/download_micromkl.sh with_gpu="${with_gpu}"
+    source /opt/intel/oneapi/mkl/latest/env/vars.sh
 elif [ "${backend_config}" == "ref" ] && [ ! -z "${BLAS_INSTALL_DIR}" ]; then
     export OPENBLASROOT="${BLAS_INSTALL_DIR}"
 elif [ "${backend_config}" == "ref" ]; then

.ci/scripts/install_windows.bat

@@ -0,0 +1,31 @@
+@echo off
+rem ============================================================================
+rem Copyright contributors to the oneDAL project
+rem
+rem Licensed under the Apache License, Version 2.0 (the "License");
+rem you may not use this file except in compliance with the License.
+rem You may obtain a copy of the License at
+rem
+rem     http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+rem ============================================================================
+
+set URL=%1
+set COMPONENTS=%2
+
+curl.exe --output %TEMP%\webimage.exe --url %URL% --retry 5 --retry-delay 5
+start /b /wait %TEMP%\webimage.exe -s -x -f webimage_extracted --log extract.log
+del %TEMP%\webimage.exe
+if "%COMPONENTS%"=="" (
+  webimage_extracted\bootstrapper.exe -s --action install --eula=accept --install-dir=C:\temp\oneapi\ -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=.
+) else (
+  webimage_extracted\bootstrapper.exe -s --action install --components=%COMPONENTS% --eula=accept --install-dir=C:\temp\oneapi\ -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=.
+)
+set installer_exit_code=%ERRORLEVEL%
+rd /s/q "webimage_extracted"
+exit /b %installer_exit_code%

.ci/scripts/test.bat

@@ -34,8 +34,20 @@ echo CPUCOUNT=%CPUCOUNT%
 echo PATH=C:\msys64\usr\bin;%PATH%
 set PATH=C:\msys64\usr\bin;%PATH%
 
-echo call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
-call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64 || set errorcode=1
+echo "%VISUALSTUDIOVERSION% HERE"
+
+IF "%VS_VER%"=="2017_build_tools" (
+    @call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    echo "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+) ELSE (
+    IF "%VS_VER%"=="2019_build_tools" (
+        @call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+        echo "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+    ) ELSE (
+          @call "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+          echo "C:\temp\oneapi\setvars-vcvarsall.bat" %VS_VER%
+    )
+)
 
 echo call __release_win_vc\daal\latest\env\vars.bat
 call __release_win_vc\daal\latest\env\vars.bat || set errorcode=1

.github/CODEOWNERS

@@ -1,6 +1,6 @@
 # Documentation owners and reviewers
-/docs/         @Vika-F @maria-Petrova @Alexsandruss @bdmoore1
-*.md           @Vika-F @maria-Petrova @Alexsandruss @bdmoore1
+/docs/         @Vika-F @maria-Petrova @Alexsandruss @emmwalsh
+*.md           @Vika-F @maria-Petrova @Alexsandruss @emmwalsh
 
 # TTP files
 third-party*   @maria-Petrova

.github/ISSUE_TEMPLATE/RFC.md

@@ -0,0 +1,20 @@
+---
+name: RFC for new interface
+about: Use this template to request new functionality or change the behavior of the library
+title: ''
+labels: 'RFC'
+assignees: ''
+---
+
+**Summary**
+Include a short summary of the request. Sections below provide guidance on
+what factors are considered important. 
+
+**Problem statement**
+Describe the problem you are trying to solve with a reasonable level of detail.
+
+**Details**
+* The definition of the function including interface and semantics. Please include how this
+interface will be extendable for different hardware implementations.
+* What existing libraries have implementation of this function and can be used
+under oneDAL interface.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -24,6 +24,6 @@ Describe what your are expecting from steps above
 If applicable, add output/screenshots to help explain your problem.
 
 **Environment:**
- - OS: [e.g. Ubuntu 18.04]
- - Compiler: [e.g. GCC9.2]
- - Version: [e.g. 2019 Update 4]
+ - OS: [e.g. Ubuntu 22.04]
+ - Compiler: [e.g. GCC12.1]
+ - Version: [e.g. 2025.1]

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,19 @@
+---
+name: Request a documentation change
+about: Use this template to report documentation issue or request documentation changes
+title: ''
+labels: 'documentation'
+assignees: ''
+---
+
+**Summary**
+Include a short summary of the issue or request. Sections below provide
+guidance on what factors are considered important for a documentation
+issue.
+
+**URLs**
+Include pointers to documents that are impacted.
+
+**Additional details**
+Please provide a detailed description of the expected changes in documentation
+and any suggestions that you may have.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Request a feature
+about: Use this template to request new functionality or change the behavior of the library
+title: ''
+labels: 'new feature'
+assignees: ''
+---
+
+**Summary**
+Include a short summary of the request. 
+
+See the sections below for factors important for a feature request.
+
+**Problem Statement**
+Describe the problem you want to solve with a reasonable level of detail.
+
+**Preferred Solution**
+Provide your ideas regarding problem solutions.

.github/workflows/label-enforcement.yml

@@ -3,6 +3,9 @@ on:
   pull_request:
     branches: [ "main" ]
 
+permissions:
+  contents: read
+
 jobs:
   label_checker:
     name: Please include labels on your pull request