[CI, enhancement] enable external oneDAL builds in build-and-test-lnx.yml Azure Pipelines

.ci/pipeline/build-and-test-lnx.yml

@@ -18,6 +18,7 @@ steps:
     displayName: "apt-get"
   - script: |
       bash .ci/scripts/install_dpcpp.sh
+    condition: and(eq(variables.NO_DPC, ''), eq(variables.DPCPPROOT, ''))
     displayName: "dpcpp installation"
   - script: |
       source /opt/intel/oneapi/compiler/latest/env/vars.sh
@@ -27,47 +28,73 @@ steps:
       conda config --add channels conda-forge
       conda config --set channel_priority strict
       conda update -y -q conda
-      conda create -q -y -n CB -c conda-forge python=$(PYTHON_VERSION) dal-devel mpich pyyaml "dpcpp-cpp-rt=2024.2.0"
+      conda create -q -y -n CB -c conda-forge python=$(PYTHON_VERSION) mpich pyyaml "dpcpp-cpp-rt=2024.2.0"
     displayName: "Conda create"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
+      if [ -z "${DALROOT}" ]; then conda install -q -y -c conda-forge dal-devel; fi
       pip install -r dependencies-dev
       pip list
+    env:
+      DALROOT: ${{ variables.DALROOT }}
     displayName: "Install develop requirements"
   - script: |
-      export DPCPPROOT=/opt/intel/oneapi/compiler/latest
+      if [ -z "${DPCPPROOT}" ]; then export DPCPPROOT=/opt/intel/oneapi/compiler/latest; fi
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
-      export DALROOT=$CONDA_PREFIX
+      if [ -z "${DALROOT}" ]; then export DALROOT=$CONDA_PREFIX; fi
       ./conda-recipe/build.sh
+    env:
+      NO_DPC: ${{ variables.NO_DPC }}
+      NO_DIST: ${{ variables.NO_DIST }}
+      DPCPPROOT: ${{ variables.DPCPPROOT }}
+      DALROOT: ${{ variables.DALROOT }}
     displayName: "Build daal4py/sklearnex"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       bash .ci/scripts/setup_sklearn.sh $(SKLEARN_VERSION)
       pip install --upgrade -r requirements-test.txt
       pip install $(python .ci/scripts/get_compatible_scipy_version.py)
-      if [ $(echo $(PYTHON_VERSION) | grep '3.9\|3.11') ] && [ $(SKLEARN_VERSION) != "1.0" ]; then conda install -q -y -c https://software.repos.intel.com/python/conda/ dpctl=0.17.0 dpnp=0.15.0; fi
+      if [ $(echo $(PYTHON_VERSION) | grep '3.9\|3.11') ] && [ $(SKLEARN_VERSION) != "1.0" ] && [ -z ${NO_DPC} ]; then conda install -q -y -c https://software.repos.intel.com/python/conda/ dpctl=0.17.0 dpnp=0.15.0; fi
       pip list
+    env:
+      NO_DPC: ${{ variables.NO_DPC }}
     displayName: "Install testing requirements"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       cd ..
       ./s/conda-recipe/run_test.sh --json-report
+    env:
+      DALROOT: ${{ variables.DALROOT }}
+      TBBROOT: ${{ variables.TBBROOT }}
     displayName: "Sklearnex testing"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
-      bash .ci/scripts/run_sklearn_tests.sh cpu
+      if [ -n "${DALROOT}" ] && [ "${DALROOT}" != "${CONDA_PREFIX}" ]; then source ${DALROOT}/env/vars.sh; fi
+      if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then source ${TBBROOT}/env/vars.sh; fi
+      if [ -z "${NO_DPC}" ]; then export CPU="cpu"; fi
+      bash .ci/scripts/run_sklearn_tests.sh $CPU
+    env:
+      DALROOT: ${{ variables.DALROOT }}
+      TBBROOT: ${{ variables.TBBROOT }}
+      NO_DPC: ${{ variables.NO_DPC }}
     displayName: "Sklearn testing"
     condition: succeededOrFailed()
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
-      bash .ci/scripts/run_sklearn_tests.sh cpu
+      if [ -n "${DALROOT}" ] && [ "${DALROOT}" != "${CONDA_PREFIX}" ]; then source ${DALROOT}/env/vars.sh; fi
+      if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then source ${TBBROOT}/env/vars.sh; fi
+      if [ -z "${NO_DPC}" ]; then export CPU="cpu"; fi
+      bash .ci/scripts/run_sklearn_tests.sh $CPU
     env:
+      DALROOT: ${{ variables.DALROOT }}
+      TBBROOT: ${{ variables.TBBROOT }}
+      NO_DPC: ${{ variables.NO_DPC }}
       SKLEARNEX_PREVIEW: "YES"
     displayName: "Sklearn testing [preview]"
     condition: succeededOrFailed()

conda-recipe/build.sh

@@ -25,6 +25,14 @@ fi
 
 if [ -z "${DALROOT}" ]; then
     export DALROOT=${PREFIX}
+elif [ "${DALROOT}" != "${CONDA_PREFIX}" ]; then
+    # source oneDAL if DALROOT is set outside of conda-build
+    source ${DALROOT}/env/vars.sh
+fi
+
+if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then
+# source TBB if TBBROOT is set outside of conda-build
+    source ${TBBROOT}/env/vars.sh
 fi
 
 if [ -z "${MPIROOT}" ] && [ -z "${NO_DIST}" ]; then

conda-recipe/run_test.sh

@@ -36,6 +36,16 @@ if [ -z "${PYTHON}" ]; then
     export PYTHON=python
 fi
 
+if [ -n "${DALROOT}" ] && [ "${DALROOT}" != "${CONDA_PREFIX}" ]; then
+# source oneDAL if DALROOT is set outside of conda-build
+    source ${DALROOT}/env/vars.sh
+fi
+
+if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then
+# source TBB if TBBROOT is set outside of conda-build
+    source ${TBBROOT}/env/vars.sh
+fi
+
 # Note: execute with argument --json-report in order to produce
 # a JSON report under folder '.pytest_reports'. Other arguments
 # will also be forwarded to pytest.

setup.py

@@ -84,13 +84,14 @@
 
 trues = ["true", "True", "TRUE", "1", "t", "T", "y", "Y", "Yes", "yes", "YES"]
 no_dist = True if "NO_DIST" in os.environ and os.environ["NO_DIST"] in trues else False
+no_dpc = True if "NO_DPC" in os.environ and os.environ["NO_DPC"] in trues else False
 no_stream = "NO_STREAM" in os.environ and os.environ["NO_STREAM"] in trues
 debug_build = os.getenv("DEBUG_BUILD") == "1"
 mpi_root = None if no_dist else os.environ["MPIROOT"]
 dpcpp = (
     shutil.which("icpx") is not None
     and "onedal_dpc" in get_onedal_shared_libs(dal_root)
-    and os.environ.get("NO_DPC", None) is None
+    and not no_dpc
     and not (IS_WIN and debug_build)
 )