From 6eb63c1915e6c0d6bb9c5e6774f7efd26470dcb5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 28 Sep 2023 15:40:22 +0100 Subject: [PATCH] oneDAL development env docker image (#2513) * removing JDK deps * Fixing env and introduce docker * Creating bazel env script * adding readme for docker * updating * adding github validation * new line * remove bash invocation for scripts * fixing ci.yml * add execute permissions * fixing ci.yml * fixing github flow name * adding sudo to docker image * fixing install dpcpp script * fixing base docker deps * fixing bazelisk * fixing ci.yml * fixing windows ci with bash call * fixing bazel ci * updating bazel readme to current state * fixing bazel caching and nuget windows build * adding git binary * adding build validation and conda to env * enable larger validation in CI * fixing CLI * adding conda to docker * fixing docker * fixing docker * adding deps and build * adding docs env and more tests for nightly * not passing param * switching to conda env file * adding conda env * fixign path to env.yml * drop caching version to 1 as bazel version was fixed * bump version for spxinks-prompt to 1.8 * revert version * disabling doc reqs * switchign to latest tag * actually running code in docker * merging docker commands in one * merging docker commands in one * merging docker commands in one * installing bazel in docker image * adding build with OpenBLAS to validation * removing build with OpenBLAS to validation * Apply suggestions from code review Co-authored-by: Alexandra * Apply suggestions from code review Co-authored-by: Alexander Andreev --------- Co-authored-by: Alexandra Co-authored-by: Alexander Andreev --- .ci/env/{oneapi.sh => apt.sh} | 36 ++++++++--- .../install_dpcpp.sh => env/bazelisk.sh} | 19 +++--- .ci/env/environment.yml | 7 +++ .ci/env/tbb.bat | 0 .ci/pipeline/ci.yml | 49 ++++++--------- .github/workflows/docker-validation-ci.yml | 19 ++++++ .../workflows/docker-validation-nightly.yml | 35 +++++++++++ INSTALL.md | 5 ++ dev/bazel/README.md | 28 ++++++--- dev/bazel/TODO.md | 26 +++++--- dev/docker/README.md | 28 +++++++++ dev/docker/onedal-dev.Dockerfile | 59 +++++++++++++++++++ 12 files changed, 247 insertions(+), 64 deletions(-) rename .ci/env/{oneapi.sh => apt.sh} (68%) rename .ci/{scripts/install_dpcpp.sh => env/bazelisk.sh} (54%) create mode 100644 .ci/env/environment.yml mode change 100644 => 100755 .ci/env/tbb.bat create mode 100644 .github/workflows/docker-validation-ci.yml create mode 100644 .github/workflows/docker-validation-nightly.yml create mode 100644 dev/docker/README.md create mode 100644 dev/docker/onedal-dev.Dockerfile diff --git a/.ci/env/oneapi.sh b/.ci/env/apt.sh similarity index 68% rename from .ci/env/oneapi.sh rename to .ci/env/apt.sh index 55ecdc9f3ed..420b958385d 100755 --- a/.ci/env/oneapi.sh +++ b/.ci/env/apt.sh @@ -17,6 +17,10 @@ component=$1 +function update { + sudo apt-get update +} + function add_repo { wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB @@ -27,14 +31,7 @@ function add_repo { } function install_dpcpp { - sudo apt-get install \ - intel-oneapi-common-vars \ - intel-oneapi-common-licensing \ - intel-oneapi-tbb-devel \ - intel-oneapi-dpcpp-cpp-compiler \ - intel-oneapi-dev-utilities \ - intel-oneapi-libdpstd-devel \ - cmake + sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1 sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd' sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_ } @@ -43,14 +40,35 @@ function install_mkl { sudo apt-get install intel-oneapi-mkl-devel } +function install_clang-format { + sudo apt-get install -y clang-format-14 + sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100 + sudo update-alternatives --set clang-format /usr/bin/clang-format-14 +} + +function install_dev-base { + sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree +} + +function install_dev-base-conda { + conda env create -f .ci/env/environment.yml +} + if [ "${component}" == "dpcpp" ]; then add_repo install_dpcpp elif [ "${component}" == "mkl" ]; then add_repo install_mkl +elif [ "${component}" == "clang-format" ]; then + update + install_clang-format +elif [ "${component}" == "dev-base" ]; then + update + install_dev-base + install_dev-base-conda else echo "Usage:" - echo " $0 [dpcpp|mkl]" + echo " $0 [dpcpp|mkl|clang-format|dev-base]" exit 1 fi diff --git a/.ci/scripts/install_dpcpp.sh b/.ci/env/bazelisk.sh similarity index 54% rename from .ci/scripts/install_dpcpp.sh rename to .ci/env/bazelisk.sh index 7df36449d9d..0d772327b75 100755 --- a/.ci/scripts/install_dpcpp.sh +++ b/.ci/env/bazelisk.sh @@ -15,12 +15,13 @@ # limitations under the License. #=============================================================================== -wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB -sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB -rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB -echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list -sudo add-apt-repository -y "deb https://apt.repos.intel.com/oneapi all main" -sudo apt-get update -sudo apt-get install -y intel-dpcpp-cpp-compiler-2023.2.1 -sudo bash -c 'echo libintelocl.so > /etc/OpenCL/vendors/intel-cpu.icd' -sudo mv -f /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga_ +# Download Bazelisk +export SHA256="ce52caa51ef9e509fb6b7e5ad892e5cf10feb0794b0aed4d2f36adb00a1a2779 bazelisk-linux-amd64" +wget https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 +echo ${SHA256} | sha256sum --check +# "Install" bazelisk +chmod +x bazelisk-linux-amd64 +mkdir -p bazel/bin +mv bazelisk-linux-amd64 bazel/bin/bazel +export BAZEL_VERSION=$(./bazel/bin/bazel --version | awk '{print $2}') +export PATH=$PATH:$(pwd)/bazel/bin diff --git a/.ci/env/environment.yml b/.ci/env/environment.yml new file mode 100644 index 00000000000..0a704f66f14 --- /dev/null +++ b/.ci/env/environment.yml @@ -0,0 +1,7 @@ +name: ci-env +channels: + - conda-forge + - intel + - defaults +dependencies: + - impi-devel=2021.10.0 diff --git a/.ci/env/tbb.bat b/.ci/env/tbb.bat old mode 100644 new mode 100755 diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 947c7cc0151..6a8bf2457c4 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -28,9 +28,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y clang-format-14 - sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14 100 - sudo update-alternatives --set clang-format /usr/bin/clang-format-14 + .ci/env/apt.sh clang-format displayName: 'apt-get' - script: | .ci/scripts/clang-format.sh @@ -46,8 +44,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree - conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.10.0 openjdk=17.0.3 + .ci/env/apt.sh dev-base displayName: 'apt-get and conda install' - script: | .ci/scripts/describe_system.sh @@ -95,8 +92,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix tree - conda create -n ci-env -q -y -c intel -c conda-forge impi-devel=2021.8.0 openjdk=17.0.3 + .ci/env/apt.sh dev-base displayName: 'apt-get and conda install' - script: | .ci/scripts/describe_system.sh @@ -136,11 +132,10 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - sudo apt-get update && sudo apt-get install -y gcc-multilib g++-multilib dos2unix - conda create -n ci-env -q -y -c conda-forge -c intel impi-devel=2021.10.0 openjdk=17.0.3 + .ci/env/apt.sh dev-base displayName: 'apt-get and conda install' - script: | - bash .ci/scripts/install_dpcpp.sh + .ci/env/apt.sh dpcpp displayName: 'dpcpp installation' - script: | source /opt/intel/oneapi/compiler/latest/env/vars.sh @@ -190,27 +185,21 @@ jobs: BAZEL_CACHE_MAX_SIZE_KB: 4194304 # Size in kilobytes ~ 4Gb steps: - script: | - # Download Bazelisk - export SHA256="168851e70cf5f95c0e215e7f3aaca5132ffc3c8dd8f585a4157b0be2b53cfe32 bazelisk-linux-amd64" - wget https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 - echo ${SHA256} | sha256sum --check - # "Install" bazelisk - chmod +x bazelisk-linux-amd64 - mv bazelisk-linux-amd64 ${BAZEL_VERSION} - - displayName: 'install-deps' + # sourcing done to set bazel version value from script + source .ci/env/bazelisk.sh + echo "##vso[task.setvariable variable=BAZEL_VERSION]${BAZEL_VERSION}" + displayName: 'install-bazel' - script: | .ci/scripts/describe_system.sh displayName: 'System info' - - task: Cache@2 inputs: # Commit ID is added to a cache key. Caches are immutable by design, # so we always need to change a key to upload the last version # of the Bazel cache. Cache lookup is based on `restoreKeys` option. - key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2" | "$(Build.SourceVersion)"' + key: '"$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1" | "$(Build.SourceVersion)"' restoreKeys: | - "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v2" + "$(BAZEL_VERSION)" | "$(Agent.OS)" | "v1" path: $(BAZEL_CACHE_DIR) displayName: 'bazel-cache' @@ -227,13 +216,13 @@ jobs: echo "build --disk_cache=$(BAZEL_CACHE_DIR) --cpu=avx2" > ~/.bazelrc # Display oneDAL build configuration - $(BAZEL_VERSION) build @config//:dump + bazel build @config//:dump echo cat bazel-bin/external/config/config.json echo displayName: 'bazel-configure' - script: | - $(BAZEL_VERSION) build :release + bazel build :release displayName: 'release' - task: PublishPipelineArtifact@1 inputs: @@ -241,27 +230,27 @@ jobs: targetPath: '$(Build.Repository.LocalPath)/bazel-bin/release' displayName: 'Upload build artifacts' - script: | - $(BAZEL_VERSION) test //examples/oneapi/cpp:all \ + bazel test //examples/oneapi/cpp:all \ --test_link_mode=dev \ --test_thread_mode=par displayName: 'cpp-examples-thread-dev' - script: | export DALROOT=`pwd`/bazel-bin/release/daal/latest - $(BAZEL_VERSION) test //examples/oneapi/cpp:all \ + bazel test //examples/oneapi/cpp:all \ --test_link_mode=release_static \ --test_thread_mode=par displayName: 'cpp-examples-thread-release-static' - script: | export DALROOT=`pwd`/bazel-bin/release/daal/latest - $(BAZEL_VERSION) test //examples/oneapi/cpp:all \ + bazel test //examples/oneapi/cpp:all \ --test_link_mode=release_dynamic \ --test_thread_mode=par displayName: 'cpp-examples-thread-release-dynamic' - script: | - $(BAZEL_VERSION) test //cpp/oneapi/dal:tests \ + bazel test //cpp/oneapi/dal:tests \ --config=host \ --test_link_mode=dev \ --test_thread_mode=par @@ -310,7 +299,7 @@ jobs: inputs: artifact: '$(platform.type) build' path: $(Pipeline.Workspace) - - script: sudo apt-get update && sudo apt-get install -y git gcc-multilib g++-multilib + - script: .ci/env/apt.sh dev-base displayName: 'apt-get' - script: | .ci/scripts/describe_system.sh @@ -392,7 +381,7 @@ jobs: steps: - script: | brew install dos2unix tree - conda create -n ci-env -q -y -c conda-forge python=3.9 openjdk=17.0.3 + conda create -n ci-env -q -y -c conda-forge python=3.10 source /usr/local/miniconda/etc/profile.d/conda.sh conda activate ci-env pip install -q cpufeature diff --git a/.github/workflows/docker-validation-ci.yml b/.github/workflows/docker-validation-ci.yml new file mode 100644 index 00000000000..3c750f8712b --- /dev/null +++ b/.github/workflows/docker-validation-ci.yml @@ -0,0 +1,19 @@ +name: docker-validation CI + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + validate: + name: Docker validation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + - name: Build docker image + run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest diff --git a/.github/workflows/docker-validation-nightly.yml b/.github/workflows/docker-validation-nightly.yml new file mode 100644 index 00000000000..c4416bc42a0 --- /dev/null +++ b/.github/workflows/docker-validation-nightly.yml @@ -0,0 +1,35 @@ +name: docker-validation Nightly + +on: + schedule: + - cron: "0 23 * * *" + pull_request: + branches: + - master + paths: + - .github/workflows/docker-validation-nightly.yml + - dev/docker/onedal-dev.Dockerfile + - .ci/env + - .ci/scripts + push: + branches: + - master + paths: + - .github/workflows/docker-validation-nightly.yml + - dev/docker/onedal-dev.Dockerfile + - .ci/env + - .ci/scripts + +jobs: + validate: + name: Docker validation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + - name: Build docker image + run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest + - name: Building oneDAL + run: docker run onedal-dev .ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal --conda-env ci-env + - name: Building oneDAL with bazel + run: docker run onedal-dev bazel build :release diff --git a/INSTALL.md b/INSTALL.md index b42ae9f988c..ef09744498b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -28,6 +28,11 @@ Required Software: For details, see [System Requirements for oneDAL](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/system-requirements-for-oneapi-data-analytics-library.html). +## Docker Development Environment + +[Docker file](https://github.com/oneapi-src/oneDAL/tree/master/dev/docker) with the oneDAL development environment +is available as an alternative to the manual setup. + ## Installation Steps 1. Clone the sources from GitHub\* as follows: diff --git a/dev/bazel/README.md b/dev/bazel/README.md index e806f0d39e8..f569e2bb429 100644 --- a/dev/bazel/README.md +++ b/dev/bazel/README.md @@ -1,8 +1,24 @@ + # Bazel Guide ## Install Bazel on Linux -1. Download Bazel 4.0.0 +1. Download Bazelisk ```sh - wget -O bazel https://github.com/bazelbuild/bazel/releases/download/4.0.0/bazel-4.0.0-linux-x86_64 + wget -O bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 ``` > Note: If you are using proxy don't forget to set `http_proxy` and `https_proxy` environment variables @@ -10,8 +26,8 @@ 2. Put it somewhere on **local disk** (use of remote shared drives is not recommended) and set executable attributes. ```sh - mv bazel /bin chmod +x bazel + mv bazel /bin ``` 3. Create Bazel cache directory on **local disk** and @@ -24,7 +40,7 @@ 4. Add `bazel` to the `$PATH`. ```sh export PATH=/bin:$PATH - bazel --version # Should be "bazel 4.0.0" + bazel --version ``` ### Compiler choice @@ -197,10 +213,6 @@ The most used Bazel commands are `build`, `test` and `run`. bazel test --test_link_mode=release_dynamic //cpp/oneapi/dal:tests ``` -- `--test_thread_mode` Specifies threading mode for tests. \ - Possible values: - - `par` _default_ Links against `onedal_thread`. - Example: ```sh bazel test --test_thread_mode=par //cpp/oneapi/dal:tests diff --git a/dev/bazel/TODO.md b/dev/bazel/TODO.md index 34a71a5d0bb..4a81faf5a5b 100644 --- a/dev/bazel/TODO.md +++ b/dev/bazel/TODO.md @@ -1,3 +1,20 @@ + + ### TODO: - [ ] **Windows support.** Bazel shall provide toolchain implementation for Windows. @@ -9,18 +26,11 @@ | | Intel | DPC++ | GCC | Clang | MSVC | |---------|:------------------:|:------------------:|:------------------:|:------------------:|:------------------:| | Linux | :x: | :x: | :heavy_check_mark: | | | - | MacOs | :x: | :x: | | :x: | | | Windows | :x: | :x: | | | :x: | -- [ ] **Automatic makefile generation for examples on Linux/MacOs.** Bazel - shall generate Makefile for examples when release build is required. - -- [ ] **Automatic VS solution generation for examples on Windows.** Bazel shall - generate Visual Studio solution for examples then release build is required. - - [ ] **Automatic host architecture identification.** Bazel shall detect host machine architecture and configure best CPU id automatically. - [ ] **Toolchain code unification.** There is logic duplication for toolchain - configuration on Linux/MacOs/Windows. + configuration on Linux/Windows. diff --git a/dev/docker/README.md b/dev/docker/README.md new file mode 100644 index 00000000000..df4ba32bd63 --- /dev/null +++ b/dev/docker/README.md @@ -0,0 +1,28 @@ + + +# Docker Development Environment +## How To Use +There is a simple docker dev environment for the oneDAL development and build process. +It includes dependencies for building all oneDAL components with ``make`` and ``bazel`` + +Note: The docker setup assumes that it is executed from the oneDAL repo and copies repo files inside the container + +For that, run: + ```sh + docker run -it onedal-dev /bin/bash + ``` diff --git a/dev/docker/onedal-dev.Dockerfile b/dev/docker/onedal-dev.Dockerfile new file mode 100644 index 00000000000..36ae0702dff --- /dev/null +++ b/dev/docker/onedal-dev.Dockerfile @@ -0,0 +1,59 @@ +#=============================================================================== +# Copyright 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +FROM ubuntu:22.04 + +ARG workdirectory="/sources/oneDAL" + +ADD ../../ ${workdirectory} + +WORKDIR ${workdirectory} + +#Env setup +RUN apt-get update && \ + apt-get -y install sudo wget gnupg git make python3-setuptools doxygen + +# Install miniconda +ENV CONDA_DIR /opt/conda +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda + +# Put conda in path to use conda activate +ENV PATH=$CONDA_DIR/bin:$PATH + +# Installing environment for base development dependencies +RUN .ci/env/apt.sh dev-base + +# Installing environment for DPCPP development dependencies +RUN .ci/env/apt.sh dpcpp + +# Installing environment for clang-format +RUN .ci/env/apt.sh clang-format + +# Installing environment for bazel +RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 && \ + chmod 755 bazelisk-linux-amd64 && \ + mv bazelisk-linux-amd64 /usr/bin/bazel + +# Installing openBLAS dependency +RUN .ci/env/openblas.sh + +# Installing MKL dependency +RUN ./dev/download_micromkl.sh + +# Installing oneTBB dependency +RUN ./dev/download_tbb.sh +