From 41612fbab023326f4d571af75ebd92ed83c04696 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 15 Jul 2024 14:27:30 -0700 Subject: [PATCH] Cleanup docker image build workflows and scripts --- common/aotriton_version.txt | 5 - common/install_aotriton.sh | 21 - common/install_conda.sh | 20 - common/install_cpython.sh | 94 ---- common/install_cuda.sh | 239 ---------- common/install_cuda_aarch64.sh | 92 ---- common/install_jni.sh | 6 - common/install_libpng.sh | 22 - common/install_magma.sh | 28 -- common/install_miopen.sh | 133 ------ common/install_mkl.sh | 15 - common/install_mnist.sh | 12 - common/install_openblas.sh | 21 - common/install_openssl.sh | 14 - common/install_patchelf.sh | 15 - common/install_rocm.sh | 152 ------ common/install_rocm_drm.sh | 149 ------ common/install_rocm_magma.sh | 47 -- common/install_xpu.sh | 156 ------ conda/Dockerfile | 95 ---- conda/README.md | 8 - conda/build_all_docker.sh | 9 - conda/build_docker.sh | 75 --- libtorch/Dockerfile | 106 ----- libtorch/build_docker.sh | 92 ---- manywheel/Dockerfile | 201 -------- manywheel/Dockerfile_2014 | 152 ------ manywheel/Dockerfile_2_28 | 153 ------ manywheel/Dockerfile_2_28_aarch64 | 56 --- manywheel/Dockerfile_aarch64 | 93 ---- manywheel/Dockerfile_cuda_aarch64 | 90 ---- manywheel/Dockerfile_cxx11-abi | 71 --- manywheel/Dockerfile_s390x | 73 --- manywheel/LICENSE | 21 - manywheel/README.md | 28 -- manywheel/build.sh | 25 - manywheel/build_all_docker.sh | 26 - manywheel/build_common.sh | 501 -------------------- manywheel/build_cpu.sh | 68 --- manywheel/build_cuda.sh | 304 ------------ manywheel/build_docker.sh | 149 ------ manywheel/build_libtorch.sh | 357 -------------- manywheel/build_rocm.sh | 288 ----------- manywheel/build_scripts/build.sh | 130 ----- manywheel/build_scripts/build_utils.sh | 90 ---- manywheel/build_scripts/manylinux1-check.py | 53 --- manywheel/build_scripts/ssl-check.py | 33 -- manywheel/test_wheel.sh | 27 -- 48 files changed, 4615 deletions(-) delete mode 100644 common/aotriton_version.txt delete mode 100644 common/install_aotriton.sh delete mode 100644 common/install_conda.sh delete mode 100755 common/install_cpython.sh delete mode 100644 common/install_cuda.sh delete mode 100644 common/install_cuda_aarch64.sh delete mode 100644 common/install_jni.sh delete mode 100644 common/install_libpng.sh delete mode 100644 common/install_magma.sh delete mode 100644 common/install_miopen.sh delete mode 100644 common/install_mkl.sh delete mode 100644 common/install_mnist.sh delete mode 100644 common/install_openblas.sh delete mode 100644 common/install_openssl.sh delete mode 100644 common/install_patchelf.sh delete mode 100644 common/install_rocm.sh delete mode 100644 common/install_rocm_drm.sh delete mode 100644 common/install_rocm_magma.sh delete mode 100644 common/install_xpu.sh delete mode 100644 conda/Dockerfile delete mode 100755 conda/build_all_docker.sh delete mode 100755 conda/build_docker.sh delete mode 100644 libtorch/Dockerfile delete mode 100755 libtorch/build_docker.sh delete mode 100644 manywheel/Dockerfile delete mode 100644 manywheel/Dockerfile_2014 delete mode 100644 manywheel/Dockerfile_2_28 delete mode 100644 manywheel/Dockerfile_2_28_aarch64 delete mode 100644 manywheel/Dockerfile_aarch64 delete mode 100644 manywheel/Dockerfile_cuda_aarch64 delete mode 100644 manywheel/Dockerfile_cxx11-abi delete mode 100644 manywheel/Dockerfile_s390x delete mode 100644 manywheel/LICENSE delete mode 100644 manywheel/README.md delete mode 100755 manywheel/build.sh delete mode 100644 manywheel/build_all_docker.sh delete mode 100644 manywheel/build_common.sh delete mode 100755 manywheel/build_cpu.sh delete mode 100644 manywheel/build_cuda.sh delete mode 100755 manywheel/build_docker.sh delete mode 100644 manywheel/build_libtorch.sh delete mode 100755 manywheel/build_rocm.sh delete mode 100644 manywheel/build_scripts/build.sh delete mode 100755 manywheel/build_scripts/build_utils.sh delete mode 100644 manywheel/build_scripts/manylinux1-check.py delete mode 100644 manywheel/build_scripts/ssl-check.py delete mode 100755 manywheel/test_wheel.sh diff --git a/common/aotriton_version.txt b/common/aotriton_version.txt deleted file mode 100644 index 00f3f90cb..000000000 --- a/common/aotriton_version.txt +++ /dev/null @@ -1,5 +0,0 @@ -0.6b -manylinux_2_17 -rocm6.1 -04b5df8c8123f90cba3ede7e971e6fbc6040d506 -77c29fa3f3b614e187d7213d745e989a92708cee2bc6020419ab49019af399d1 diff --git a/common/install_aotriton.sh b/common/install_aotriton.sh deleted file mode 100644 index 35de8bcf8..000000000 --- a/common/install_aotriton.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ex - -TARBALL='aotriton.tar.bz2' -# This read command alwasy returns with exit code 1 -read -d "\n" VER MANYLINUX ROCMBASE PINNED_COMMIT SHA256 < aotriton_version.txt || true -ARCH=$(uname -m) -AOTRITON_INSTALL_PREFIX="$1" -AOTRITON_URL="https://github.com/ROCm/aotriton/releases/download/${VER}/aotriton-${VER}-${MANYLINUX}_${ARCH}-${ROCMBASE}-shared.tar.bz2" - -cd "${AOTRITON_INSTALL_PREFIX}" -# Must use -L to follow redirects -curl -L --retry 3 -o "${TARBALL}" "${AOTRITON_URL}" -ACTUAL_SHA256=$(sha256sum "${TARBALL}" | cut -d " " -f 1) -if [ "${SHA256}" != "${ACTUAL_SHA256}" ]; then - echo -n "Error: The SHA256 of downloaded tarball is ${ACTUAL_SHA256}," - echo " which does not match the expected value ${SHA256}." - exit -fi -tar xf "${TARBALL}" && rm -rf "${TARBALL}" diff --git a/common/install_conda.sh b/common/install_conda.sh deleted file mode 100644 index 6ae978f05..000000000 --- a/common/install_conda.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -ex - -# Anaconda -# Latest anaconda is using openssl-3 which is incompatible with all currently published versions of git -# Which are using openssl-1.1.1, see https://anaconda.org/anaconda/git/files?version=2.40.1 for example -MINICONDA_URL=https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh -wget -q $MINICONDA_URL -# NB: Manually invoke bash per https://github.com/conda/conda/issues/10431 -bash $(basename "$MINICONDA_URL") -b -p /opt/conda -rm $(basename "$MINICONDA_URL") -export PATH=/opt/conda/bin:$PATH -# See https://github.com/pytorch/builder/issues/1473 -# Pin conda to 23.5.2 as it's the last one compatible with openssl-1.1.1 -conda install -y conda=23.5.2 conda-build anaconda-client git ninja -# The cmake version here needs to match with the minimum version of cmake -# supported by PyTorch (3.18). There is only 3.18.2 on anaconda -/opt/conda/bin/pip3 install cmake==3.18.2 -conda remove -y --force patchelf diff --git a/common/install_cpython.sh b/common/install_cpython.sh deleted file mode 100755 index 4e2360653..000000000 --- a/common/install_cpython.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash -set -uex -o pipefail - -PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python -PYTHON_DOWNLOAD_GITHUB_BRANCH=https://github.com/python/cpython/archive/refs/heads -GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py - -# Python versions to be installed in /opt/$VERSION_NO -CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"3.8.1 3.9.0 3.10.1 3.11.0 3.12.0 3.13.0"} - -function check_var { - if [ -z "$1" ]; then - echo "required variable not defined" - exit 1 - fi -} - -function do_cpython_build { - local py_ver=$1 - local py_folder=$2 - check_var $py_ver - check_var $py_folder - tar -xzf Python-$py_ver.tgz - pushd $py_folder - - local prefix="/opt/_internal/cpython-${py_ver}" - mkdir -p ${prefix}/lib - if [[ -n $(which patchelf) ]]; then - local shared_flags="--enable-shared" - else - local shared_flags="--disable-shared" - fi - if [[ -z "${WITH_OPENSSL+x}" ]]; then - local openssl_flags="" - else - local openssl_flags="--with-openssl=${WITH_OPENSSL} --with-openssl-rpath=auto" - fi - - # -Wformat added for https://bugs.python.org/issue17547 on Python 2.6 - CFLAGS="-Wformat" ./configure --prefix=${prefix} ${openssl_flags} ${shared_flags} > /dev/null - - make -j40 > /dev/null - make install > /dev/null - - if [[ "${shared_flags}" == "--enable-shared" ]]; then - patchelf --set-rpath '$ORIGIN/../lib' ${prefix}/bin/python3 - fi - - popd - rm -rf $py_folder - # Some python's install as bin/python3. Make them available as - # bin/python. - if [ -e ${prefix}/bin/python3 ]; then - ln -s python3 ${prefix}/bin/python - fi - ${prefix}/bin/python get-pip.py - if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then - ln -s pip3 ${prefix}/bin/pip - fi - ${prefix}/bin/pip install wheel==0.34.2 - local abi_tag=$(${prefix}/bin/python -c "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag; print('{0}{1}-{2}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag()))") - ln -s ${prefix} /opt/python/${abi_tag} -} - -function build_cpython { - local py_ver=$1 - check_var $py_ver - check_var $PYTHON_DOWNLOAD_URL - local py_ver_folder=$py_ver - if [ "$py_ver" = "3.13.0" ]; then - PY_VER_SHORT="3.13" - check_var $PYTHON_DOWNLOAD_GITHUB_BRANCH - wget $PYTHON_DOWNLOAD_GITHUB_BRANCH/$PY_VER_SHORT.tar.gz -O Python-$py_ver.tgz - do_cpython_build $py_ver cpython-$PY_VER_SHORT - else - wget -q $PYTHON_DOWNLOAD_URL/$py_ver_folder/Python-$py_ver.tgz - do_cpython_build $py_ver Python-$py_ver - fi - - rm -f Python-$py_ver.tgz -} - -function build_cpythons { - check_var $GET_PIP_URL - curl -sLO $GET_PIP_URL - for py_ver in $@; do - build_cpython $py_ver - done - rm -f get-pip.py -} - -mkdir -p /opt/python -mkdir -p /opt/_internal -build_cpythons $CPYTHON_VERSIONS diff --git a/common/install_cuda.sh b/common/install_cuda.sh deleted file mode 100644 index 2088447a7..000000000 --- a/common/install_cuda.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/bin/bash - -set -ex - -NCCL_VERSION=v2.21.5-1 -CUDNN_VERSION=9.1.0.70 - -function install_cusparselt_040 { - # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html - mkdir tmp_cusparselt && pushd tmp_cusparselt - wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.4.0.7-archive.tar.xz - tar xf libcusparse_lt-linux-x86_64-0.4.0.7-archive.tar.xz - cp -a libcusparse_lt-linux-x86_64-0.4.0.7-archive/include/* /usr/local/cuda/include/ - cp -a libcusparse_lt-linux-x86_64-0.4.0.7-archive/lib/* /usr/local/cuda/lib64/ - popd - rm -rf tmp_cusparselt -} - -function install_cusparselt_052 { - # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html - mkdir tmp_cusparselt && pushd tmp_cusparselt - wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.5.2.1-archive.tar.xz - tar xf libcusparse_lt-linux-x86_64-0.5.2.1-archive.tar.xz - cp -a libcusparse_lt-linux-x86_64-0.5.2.1-archive/include/* /usr/local/cuda/include/ - cp -a libcusparse_lt-linux-x86_64-0.5.2.1-archive/lib/* /usr/local/cuda/lib64/ - popd - rm -rf tmp_cusparselt -} - -function install_118 { - echo "Installing CUDA 11.8 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and cuSparseLt-0.4.0" - rm -rf /usr/local/cuda-11.8 /usr/local/cuda - # install CUDA 11.8.0 in the same container - wget -q https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run - chmod +x cuda_11.8.0_520.61.05_linux.run - ./cuda_11.8.0_520.61.05_linux.run --toolkit --silent - rm -f cuda_11.8.0_520.61.05_linux.run - rm -f /usr/local/cuda && ln -s /usr/local/cuda-11.8 /usr/local/cuda - - # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement - mkdir tmp_cudnn && cd tmp_cudnn - wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz - tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz - cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive/include/* /usr/local/cuda/include/ - cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf tmp_cudnn - - # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses - # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build - git clone -b $NCCL_VERSION --depth 1 https://github.com/NVIDIA/nccl.git - cd nccl && make -j src.build - cp -a build/include/* /usr/local/cuda/include/ - cp -a build/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf nccl - - install_cusparselt_040 - - ldconfig -} - -function install_121 { - echo "Installing CUDA 12.1 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and cuSparseLt-0.5.2" - rm -rf /usr/local/cuda-12.1 /usr/local/cuda - # install CUDA 12.1.0 in the same container - wget -q https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_530.30.02_linux.run - chmod +x cuda_12.1.1_530.30.02_linux.run - ./cuda_12.1.1_530.30.02_linux.run --toolkit --silent - rm -f cuda_12.1.1_530.30.02_linux.run - rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.1 /usr/local/cuda - - # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement - mkdir tmp_cudnn && cd tmp_cudnn - wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz - tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz - cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/ - cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf tmp_cudnn - - # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses - # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build - git clone -b $NCCL_VERSION --depth 1 https://github.com/NVIDIA/nccl.git - cd nccl && make -j src.build - cp -a build/include/* /usr/local/cuda/include/ - cp -a build/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf nccl - - install_cusparselt_052 - - ldconfig -} - -function install_124 { - echo "Installing CUDA 12.4 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and cuSparseLt-0.5.2" - rm -rf /usr/local/cuda-12.4 /usr/local/cuda - # install CUDA 12.4.0 in the same container - wget -q https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_550.54.14_linux.run - chmod +x cuda_12.4.0_550.54.14_linux.run - ./cuda_12.4.0_550.54.14_linux.run --toolkit --silent - rm -f cuda_12.4.0_550.54.14_linux.run - rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.4 /usr/local/cuda - - # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement - mkdir tmp_cudnn && cd tmp_cudnn - wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz - tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz - cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/ - cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf tmp_cudnn - - # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses - # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build - git clone -b $NCCL_VERSION --depth 1 https://github.com/NVIDIA/nccl.git - cd nccl && make -j src.build - cp -a build/include/* /usr/local/cuda/include/ - cp -a build/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf nccl - - install_cusparselt_052 - - ldconfig -} - -function prune_118 { - echo "Pruning CUDA 11.8 and cuDNN" - ##################################################################################### - # CUDA 11.8 prune static libs - ##################################################################################### - export NVPRUNE="/usr/local/cuda-11.8/bin/nvprune" - export CUDA_LIB_DIR="/usr/local/cuda-11.8/lib64" - - export GENCODE="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - export GENCODE_CUDNN="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - - if [[ -n "$OVERRIDE_GENCODE" ]]; then - export GENCODE=$OVERRIDE_GENCODE - fi - - # all CUDA libs except CuDNN and CuBLAS (cudnn and cublas need arch 3.7 included) - ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis" \ - | xargs -I {} bash -c \ - "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}" - - # prune CuDNN and CuBLAS - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a - - ##################################################################################### - # CUDA 11.8 prune visual tools - ##################################################################################### - export CUDA_BASE="/usr/local/cuda-11.8/" - rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2022.3.0 $CUDA_BASE/nsight-systems-2022.4.2/ -} - -function prune_121 { - echo "Pruning CUDA 12.1" - ##################################################################################### - # CUDA 12.1 prune static libs - ##################################################################################### - export NVPRUNE="/usr/local/cuda-12.1/bin/nvprune" - export CUDA_LIB_DIR="/usr/local/cuda-12.1/lib64" - - export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - - if [[ -n "$OVERRIDE_GENCODE" ]]; then - export GENCODE=$OVERRIDE_GENCODE - fi - - # all CUDA libs except CuDNN and CuBLAS - ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis" \ - | xargs -I {} bash -c \ - "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}" - - # prune CuDNN and CuBLAS - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a - - ##################################################################################### - # CUDA 12.1 prune visual tools - ##################################################################################### - export CUDA_BASE="/usr/local/cuda-12.1/" - rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2023.1.0 $CUDA_BASE/nsight-systems-2023.1.2/ -} - -function prune_124 { - echo "Pruning CUDA 12.4" - ##################################################################################### - # CUDA 12.4 prune static libs - ##################################################################################### - export NVPRUNE="/usr/local/cuda-12.4/bin/nvprune" - export CUDA_LIB_DIR="/usr/local/cuda-12.4/lib64" - - export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - - if [[ -n "$OVERRIDE_GENCODE" ]]; then - export GENCODE=$OVERRIDE_GENCODE - fi - if [[ -n "$OVERRIDE_GENCODE_CUDNN" ]]; then - export GENCODE_CUDNN=$OVERRIDE_GENCODE_CUDNN - fi - - # all CUDA libs except CuDNN and CuBLAS - ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis" \ - | xargs -I {} bash -c \ - "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}" - - # prune CuDNN and CuBLAS - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a - - ##################################################################################### - # CUDA 12.1 prune visual tools - ##################################################################################### - export CUDA_BASE="/usr/local/cuda-12.4/" - rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2024.1.0 $CUDA_BASE/nsight-systems-2023.4.4/ -} - -# idiomatic parameter and option handling in sh -while test $# -gt 0 -do - case "$1" in - 11.8) install_118; prune_118 - ;; - 12.1) install_121; prune_121 - ;; - 12.4) install_124; prune_124 - ;; - *) echo "bad argument $1"; exit 1 - ;; - esac - shift -done diff --git a/common/install_cuda_aarch64.sh b/common/install_cuda_aarch64.sh deleted file mode 100644 index b2dc0214a..000000000 --- a/common/install_cuda_aarch64.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -set -ex - -NCCL_VERSION=v2.21.5-1 - -function install_cusparselt_052 { - # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html - mkdir tmp_cusparselt && pushd tmp_cusparselt - wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-sbsa/libcusparse_lt-linux-sbsa-0.5.2.1-archive.tar.xz - tar xf libcusparse_lt-linux-sbsa-0.5.2.1-archive.tar.xz - cp -a libcusparse_lt-linux-sbsa-0.5.2.1-archive/include/* /usr/local/cuda/include/ - cp -a libcusparse_lt-linux-sbsa-0.5.2.1-archive/lib/* /usr/local/cuda/lib64/ - popd - rm -rf tmp_cusparselt -} - -function install_124 { - echo "Installing CUDA 12.4 and cuDNN 9.1 and NCCL ${NCCL_VERSION} and cuSparseLt-0.5.2" - rm -rf /usr/local/cuda-12.4 /usr/local/cuda - # install CUDA 12.4.0 in the same container - wget -q https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_550.54.14_linux_sbsa.run - chmod +x cuda_12.4.0_550.54.14_linux_sbsa.run - ./cuda_12.4.0_550.54.14_linux_sbsa.run --toolkit --silent - rm -f cuda_12.4.0_550.54.14_linux_sbsa.run - rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.4 /usr/local/cuda - - # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement - mkdir tmp_cudnn && cd tmp_cudnn - wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-sbsa/cudnn-linux-sbsa-9.1.0.70_cuda12-archive.tar.xz -O cudnn-linux-sbsa-9.1.0.70_cuda12-archive.tar.xz - tar xf cudnn-linux-sbsa-9.1.0.70_cuda12-archive.tar.xz - cp -a cudnn-linux-sbsa-9.1.0.70_cuda12-archive/include/* /usr/local/cuda/include/ - cp -a cudnn-linux-sbsa-9.1.0.70_cuda12-archive/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf tmp_cudnn - - # NCCL license: https://docs.nvidia.com/deeplearning/nccl/#licenses - # Follow build: https://github.com/NVIDIA/nccl/tree/master?tab=readme-ov-file#build - git clone -b ${NCCL_VERSION} --depth 1 https://github.com/NVIDIA/nccl.git - cd nccl && make -j src.build - cp -a build/include/* /usr/local/cuda/include/ - cp -a build/lib/* /usr/local/cuda/lib64/ - cd .. - rm -rf nccl - - install_cusparselt_052 - - ldconfig -} - -function prune_124 { - echo "Pruning CUDA 12.4" - ##################################################################################### - # CUDA 12.4 prune static libs - ##################################################################################### - export NVPRUNE="/usr/local/cuda-12.4/bin/nvprune" - export CUDA_LIB_DIR="/usr/local/cuda-12.4/lib64" - - export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90" - - if [[ -n "$OVERRIDE_GENCODE" ]]; then - export GENCODE=$OVERRIDE_GENCODE - fi - - # all CUDA libs except CuDNN and CuBLAS - ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis" \ - | xargs -I {} bash -c \ - "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}" - - # prune CuDNN and CuBLAS - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a - $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a - - ##################################################################################### - # CUDA 12.1 prune visual tools - ##################################################################################### - export CUDA_BASE="/usr/local/cuda-12.4/" - rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2024.1.0 $CUDA_BASE/nsight-systems-2023.4.4/ -} - -# idiomatic parameter and option handling in sh -while test $# -gt 0 -do - case "$1" in - 12.4) install_124; prune_124 - ;; - *) echo "bad argument $1"; exit 1 - ;; - esac - shift -done diff --git a/common/install_jni.sh b/common/install_jni.sh deleted file mode 100644 index 10b156a70..000000000 --- a/common/install_jni.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -ex - -mkdir -p /usr/local/include -cp jni.h /usr/local/include diff --git a/common/install_libpng.sh b/common/install_libpng.sh deleted file mode 100644 index 475f574de..000000000 --- a/common/install_libpng.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -ex - -LIBPNG_VERSION=1.6.37 - -mkdir -p libpng -pushd libpng - -wget http://download.sourceforge.net/libpng/libpng-$LIBPNG_VERSION.tar.gz -tar -xvzf libpng-$LIBPNG_VERSION.tar.gz - -pushd libpng-$LIBPNG_VERSION - -./configure -make -make install - -popd - -popd -rm -rf libpng diff --git a/common/install_magma.sh b/common/install_magma.sh deleted file mode 100644 index efaff052e..000000000 --- a/common/install_magma.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -MAGMA_VERSION="2.5.2" - -function do_install() { - cuda_version=$1 - cuda_version_nodot=${1/./} - - MAGMA_VERSION="2.6.1" - magma_archive="magma-cuda${cuda_version_nodot}-${MAGMA_VERSION}-1.tar.bz2" - - cuda_dir="/usr/local/cuda-${cuda_version}" - ( - set -x - tmp_dir=$(mktemp -d) - pushd ${tmp_dir} - curl -OLs https://anaconda.org/pytorch/magma-cuda${cuda_version_nodot}/${MAGMA_VERSION}/download/linux-64/${magma_archive} - tar -xvf "${magma_archive}" - mkdir -p "${cuda_dir}/magma" - mv include "${cuda_dir}/magma/include" - mv lib "${cuda_dir}/magma/lib" - popd - ) -} - -do_install $1 diff --git a/common/install_miopen.sh b/common/install_miopen.sh deleted file mode 100644 index 3aef952ef..000000000 --- a/common/install_miopen.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash - -set -ex - -ROCM_VERSION=$1 - -if [[ -z $ROCM_VERSION ]]; then - echo "missing ROCM_VERSION" - exit 1; -fi - -# To make version comparison easier, create an integer representation. -save_IFS="$IFS" -IFS=. ROCM_VERSION_ARRAY=(${ROCM_VERSION}) -IFS="$save_IFS" -if [[ ${#ROCM_VERSION_ARRAY[@]} == 2 ]]; then - ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} - ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} - ROCM_VERSION_PATCH=0 -elif [[ ${#ROCM_VERSION_ARRAY[@]} == 3 ]]; then - ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} - ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} - ROCM_VERSION_PATCH=${ROCM_VERSION_ARRAY[2]} -else - echo "Unhandled ROCM_VERSION ${ROCM_VERSION}" - exit 1 -fi -ROCM_INT=$(($ROCM_VERSION_MAJOR * 10000 + $ROCM_VERSION_MINOR * 100 + $ROCM_VERSION_PATCH)) - -# Install custom MIOpen + COMgr for ROCm >= 4.0.1 -if [[ $ROCM_INT -lt 40001 ]]; then - echo "ROCm version < 4.0.1; will not install custom MIOpen" - exit 0 -fi - -# Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -# Build custom MIOpen to use comgr for offline compilation. - -## Need a sanitized ROCM_VERSION without patchlevel; patchlevel version 0 must be added to paths. -ROCM_DOTS=$(echo ${ROCM_VERSION} | tr -d -c '.' | wc -c) -if [[ ${ROCM_DOTS} == 1 ]]; then - ROCM_VERSION_NOPATCH="${ROCM_VERSION}" - ROCM_INSTALL_PATH="/opt/rocm-${ROCM_VERSION}.0" -else - ROCM_VERSION_NOPATCH="${ROCM_VERSION%.*}" - ROCM_INSTALL_PATH="/opt/rocm-${ROCM_VERSION}" -fi - -# MIOPEN_USE_HIP_KERNELS is a Workaround for COMgr issues -MIOPEN_CMAKE_COMMON_FLAGS=" --DMIOPEN_USE_COMGR=ON --DMIOPEN_BUILD_DRIVER=OFF -" -# Pull MIOpen repo and set DMIOPEN_EMBED_DB based on ROCm version -if [[ $ROCM_INT -ge 60100 ]] && [[ $ROCM_INT -lt 60200 ]]; then - echo "ROCm 6.1 MIOpen does not need any patches, do not build from source" - exit 0 -elif [[ $ROCM_INT -ge 60000 ]] && [[ $ROCM_INT -lt 60100 ]]; then - echo "ROCm 6.0 MIOpen does not need any patches, do not build from source" - exit 0 -elif [[ $ROCM_INT -ge 50700 ]] && [[ $ROCM_INT -lt 60000 ]]; then - echo "ROCm 5.7 MIOpen does not need any patches, do not build from source" - exit 0 -elif [[ $ROCM_INT -ge 50600 ]] && [[ $ROCM_INT -lt 50700 ]]; then - MIOPEN_BRANCH="release/rocm-rel-5.6-staging" -elif [[ $ROCM_INT -ge 50500 ]] && [[ $ROCM_INT -lt 50600 ]]; then - MIOPEN_BRANCH="release/rocm-rel-5.5-gfx11" -elif [[ $ROCM_INT -ge 50400 ]] && [[ $ROCM_INT -lt 50500 ]]; then - MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36 -DMIOPEN_USE_MLIR=Off" - MIOPEN_BRANCH="release/rocm-rel-5.4-staging" -elif [[ $ROCM_INT -ge 50300 ]] && [[ $ROCM_INT -lt 50400 ]]; then - MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36 -DMIOPEN_USE_MLIR=Off" - MIOPEN_BRANCH="release/rocm-rel-5.3-staging" -elif [[ $ROCM_INT -ge 50200 ]] && [[ $ROCM_INT -lt 50300 ]]; then - MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36 -DMIOPEN_USE_MLIR=Off" - MIOPEN_BRANCH="release/rocm-rel-5.2-staging" -elif [[ $ROCM_INT -ge 50100 ]] && [[ $ROCM_INT -lt 50200 ]]; then - MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36" - MIOPEN_BRANCH="release/rocm-rel-5.1-staging" -elif [[ $ROCM_INT -ge 50000 ]] && [[ $ROCM_INT -lt 50100 ]]; then - MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx900_56;gfx906_60;gfx90878;gfx90a6e;gfx1030_36" - MIOPEN_BRANCH="release/rocm-rel-5.0-staging" -else - echo "Unhandled ROCM_VERSION ${ROCM_VERSION}" - exit 1 -fi - -yum remove -y miopen-hip - -git clone https://github.com/ROCm/MIOpen -b ${MIOPEN_BRANCH} -pushd MIOpen -# remove .git to save disk space since CI runner was running out -rm -rf .git -# Don't build MLIR to save docker build time -# since we are disabling MLIR backend for MIOpen anyway -if [[ $ROCM_INT -ge 50400 ]] && [[ $ROCM_INT -lt 50500 ]]; then - sed -i '/rocMLIR/d' requirements.txt -elif [[ $ROCM_INT -ge 50200 ]] && [[ $ROCM_INT -lt 50400 ]]; then - sed -i '/llvm-project-mlir/d' requirements.txt -fi -## MIOpen minimum requirements -cmake -P install_deps.cmake --minimum - -# clean up since CI runner was running out of disk space -rm -rf /tmp/* -yum clean all -rm -rf /var/cache/yum -rm -rf /var/lib/yum/yumdb -rm -rf /var/lib/yum/history - -## Build MIOpen -mkdir -p build -cd build -PKG_CONFIG_PATH=/usr/local/lib/pkgconfig CXX=${ROCM_INSTALL_PATH}/llvm/bin/clang++ cmake .. \ - ${MIOPEN_CMAKE_COMMON_FLAGS} \ - ${MIOPEN_CMAKE_DB_FLAGS} \ - -DCMAKE_PREFIX_PATH="${ROCM_INSTALL_PATH}/hip;${ROCM_INSTALL_PATH}" -make MIOpen -j $(nproc) - -# Build MIOpen package -make -j $(nproc) package - -# clean up since CI runner was running out of disk space -rm -rf /usr/local/cget - -yum install -y miopen-*.rpm - -popd -rm -rf MIOpen diff --git a/common/install_mkl.sh b/common/install_mkl.sh deleted file mode 100644 index 42a1779b0..000000000 --- a/common/install_mkl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -ex - -# MKL -MKL_VERSION=2024.2.0 - -mkdir -p /opt/intel/ -pushd /tmp - -python3 -mpip install wheel -python3 -mpip download -d . mkl-static==${MKL_VERSION} -python3 -m wheel unpack mkl_static-${MKL_VERSION}-py2.py3-none-manylinux1_x86_64.whl -python3 -m wheel unpack mkl_include-${MKL_VERSION}-py2.py3-none-manylinux1_x86_64.whl -mv mkl_static-${MKL_VERSION}/mkl_static-${MKL_VERSION}.data/data/lib /opt/intel/ -mv mkl_include-${MKL_VERSION}/mkl_include-${MKL_VERSION}.data/data/include /opt/intel/ diff --git a/common/install_mnist.sh b/common/install_mnist.sh deleted file mode 100644 index b0d6937e1..000000000 --- a/common/install_mnist.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -ex - -mkdir -p /usr/local/mnist/ - -cd /usr/local/mnist - -for img in train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz; do - wget -q https://ossci-datasets.s3.amazonaws.com/mnist/$img - gzip -d $img -done diff --git a/common/install_openblas.sh b/common/install_openblas.sh deleted file mode 100644 index e2deec811..000000000 --- a/common/install_openblas.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ex - -cd / -git clone https://github.com/OpenMathLib/OpenBLAS.git -b v0.3.25 --depth 1 --shallow-submodules - - -OPENBLAS_BUILD_FLAGS=" -NUM_THREADS=128 -USE_OPENMP=1 -NO_SHARED=0 -DYNAMIC_ARCH=1 -TARGET=ARMV8 -CFLAGS=-O3 -" - -OPENBLAS_CHECKOUT_DIR="OpenBLAS" - -make -j8 ${OPENBLAS_BUILD_FLAGS} -C ${OPENBLAS_CHECKOUT_DIR} -make -j8 ${OPENBLAS_BUILD_FLAGS} install -C ${OPENBLAS_CHECKOUT_DIR} \ No newline at end of file diff --git a/common/install_openssl.sh b/common/install_openssl.sh deleted file mode 100644 index 8ffb04721..000000000 --- a/common/install_openssl.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -ex - -OPENSSL=openssl-1.1.1l - -wget -q -O ${OPENSSL}.tar.gz "https://ossci-linux.s3.amazonaws.com/${OPENSSL}.tar.gz" -tar xf "${OPENSSL}.tar.gz" -cd "${OPENSSL}" -./config --prefix=/opt/openssl -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)' -# NOTE: opensl errors out when built with the -j option -make install_sw -cd .. -rm -rf "${OPENSSL}" diff --git a/common/install_patchelf.sh b/common/install_patchelf.sh deleted file mode 100644 index 37b69415e..000000000 --- a/common/install_patchelf.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -ex - -# Pin the version to latest release 0.17.2, building newer commit starts -# to fail on the current image -git clone -b 0.17.2 --single-branch https://github.com/NixOS/patchelf -cd patchelf -sed -i 's/serial/parallel/g' configure.ac -./bootstrap.sh -./configure -make -make install -cd .. -rm -rf patchelf diff --git a/common/install_rocm.sh b/common/install_rocm.sh deleted file mode 100644 index 75e7e6e27..000000000 --- a/common/install_rocm.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash - -# TODO upstream differences from this file is into the one in pytorch -# - (1) UBUNTU_VERSION read from /etc/os-release -# - (2) miopen kernel packages do not make sense for wheel and libtorch builds - -set -ex - -ver() { - printf "%3d%03d%03d%03d" $(echo "$1" | tr '.' ' '); -} - -# Map ROCm version to AMDGPU version -declare -A AMDGPU_VERSIONS=( ["5.0"]="21.50" ["5.1.1"]="22.10.1" ["5.2"]="22.20" ) - -install_ubuntu() { - apt-get update - if [[ $UBUNTU_VERSION == 18.04 ]]; then - # gpg-agent is not available by default on 18.04 - apt-get install -y --no-install-recommends gpg-agent - fi - if [[ $UBUNTU_VERSION == 20.04 ]]; then - # gpg-agent is not available by default on 20.04 - apt-get install -y --no-install-recommends gpg-agent - fi - apt-get install -y kmod - apt-get install -y wget - - # Need the libc++1 and libc++abi1 libraries to allow torch._C to load at runtime - apt-get install -y libc++1 - apt-get install -y libc++abi1 - - if [[ $(ver $ROCM_VERSION) -ge $(ver 4.5) ]]; then - # Add amdgpu repository - UBUNTU_VERSION_NAME=`cat /etc/os-release | grep UBUNTU_CODENAME | awk -F= '{print $2}'` - local amdgpu_baseurl - if [[ $(ver $ROCM_VERSION) -ge $(ver 5.3) ]]; then - amdgpu_baseurl="https://repo.radeon.com/amdgpu/${ROCM_VERSION}/ubuntu" - else - amdgpu_baseurl="https://repo.radeon.com/amdgpu/${AMDGPU_VERSIONS[$ROCM_VERSION]}/ubuntu" - fi - echo "deb [arch=amd64] ${amdgpu_baseurl} ${UBUNTU_VERSION_NAME} main" > /etc/apt/sources.list.d/amdgpu.list - fi - - ROCM_REPO="ubuntu" - if [[ $(ver $ROCM_VERSION) -lt $(ver 4.2) ]]; then - ROCM_REPO="xenial" - fi - - if [[ $(ver $ROCM_VERSION) -ge $(ver 5.3) ]]; then - ROCM_REPO="${UBUNTU_VERSION_NAME}" - fi - - # Add rocm repository - wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - - local rocm_baseurl="http://repo.radeon.com/rocm/apt/${ROCM_VERSION}" - echo "deb [arch=amd64] ${rocm_baseurl} ${ROCM_REPO} main" > /etc/apt/sources.list.d/rocm.list - apt-get update --allow-insecure-repositories - - DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ - rocm-dev \ - rocm-utils \ - rocm-libs \ - rccl \ - rocprofiler-dev \ - roctracer-dev - - # TODO (2) - ## precompiled miopen kernels added in ROCm 3.5; search for all unversioned packages - ## if search fails it will abort this script; use true to avoid case where search fails - #MIOPENKERNELS=$(apt-cache search --names-only miopenkernels | awk '{print $1}' | grep -F -v . || true) - #if [[ "x${MIOPENKERNELS}" = x ]]; then - # echo "miopenkernels package not available" - #else - # DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated ${MIOPENKERNELS} - #fi - - # Cleanup - apt-get autoclean && apt-get clean - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -} - -install_centos() { - - yum update -y - yum install -y kmod - yum install -y wget - yum install -y openblas-devel - - yum install -y epel-release - yum install -y dkms kernel-headers-`uname -r` kernel-devel-`uname -r` - - if [[ $(ver $ROCM_VERSION) -ge $(ver 4.5) ]]; then - # Add amdgpu repository - local amdgpu_baseurl - if [[ $(ver $ROCM_VERSION) -ge $(ver 5.3) ]]; then - amdgpu_baseurl="https://repo.radeon.com/amdgpu/${ROCM_VERSION}/rhel/7.9/main/x86_64" - else - amdgpu_baseurl="https://repo.radeon.com/amdgpu/${AMDGPU_VERSIONS[$ROCM_VERSION]}/rhel/7.9/main/x86_64" - fi - echo "[AMDGPU]" > /etc/yum.repos.d/amdgpu.repo - echo "name=AMDGPU" >> /etc/yum.repos.d/amdgpu.repo - echo "baseurl=${amdgpu_baseurl}" >> /etc/yum.repos.d/amdgpu.repo - echo "enabled=1" >> /etc/yum.repos.d/amdgpu.repo - echo "gpgcheck=1" >> /etc/yum.repos.d/amdgpu.repo - echo "gpgkey=http://repo.radeon.com/rocm/rocm.gpg.key" >> /etc/yum.repos.d/amdgpu.repo - fi - - local rocm_baseurl="http://repo.radeon.com/rocm/yum/${ROCM_VERSION}" - if [[ $(ver $ROCM_VERSION) -ge $(ver 5.1) ]]; then - local rocm_baseurl="http://repo.radeon.com/rocm/yum/${ROCM_VERSION}/main" - fi - echo "[ROCm]" > /etc/yum.repos.d/rocm.repo - echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo - echo "baseurl=${rocm_baseurl}" >> /etc/yum.repos.d/rocm.repo - echo "enabled=1" >> /etc/yum.repos.d/rocm.repo - echo "gpgcheck=1" >> /etc/yum.repos.d/rocm.repo - echo "gpgkey=http://repo.radeon.com/rocm/rocm.gpg.key" >> /etc/yum.repos.d/rocm.repo - - yum update -y - - yum install -y \ - rocm-dev \ - rocm-utils \ - rocm-libs \ - rccl \ - rocprofiler-dev \ - roctracer-dev - - # Cleanup - yum clean all - rm -rf /var/cache/yum - rm -rf /var/lib/yum/yumdb - rm -rf /var/lib/yum/history -} - -# Install Python packages depending on the base OS -ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') -case "$ID" in - ubuntu) - # TODO (1) - UBUNTU_VERSION=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"') - install_ubuntu - ;; - centos) - install_centos - ;; - *) - echo "Unable to determine OS..." - exit 1 - ;; -esac diff --git a/common/install_rocm_drm.sh b/common/install_rocm_drm.sh deleted file mode 100644 index ed7a327c4..000000000 --- a/common/install_rocm_drm.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash - -########################### -### prereqs -########################### -# Install Python packages depending on the base OS -ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') -case "$ID" in - ubuntu) - apt-get update -y - apt-get install -y libpciaccess-dev pkg-config - apt-get clean - ;; - centos) - yum install -y libpciaccess-devel pkgconfig - ;; - *) - echo "Unable to determine OS..." - exit 1 - ;; -esac -python3 -m pip install meson ninja - -########################### -### clone repo -########################### -GIT_SSL_NO_VERIFY=true git clone https://gitlab.freedesktop.org/mesa/drm.git -pushd drm - -########################### -### patch -########################### -patch -p1 <<'EOF' -diff --git a/amdgpu/amdgpu_asic_id.c b/amdgpu/amdgpu_asic_id.c -index a5007ffc..13fa07fc 100644 ---- a/amdgpu/amdgpu_asic_id.c -+++ b/amdgpu/amdgpu_asic_id.c -@@ -22,6 +22,13 @@ - * - */ - -+#define _XOPEN_SOURCE 700 -+#define _LARGEFILE64_SOURCE -+#define _FILE_OFFSET_BITS 64 -+#include -+#include -+#include -+ - #include - #include - #include -@@ -34,6 +41,19 @@ - #include "amdgpu_drm.h" - #include "amdgpu_internal.h" - -+static char *amdgpuids_path = NULL; -+static const char* amdgpuids_path_msg = NULL; -+ -+static int check_for_location_of_amdgpuids(const char *filepath, const struct stat *info, const int typeflag, struct FTW *pathinfo) -+{ -+ if (typeflag == FTW_F && strstr(filepath, "amdgpu.ids")) { -+ amdgpuids_path = strdup(filepath); -+ return 1; -+ } -+ -+ return 0; -+} -+ - static int parse_one_line(struct amdgpu_device *dev, const char *line) - { - char *buf, *saveptr; -@@ -113,10 +133,46 @@ void amdgpu_parse_asic_ids(struct amdgpu_device *dev) - int line_num = 1; - int r = 0; - -+ // attempt to find typical location for amdgpu.ids file - fp = fopen(AMDGPU_ASIC_ID_TABLE, "r"); -+ -+ // if it doesn't exist, search -+ if (!fp) { -+ -+ char self_path[ PATH_MAX ]; -+ ssize_t count; -+ ssize_t i; -+ -+ count = readlink( "/proc/self/exe", self_path, PATH_MAX ); -+ if (count > 0) { -+ self_path[count] = '\0'; -+ -+ // remove '/bin/python' from self_path -+ for (i=count; i>0; --i) { -+ if (self_path[i] == '/') break; -+ self_path[i] = '\0'; -+ } -+ self_path[i] = '\0'; -+ for (; i>0; --i) { -+ if (self_path[i] == '/') break; -+ self_path[i] = '\0'; -+ } -+ self_path[i] = '\0'; -+ -+ if (1 == nftw(self_path, check_for_location_of_amdgpuids, 5, FTW_PHYS)) { -+ fp = fopen(amdgpuids_path, "r"); -+ amdgpuids_path_msg = amdgpuids_path; -+ } -+ } -+ -+ } -+ else { -+ amdgpuids_path_msg = AMDGPU_ASIC_ID_TABLE; -+ } -+ -+ // both hard-coded location and search have failed - if (!fp) { -- fprintf(stderr, "%s: %s\n", AMDGPU_ASIC_ID_TABLE, -- strerror(errno)); -+ fprintf(stderr, "amdgpu.ids: No such file or directory\n"); - return; - } - -@@ -132,7 +188,7 @@ void amdgpu_parse_asic_ids(struct amdgpu_device *dev) - continue; - } - -- drmMsg("%s version: %s\n", AMDGPU_ASIC_ID_TABLE, line); -+ drmMsg("%s version: %s\n", amdgpuids_path_msg, line); - break; - } - -@@ -150,7 +206,7 @@ void amdgpu_parse_asic_ids(struct amdgpu_device *dev) - - if (r == -EINVAL) { - fprintf(stderr, "Invalid format: %s: line %d: %s\n", -- AMDGPU_ASIC_ID_TABLE, line_num, line); -+ amdgpuids_path_msg, line_num, line); - } else if (r && r != -EAGAIN) { - fprintf(stderr, "%s: Cannot parse ASIC IDs: %s\n", - __func__, strerror(-r)); -EOF - -########################### -### build -########################### -meson builddir --prefix=/opt/amdgpu -pushd builddir -ninja install - -popd -popd diff --git a/common/install_rocm_magma.sh b/common/install_rocm_magma.sh deleted file mode 100644 index c8e43f675..000000000 --- a/common/install_rocm_magma.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# TODO upstream differences from this file is into the (eventual) one in pytorch -# - (1) check for static lib mkl -# - (2) MKLROOT as env var - -set -ex - -# TODO (2) -MKLROOT=${MKLROOT:-/opt/intel} - -# "install" hipMAGMA into /opt/rocm/magma by copying after build -git clone https://bitbucket.org/icl/magma.git -pushd magma -if [[ $PYTORCH_BRANCH == "release/1.10.1" ]]; then - git checkout magma_ctrl_launch_bounds -else - git checkout a1625ff4d9bc362906bd01f805dbbe12612953f6 -fi -cp make.inc-examples/make.inc.hip-gcc-mkl make.inc -echo 'LIBDIR += -L$(MKLROOT)/lib' >> make.inc -# TODO (1) -if [[ -f "${MKLROOT}/lib/libmkl_core.a" ]]; then - echo 'LIB = -Wl,--start-group -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -Wl,--end-group -lpthread -lstdc++ -lm -lgomp -lhipblas -lhipsparse' >> make.inc -fi -echo 'LIB += -Wl,--enable-new-dtags -Wl,--rpath,/opt/rocm/lib -Wl,--rpath,$(MKLROOT)/lib -Wl,--rpath,/opt/rocm/magma/lib -ldl' >> make.inc -echo 'DEVCCFLAGS += --gpu-max-threads-per-block=256' >> make.inc -export PATH="${PATH}:/opt/rocm/bin" -if [[ -n "$PYTORCH_ROCM_ARCH" ]]; then - amdgpu_targets=`echo $PYTORCH_ROCM_ARCH | sed 's/;/ /g'` -else - amdgpu_targets=`rocm_agent_enumerator | grep -v gfx000 | sort -u | xargs` -fi -for arch in $amdgpu_targets; do - echo "DEVCCFLAGS += --offload-arch=$arch" >> make.inc -done -# hipcc with openmp flag may cause isnan() on __device__ not to be found; depending on context, compiler may attempt to match with host definition -sed -i 's/^FOPENMP/#FOPENMP/g' make.inc -make -f make.gen.hipMAGMA -j $(nproc) -LANG=C.UTF-8 make lib/libmagma.so -j $(nproc) MKLROOT="${MKLROOT}" -make testing/testing_dgemm -j $(nproc) MKLROOT="${MKLROOT}" -popd -mkdir -p /opt/rocm/magma -mv magma/include /opt/rocm/magma -mv magma/lib /opt/rocm/magma -rm -rf magma - diff --git a/common/install_xpu.sh b/common/install_xpu.sh deleted file mode 100644 index c34bdd8bd..000000000 --- a/common/install_xpu.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash -set -xe - -# Intel® software for general purpose GPU capabilities. -# Refer to https://www.intel.com/content/www/us/en/developer/articles/tool/pytorch-prerequisites-for-intel-gpus.html - -# Users should update to the latest version as it becomes available - -function install_ubuntu() { - . /etc/os-release - if [[ ! " jammy " =~ " ${VERSION_CODENAME} " ]]; then - echo "Ubuntu version ${VERSION_CODENAME} not supported" - exit - fi - - apt-get update -y - apt-get install -y gpg-agent wget - - # To add the online network package repository for the GPU Driver LTS releases - wget -qO - https://repositories.intel.com/gpu/intel-graphics.key \ - | gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] \ - https://repositories.intel.com/gpu/ubuntu ${VERSION_CODENAME}/lts/2350 unified" \ - | tee /etc/apt/sources.list.d/intel-gpu-${VERSION_CODENAME}.list - # To add the online network network package repository for the Intel Support Packages - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ - | gpg --dearmor > /usr/share/keyrings/intel-for-pytorch-gpu-dev-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/intel-for-pytorch-gpu-dev-keyring.gpg] \ - https://apt.repos.intel.com/intel-for-pytorch-gpu-dev all main" \ - | tee /etc/apt/sources.list.d/intel-for-pytorch-gpu-dev.list - - # Update the packages list and repository index - apt-get update - - # The xpu-smi packages - apt-get install -y flex bison xpu-smi - # Compute and Media Runtimes - apt-get install -y \ - intel-opencl-icd intel-level-zero-gpu level-zero \ - intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \ - libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \ - libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \ - mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo - # Development Packages - apt-get install -y libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev - # Install Intel Support Packages - if [ -n "$XPU_VERSION" ]; then - apt-get install -y intel-for-pytorch-gpu-dev-${XPU_VERSION} - else - apt-get install -y intel-for-pytorch-gpu-dev - fi - - # Cleanup - apt-get autoclean && apt-get clean - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -} - -function install_rhel() { - . /etc/os-release - if [[ "${ID}" == "rhel" ]]; then - if [[ ! " 8.6 8.8 8.9 9.0 9.2 9.3 " =~ " ${VERSION_ID} " ]]; then - echo "RHEL version ${VERSION_ID} not supported" - exit - fi - elif [[ "${ID}" == "almalinux" ]]; then - # Workaround for almalinux8 which used by quay.io/pypa/manylinux_2_28_x86_64 - VERSION_ID="8.6" - fi - - dnf install -y 'dnf-command(config-manager)' - # To add the online network package repository for the GPU Driver LTS releases - dnf config-manager --add-repo \ - https://repositories.intel.com/gpu/rhel/${VERSION_ID}/lts/2350/unified/intel-gpu-${VERSION_ID}.repo - - # To add the online network network package repository for the Intel Support Packages - tee > /etc/yum.repos.d/intel-for-pytorch-gpu-dev.repo << EOF -[intel-for-pytorch-gpu-dev] -name=Intel for Pytorch GPU dev repository -baseurl=https://yum.repos.intel.com/intel-for-pytorch-gpu-dev -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -EOF - - # The xpu-smi packages - dnf install -y xpu-smi - # Compute and Media Runtimes - dnf install -y \ - intel-opencl intel-media intel-mediasdk libmfxgen1 libvpl2\ - level-zero intel-level-zero-gpu mesa-dri-drivers mesa-vulkan-drivers \ - mesa-vdpau-drivers libdrm mesa-libEGL mesa-libgbm mesa-libGL \ - mesa-libxatracker libvpl-tools intel-metrics-discovery \ - intel-metrics-library intel-igc-core intel-igc-cm \ - libva libva-utils intel-gmmlib libmetee intel-gsc intel-ocloc - # Development packages - dnf install -y --refresh \ - intel-igc-opencl-devel level-zero-devel intel-gsc-devel libmetee-devel \ - level-zero-devel - # Install Intel Support Packages - yum install -y intel-for-pytorch-gpu-dev intel-pti-dev - - # Cleanup - dnf clean all - rm -rf /var/cache/yum - rm -rf /var/lib/yum/yumdb - rm -rf /var/lib/yum/history -} - -function install_sles() { - . /etc/os-release - VERSION_SP=${VERSION_ID//./sp} - if [[ ! " 15sp4 15sp5 " =~ " ${VERSION_SP} " ]]; then - echo "SLES version ${VERSION_ID} not supported" - exit - fi - - # To add the online network package repository for the GPU Driver LTS releases - zypper addrepo -f -r \ - https://repositories.intel.com/gpu/sles/${VERSION_SP}/lts/2350/unified/intel-gpu-${VERSION_SP}.repo - rpm --import https://repositories.intel.com/gpu/intel-graphics.key - # To add the online network network package repository for the Intel Support Packages - zypper addrepo https://yum.repos.intel.com/intel-for-pytorch-gpu-dev intel-for-pytorch-gpu-dev - rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - - # The xpu-smi packages - zypper install -y lsb-release flex bison xpu-smi - # Compute and Media Runtimes - zypper install -y intel-level-zero-gpu level-zero intel-gsc intel-opencl intel-ocloc \ - intel-media-driver libigfxcmrt7 libvpl2 libvpl-tools libmfxgen1 libmfx1 - # Development packages - zypper install -y libigdfcl-devel intel-igc-cm libigfxcmrt-devel level-zero-devel - - # Install Intel Support Packages - zypper install -y intel-for-pytorch-gpu-dev intel-pti-dev - -} - - -# The installation depends on the base OS -ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') -case "$ID" in - ubuntu) - install_ubuntu - ;; - rhel|almalinux) - install_rhel - ;; - sles) - install_sles - ;; - *) - echo "Unable to determine OS..." - exit 1 - ;; -esac diff --git a/conda/Dockerfile b/conda/Dockerfile deleted file mode 100644 index f09fcb2ca..000000000 --- a/conda/Dockerfile +++ /dev/null @@ -1,95 +0,0 @@ -ARG CUDA_VERSION=10.2 -ARG BASE_TARGET=cuda${CUDA_VERSION} -FROM centos:7 as base - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -ARG DEVTOOLSET_VERSION=9 -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -RUN yum update -y -RUN yum install -y wget curl perl util-linux xz bzip2 git patch which unzip -# Just add everything as a safe.directory for git since these will be used in multiple places with git -RUN git config --global --add safe.directory '*' -RUN yum install -y yum-utils centos-release-scl -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -RUN yum install -y devtoolset-${DEVTOOLSET_VERSION}-gcc devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran devtoolset-${DEVTOOLSET_VERSION}-binutils -# EPEL for cmake -RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ - rpm -ivh epel-release-latest-7.noarch.rpm && \ - rm -f epel-release-latest-7.noarch.rpm -# cmake -RUN yum install -y cmake3 && \ - ln -s /usr/bin/cmake3 /usr/bin/cmake -ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - -RUN yum install -y autoconf aclocal automake make sudo -RUN rm -rf /usr/local/cuda-* - -FROM base as patchelf -# Install patchelf -ADD ./common/install_patchelf.sh install_patchelf.sh -RUN bash ./install_patchelf.sh && rm install_patchelf.sh && cp $(which patchelf) /patchelf - -FROM base as openssl -# Install openssl -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh - -FROM base as conda -# Install Anaconda -ADD ./common/install_conda.sh install_conda.sh -RUN bash ./install_conda.sh && rm install_conda.sh - -# Install CUDA -FROM base as cuda -ARG CUDA_VERSION=10.2 -RUN rm -rf /usr/local/cuda-* -ADD ./common/install_cuda.sh install_cuda.sh -ENV CUDA_HOME=/usr/local/cuda-${CUDA_VERSION} -# Preserve CUDA_VERSION for the builds -ENV CUDA_VERSION=${CUDA_VERSION} -# Make things in our path by default -ENV PATH=/usr/local/cuda-${CUDA_VERSION}/bin:$PATH - -FROM cuda as cuda11.8 -RUN bash ./install_cuda.sh 11.8 -ENV DESIRED_CUDA=11.8 - -FROM cuda as cuda12.1 -RUN bash ./install_cuda.sh 12.1 -ENV DESIRED_CUDA=12.1 - -FROM cuda as cuda12.4 -RUN bash ./install_cuda.sh 12.4 -ENV DESIRED_CUDA=12.4 - -# Install MNIST test data -FROM base as mnist -ADD ./common/install_mnist.sh install_mnist.sh -RUN bash ./install_mnist.sh - -FROM base as all_cuda -COPY --from=cuda11.8 /usr/local/cuda-11.8 /usr/local/cuda-11.8 -COPY --from=cuda12.1 /usr/local/cuda-12.1 /usr/local/cuda-12.1 -COPY --from=cuda12.4 /usr/local/cuda-12.4 /usr/local/cuda-12.4 - -FROM ${BASE_TARGET} as final -COPY --from=openssl /opt/openssl /opt/openssl -COPY --from=patchelf /patchelf /usr/local/bin/patchelf -COPY --from=conda /opt/conda /opt/conda -ADD ./java/jni.h /usr/local/include/jni.h -ENV PATH /opt/conda/bin:$PATH -COPY --from=mnist /usr/local/mnist /usr/local/mnist -RUN rm -rf /usr/local/cuda -RUN chmod o+rw /usr/local -RUN touch /.condarc && \ - chmod o+rw /.condarc && \ - chmod -R o+rw /opt/conda diff --git a/conda/README.md b/conda/README.md index 4c1b719f6..f8e707c8b 100644 --- a/conda/README.md +++ b/conda/README.md @@ -6,14 +6,6 @@ - `git add pytorch-$BUILD_VERSION` - Run `./build_pytorch.sh` on an OSX machine and a Linux machine -## build base docker images - -```sh -conda/build_all_docker.sh -# Will push all of the images -docker push pytorch/conda-builder -``` - ## building pytorch / torchvision etc. ```sh diff --git a/conda/build_all_docker.sh b/conda/build_all_docker.sh deleted file mode 100755 index 133773fe2..000000000 --- a/conda/build_all_docker.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -TOPDIR=$(git rev-parse --show-toplevel) - -for CUDA_VERSION in 12.4 12.1 11.8 cpu; do - CUDA_VERSION="${CUDA_VERSION}" conda/build_docker.sh -done diff --git a/conda/build_docker.sh b/conda/build_docker.sh deleted file mode 100755 index c15e55a06..000000000 --- a/conda/build_docker.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -export DOCKER_BUILDKIT=1 -TOPDIR=$(git rev-parse --show-toplevel) - -CUDA_VERSION=${CUDA_VERSION:-11.7} - -case ${CUDA_VERSION} in - cpu) - BASE_TARGET=base - DOCKER_TAG=cpu - ;; - all) - BASE_TARGET=all_cuda - DOCKER_TAG=latest - ;; - *) - BASE_TARGET=cuda${CUDA_VERSION} - DOCKER_TAG=cuda${CUDA_VERSION} - ;; -esac - -( - set -x - docker build \ - --target final \ - --build-arg "BASE_TARGET=${BASE_TARGET}" \ - --build-arg "CUDA_VERSION=${CUDA_VERSION}" \ - --build-arg "DEVTOOLSET_VERSION=9" \ - -t "pytorch/conda-builder:${DOCKER_TAG}" \ - -f "${TOPDIR}/conda/Dockerfile" \ - ${TOPDIR} -) - -DOCKER_IMAGE="pytorch/conda-builder:${DOCKER_TAG}" -GITHUB_REF=${GITHUB_REF:-$(git symbolic-ref -q HEAD || git describe --tags --exact-match)} -GIT_BRANCH_NAME=${GITHUB_REF##*/} -GIT_COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse HEAD)} -DOCKER_IMAGE_BRANCH_TAG=${DOCKER_IMAGE}-${GIT_BRANCH_NAME} -DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE}-${GIT_COMMIT_SHA} - -if [[ "${DOCKER_TAG}" =~ ^cuda* ]]; then - # Meant for legacy scripts since they only do the version without the "." - # TODO: Eventually remove this - ( - set -x - docker tag ${DOCKER_IMAGE} "pytorch/conda-builder:cuda${CUDA_VERSION/./}" - ) - # Test that we're using the right CUDA compiler - ( - set -x - docker run --rm "${DOCKER_IMAGE}" nvcc --version | grep "cuda_${CUDA_VERSION}" - ) -fi - -if [[ -n ${GITHUB_REF} ]]; then - docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_BRANCH_TAG} - docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_SHA_TAG} -fi - -if [[ "${WITH_PUSH:-}" == true ]]; then - ( - set -x - docker push "${DOCKER_IMAGE}" - if [[ -n ${GITHUB_REF} ]]; then - docker push "${DOCKER_IMAGE_BRANCH_TAG}" - docker push "${DOCKER_IMAGE_SHA_TAG}" - fi - if [[ "${DOCKER_TAG}" =~ ^cuda* ]]; then - docker push "pytorch/conda-builder:cuda${CUDA_VERSION/./}" - fi - ) -fi diff --git a/libtorch/Dockerfile b/libtorch/Dockerfile deleted file mode 100644 index 6ce4e9987..000000000 --- a/libtorch/Dockerfile +++ /dev/null @@ -1,106 +0,0 @@ -ARG BASE_TARGET=base -ARG GPU_IMAGE=ubuntu:20.04 -FROM ${GPU_IMAGE} as base - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get clean && apt-get update -RUN apt-get install -y curl locales g++ git-all autoconf automake make cmake wget unzip -# Just add everything as a safe.directory for git since these will be used in multiple places with git -RUN git config --global --add safe.directory '*' - -RUN locale-gen en_US.UTF-8 - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -# Install openssl -FROM base as openssl -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh - -# Install python -FROM base as python -ADD common/install_cpython.sh install_cpython.sh -RUN apt-get update -y && \ - apt-get install build-essential gdb lcov libbz2-dev libffi-dev \ - libgdbm-dev liblzma-dev libncurses5-dev libreadline6-dev \ - libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g-dev -y && \ - bash ./install_cpython.sh && \ - rm install_cpython.sh && \ - apt-get clean - -FROM base as conda -ADD ./common/install_conda.sh install_conda.sh -RUN bash ./install_conda.sh && rm install_conda.sh - -FROM base as cpu -# Install Anaconda -COPY --from=conda /opt/conda /opt/conda -# Install python -COPY --from=python /opt/python /opt/python -COPY --from=python /opt/_internal /opt/_internal -ENV PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -# Install MKL -ADD ./common/install_mkl.sh install_mkl.sh -RUN bash ./install_mkl.sh && rm install_mkl.sh - -FROM cpu as cuda -ADD ./common/install_cuda.sh install_cuda.sh -ADD ./common/install_magma.sh install_magma.sh -ENV CUDA_HOME /usr/local/cuda - -FROM cuda as cuda11.8 -RUN bash ./install_cuda.sh 11.8 -RUN bash ./install_magma.sh 11.8 -RUN ln -sf /usr/local/cuda-11.8 /usr/local/cuda - -FROM cuda as cuda12.1 -RUN bash ./install_cuda.sh 12.1 -RUN bash ./install_magma.sh 12.1 -RUN ln -sf /usr/local/cuda-12.1 /usr/local/cuda - -FROM cuda as cuda12.4 -RUN bash ./install_cuda.sh 12.4 -RUN bash ./install_magma.sh 12.4 -RUN ln -sf /usr/local/cuda-12.4 /usr/local/cuda - -FROM cpu as rocm -ARG PYTORCH_ROCM_ARCH -ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH} -ENV MKLROOT /opt/intel -# Adding ROCM_PATH env var so that LoadHip.cmake (even with logic updated for ROCm6.0) -# find HIP works for ROCm5.7. Not needed for ROCm6.0 and above. -# Remove below when ROCm5.7 is not in support matrix anymore. -ENV ROCM_PATH /opt/rocm -# No need to install ROCm as base docker image should have full ROCm install -#ADD ./common/install_rocm.sh install_rocm.sh -ADD ./common/install_rocm_drm.sh install_rocm_drm.sh -ADD ./common/install_rocm_magma.sh install_rocm_magma.sh -# gfortran and python needed for building magma from source for ROCm -RUN apt-get update -y && \ - apt-get install gfortran -y && \ - apt-get install python -y && \ - apt-get clean - -RUN bash ./install_rocm_drm.sh && rm install_rocm_drm.sh -RUN bash ./install_rocm_magma.sh && rm install_rocm_magma.sh - -# Install AOTriton -COPY ./common/aotriton_version.txt aotriton_version.txt -COPY ./common/install_aotriton.sh install_aotriton.sh -RUN bash ./install_aotriton.sh /opt/rocm && rm install_aotriton.sh aotriton_version.txt -ENV AOTRITON_INSTALLED_PREFIX /opt/rocm/aotriton - -FROM ${BASE_TARGET} as final -COPY --from=openssl /opt/openssl /opt/openssl -# Install patchelf -ADD ./common/install_patchelf.sh install_patchelf.sh -RUN bash ./install_patchelf.sh && rm install_patchelf.sh -# Install Anaconda -COPY --from=conda /opt/conda /opt/conda -# Install python -COPY --from=python /opt/python /opt/python -COPY --from=python /opt/_internal /opt/_internal -ENV PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH diff --git a/libtorch/build_docker.sh b/libtorch/build_docker.sh deleted file mode 100755 index b7ebdd36e..000000000 --- a/libtorch/build_docker.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -TOPDIR=$(git rev-parse --show-toplevel) - -GPU_ARCH_TYPE=${GPU_ARCH_TYPE:-cpu} -GPU_ARCH_VERSION=${GPU_ARCH_VERSION:-} - -WITH_PUSH=${WITH_PUSH:-} - -DOCKER=${DOCKER:-docker} - -case ${GPU_ARCH_TYPE} in - cpu) - BASE_TARGET=cpu - DOCKER_TAG=cpu - GPU_IMAGE=ubuntu:20.04 - DOCKER_GPU_BUILD_ARG="" - ;; - cuda) - BASE_TARGET=cuda${GPU_ARCH_VERSION} - DOCKER_TAG=cuda${GPU_ARCH_VERSION} - GPU_IMAGE=ubuntu:20.04 - DOCKER_GPU_BUILD_ARG="" - ;; - rocm) - BASE_TARGET=rocm - DOCKER_TAG=rocm${GPU_ARCH_VERSION} - GPU_IMAGE=rocm/dev-ubuntu-20.04:${GPU_ARCH_VERSION}-complete - PYTORCH_ROCM_ARCH="gfx900;gfx906;gfx908;gfx90a;gfx1030;gfx1100" - ROCM_REGEX="([0-9]+)\.([0-9]+)[\.]?([0-9]*)" - if [[ $GPU_ARCH_VERSION =~ $ROCM_REGEX ]]; then - ROCM_VERSION_INT=$((${BASH_REMATCH[1]}*10000 + ${BASH_REMATCH[2]}*100 + ${BASH_REMATCH[3]:-0})) - else - echo "ERROR: rocm regex failed" - exit 1 - fi - if [[ $ROCM_VERSION_INT -ge 60000 ]]; then - PYTORCH_ROCM_ARCH+=";gfx942" - fi - DOCKER_GPU_BUILD_ARG="--build-arg PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}" - ;; - *) - echo "ERROR: Unrecognized GPU_ARCH_TYPE: ${GPU_ARCH_TYPE}" - exit 1 - ;; -esac - -DOCKER_IMAGE=pytorch/libtorch-cxx11-builder:${DOCKER_TAG} - -( - set -x - DOCKER_BUILDKIT=1 ${DOCKER} build \ - -t "${DOCKER_IMAGE}" \ - ${DOCKER_GPU_BUILD_ARG} \ - --build-arg "GPU_IMAGE=${GPU_IMAGE}" \ - --build-arg "BASE_TARGET=${BASE_TARGET}" \ - --target final \ - -f "${TOPDIR}/libtorch/Dockerfile" \ - "${TOPDIR}" -) - -GITHUB_REF=${GITHUB_REF:-$(git symbolic-ref -q HEAD || git describe --tags --exact-match)} -GIT_BRANCH_NAME=${GITHUB_REF##*/} -GIT_COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse HEAD)} -DOCKER_IMAGE_BRANCH_TAG=${DOCKER_IMAGE}-${GIT_BRANCH_NAME} -DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE}-${GIT_COMMIT_SHA} - -if [[ -n ${GITHUB_REF} ]]; then - ${DOCKER} tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_BRANCH_TAG} - ${DOCKER} tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_SHA_TAG} -fi - -if [[ "${WITH_PUSH}" == true ]]; then - ( - set -x - ${DOCKER} push "${DOCKER_IMAGE}" - if [[ -n ${GITHUB_REF} ]]; then - ${DOCKER} push "${DOCKER_IMAGE_BRANCH_TAG}" - ${DOCKER} push "${DOCKER_IMAGE_SHA_TAG}" - fi - ) - # For legacy .circleci/config.yml generation scripts - if [[ "${GPU_ARCH_TYPE}" != "cpu" ]]; then - ( - set -x - ${DOCKER} tag ${DOCKER_IMAGE} pytorch/libtorch-cxx11-builder:${DOCKER_TAG/./} - ${DOCKER} push pytorch/libtorch-cxx11-builder:${DOCKER_TAG/./} - ) - fi -fi diff --git a/manywheel/Dockerfile b/manywheel/Dockerfile deleted file mode 100644 index a164cf8f9..000000000 --- a/manywheel/Dockerfile +++ /dev/null @@ -1,201 +0,0 @@ -# syntax = docker/dockerfile:experimental -ARG ROCM_VERSION=3.7 -ARG BASE_CUDA_VERSION=11.8 - -ARG GPU_IMAGE=centos:7 -FROM centos:7 as base - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -ARG DEVTOOLSET_VERSION=9 -# Note: This is required patch since CentOS have reached EOL -# otherwise any yum install setp will fail -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -RUN yum install -y wget curl perl util-linux xz bzip2 git patch which perl zlib-devel -# Just add everything as a safe.directory for git since these will be used in multiple places with git -RUN git config --global --add safe.directory '*' -RUN yum install -y yum-utils centos-release-scl -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -# Note: After running yum-config-manager --enable rhel-server-rhscl-7-rpms -# patch is required once again. Somehow this steps adds mirror.centos.org -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -RUN yum install -y devtoolset-${DEVTOOLSET_VERSION}-gcc devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran devtoolset-${DEVTOOLSET_VERSION}-binutils -ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - -RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ - rpm -ivh epel-release-latest-7.noarch.rpm && \ - rm -f epel-release-latest-7.noarch.rpm - -# cmake-3.18.4 from pip -RUN yum install -y python3-pip && \ - python3 -mpip install cmake==3.18.4 && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake - -RUN yum install -y autoconf aclocal automake make - -FROM base as openssl -# Install openssl (this must precede `build python` step) -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh - -# EPEL for cmake -FROM base as patchelf -# Install patchelf -ADD ./common/install_patchelf.sh install_patchelf.sh -RUN bash ./install_patchelf.sh && rm install_patchelf.sh -RUN cp $(which patchelf) /patchelf - -FROM patchelf as python -# build python -COPY manywheel/build_scripts /build_scripts -ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh -RUN bash build_scripts/build.sh && rm -r build_scripts - -FROM base as cuda -ARG BASE_CUDA_VERSION=10.2 -# Install CUDA -ADD ./common/install_cuda.sh install_cuda.sh -RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh - -FROM base as intel -# MKL -ADD ./common/install_mkl.sh install_mkl.sh -RUN bash ./install_mkl.sh && rm install_mkl.sh - -FROM base as magma -ARG BASE_CUDA_VERSION=10.2 -# Install magma -ADD ./common/install_magma.sh install_magma.sh -RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh - -FROM base as jni -# Install java jni header -ADD ./common/install_jni.sh install_jni.sh -ADD ./java/jni.h jni.h -RUN bash ./install_jni.sh && rm install_jni.sh - -FROM base as libpng -# Install libpng -ADD ./common/install_libpng.sh install_libpng.sh -RUN bash ./install_libpng.sh && rm install_libpng.sh - -FROM ${GPU_IMAGE} as common -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -RUN yum install -y \ - aclocal \ - autoconf \ - automake \ - bison \ - bzip2 \ - curl \ - diffutils \ - file \ - git \ - make \ - patch \ - perl \ - unzip \ - util-linux \ - wget \ - which \ - xz \ - yasm -RUN yum install -y \ - https://repo.ius.io/ius-release-el7.rpm \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -RUN yum swap -y git git236-core -# git236+ would refuse to run git commands in repos owned by other users -# Which causes version check to fail, as pytorch repo is bind-mounted into the image -# Override this behaviour by treating every folder as safe -# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 -RUN git config --global --add safe.directory "*" - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem -# Install LLVM version -COPY --from=openssl /opt/openssl /opt/openssl -COPY --from=python /opt/python /opt/python -COPY --from=python /opt/_internal /opt/_internal -COPY --from=python /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel -COPY --from=intel /opt/intel /opt/intel -COPY --from=patchelf /usr/local/bin/patchelf /usr/local/bin/patchelf -COPY --from=jni /usr/local/include/jni.h /usr/local/include/jni.h -COPY --from=libpng /usr/local/bin/png* /usr/local/bin/ -COPY --from=libpng /usr/local/bin/libpng* /usr/local/bin/ -COPY --from=libpng /usr/local/include/png* /usr/local/include/ -COPY --from=libpng /usr/local/include/libpng* /usr/local/include/ -COPY --from=libpng /usr/local/lib/libpng* /usr/local/lib/ -COPY --from=libpng /usr/local/lib/pkgconfig /usr/local/lib/pkgconfig - -FROM common as cpu_final -ARG BASE_CUDA_VERSION=10.1 -ARG DEVTOOLSET_VERSION=9 -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo - -RUN yum install -y yum-utils centos-release-scl -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -RUN yum install -y devtoolset-${DEVTOOLSET_VERSION}-gcc devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran devtoolset-${DEVTOOLSET_VERSION}-binutils -ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - -# cmake is already installed inside the rocm base image, so remove if present -RUN rpm -e cmake || true -# cmake-3.18.4 from pip -RUN yum install -y python3-pip && \ - python3 -mpip install cmake==3.18.4 && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake - -# ninja -RUN yum install -y ninja-build - -FROM cpu_final as cuda_final -RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=cuda /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=magma /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} -RUN ln -sf /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda -ENV PATH=/usr/local/cuda/bin:$PATH - -FROM cpu_final as rocm_final -ARG ROCM_VERSION=3.7 -ARG PYTORCH_ROCM_ARCH -ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH} -# Adding ROCM_PATH env var so that LoadHip.cmake (even with logic updated for ROCm6.0) -# find HIP works for ROCm5.7. Not needed for ROCm6.0 and above. -# Remove below when ROCm5.7 is not in support matrix anymore. -ENV ROCM_PATH /opt/rocm -# No need to install ROCm as base docker image should have full ROCm install -#ADD ./common/install_rocm.sh install_rocm.sh -#RUN ROCM_VERSION=${ROCM_VERSION} bash ./install_rocm.sh && rm install_rocm.sh -ADD ./common/install_rocm_drm.sh install_rocm_drm.sh -RUN bash ./install_rocm_drm.sh && rm install_rocm_drm.sh -# cmake3 is needed for the MIOpen build -RUN ln -sf /usr/local/bin/cmake /usr/bin/cmake3 -ADD ./common/install_rocm_magma.sh install_rocm_magma.sh -RUN bash ./install_rocm_magma.sh && rm install_rocm_magma.sh -ADD ./common/install_miopen.sh install_miopen.sh -RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh - -# Install AOTriton -COPY ./common/aotriton_version.txt aotriton_version.txt -COPY ./common/install_aotriton.sh install_aotriton.sh -RUN bash ./install_aotriton.sh /opt/rocm && rm install_aotriton.sh aotriton_version.txt -ENV AOTRITON_INSTALLED_PREFIX /opt/rocm/aotriton diff --git a/manywheel/Dockerfile_2014 b/manywheel/Dockerfile_2014 deleted file mode 100644 index 752a863fa..000000000 --- a/manywheel/Dockerfile_2014 +++ /dev/null @@ -1,152 +0,0 @@ -# syntax = docker/dockerfile:experimental -ARG ROCM_VERSION=3.7 -ARG BASE_CUDA_VERSION=10.2 -ARG GPU_IMAGE=nvidia/cuda:${BASE_CUDA_VERSION}-devel-centos7 -FROM quay.io/pypa/manylinux2014_x86_64 as base - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -RUN yum install -y wget curl perl util-linux xz bzip2 git patch which perl zlib-devel -RUN yum install -y yum-utils centos-release-scl -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -RUN yum install -y devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran devtoolset-7-binutils -ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:$LD_LIBRARY_PATH - -# cmake -RUN yum install -y cmake3 && \ - ln -s /usr/bin/cmake3 /usr/bin/cmake -FROM base as openssl -# Install openssl (this must precede `build python` step) -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh - - - -# remove unncessary python versions -RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2 -RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4 -RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6 -RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6 - -FROM base as cuda -ARG BASE_CUDA_VERSION=10.2 -# Install CUDA -ADD ./common/install_cuda.sh install_cuda.sh -RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh - -FROM base as intel -# MKL -ADD ./common/install_mkl.sh install_mkl.sh -RUN bash ./install_mkl.sh && rm install_mkl.sh - -FROM base as magma -ARG BASE_CUDA_VERSION=10.2 -# Install magma -ADD ./common/install_magma.sh install_magma.sh -RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh - -FROM base as jni -# Install java jni header -ADD ./common/install_jni.sh install_jni.sh -ADD ./java/jni.h jni.h -RUN bash ./install_jni.sh && rm install_jni.sh - -FROM base as libpng -# Install libpng -ADD ./common/install_libpng.sh install_libpng.sh -RUN bash ./install_libpng.sh && rm install_libpng.sh - -FROM ${GPU_IMAGE} as common -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -RUN sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo -RUN sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -RUN yum install -y \ - aclocal \ - autoconf \ - automake \ - bison \ - bzip2 \ - curl \ - diffutils \ - file \ - git \ - make \ - patch \ - perl \ - unzip \ - util-linux \ - wget \ - which \ - xz \ - yasm -RUN yum install -y \ - https://repo.ius.io/ius-release-el7.rpm \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -RUN yum swap -y git git236-core -# git236+ would refuse to run git commands in repos owned by other users -# Which causes version check to fail, as pytorch repo is bind-mounted into the image -# Override this behaviour by treating every folder as safe -# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 -RUN git config --global --add safe.directory "*" - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem -# Install LLVM version -COPY --from=openssl /opt/openssl /opt/openssl -COPY --from=base /opt/python /opt/python -COPY --from=base /opt/_internal /opt/_internal -COPY --from=base /usr/local/bin/auditwheel /usr/local/bin/auditwheel -COPY --from=intel /opt/intel /opt/intel -COPY --from=base /usr/local/bin/patchelf /usr/local/bin/patchelf -COPY --from=libpng /usr/local/bin/png* /usr/local/bin/ -COPY --from=libpng /usr/local/bin/libpng* /usr/local/bin/ -COPY --from=libpng /usr/local/include/png* /usr/local/include/ -COPY --from=libpng /usr/local/include/libpng* /usr/local/include/ -COPY --from=libpng /usr/local/lib/libpng* /usr/local/lib/ -COPY --from=libpng /usr/local/lib/pkgconfig /usr/local/lib/pkgconfig -COPY --from=jni /usr/local/include/jni.h /usr/local/include/jni.h - -FROM common as cpu_final -ARG BASE_CUDA_VERSION=10.2 -RUN yum install -y yum-utils centos-release-scl -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -RUN yum install -y devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran devtoolset-7-binutils -ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:$LD_LIBRARY_PATH - -# cmake -RUN yum install -y cmake3 && \ - ln -s /usr/bin/cmake3 /usr/bin/cmake - -# ninja -RUN yum install -y http://repo.okay.com.mx/centos/7/x86_64/release/okay-release-1-1.noarch.rpm -RUN yum install -y ninja-build - -FROM cpu_final as cuda_final -RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=cuda /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=magma /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} - -FROM common as rocm_final -ARG ROCM_VERSION=3.7 -# Install ROCm -ADD ./common/install_rocm.sh install_rocm.sh -RUN bash ./install_rocm.sh ${ROCM_VERSION} && rm install_rocm.sh -# cmake is already installed inside the rocm base image, but both 2 and 3 exist -# cmake3 is needed for the later MIOpen custom build, so that step is last. -RUN yum install -y cmake3 && \ - rm -f /usr/bin/cmake && \ - ln -s /usr/bin/cmake3 /usr/bin/cmake -ADD ./common/install_miopen.sh install_miopen.sh -RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh diff --git a/manywheel/Dockerfile_2_28 b/manywheel/Dockerfile_2_28 deleted file mode 100644 index 4739417b6..000000000 --- a/manywheel/Dockerfile_2_28 +++ /dev/null @@ -1,153 +0,0 @@ -# syntax = docker/dockerfile:experimental -ARG ROCM_VERSION=3.7 -ARG BASE_CUDA_VERSION=11.8 -ARG GPU_IMAGE=amd64/almalinux:8 -FROM quay.io/pypa/manylinux_2_28_x86_64 as base - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -ARG DEVTOOLSET_VERSION=11 -RUN yum install -y wget curl perl util-linux xz bzip2 git patch which perl zlib-devel yum-utils gcc-toolset-${DEVTOOLSET_VERSION}-toolchain -ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - -# cmake-3.18.4 from pip -RUN yum install -y python3-pip && \ - python3 -mpip install cmake==3.18.4 && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake3 - -FROM base as openssl -# Install openssl (this must precede `build python` step) -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh - - -# remove unncessary python versions -RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2 -RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4 -RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6 -RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6 - -FROM base as cuda -ARG BASE_CUDA_VERSION=11.8 -# Install CUDA -ADD ./common/install_cuda.sh install_cuda.sh -RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh - -FROM base as intel -# MKL -ADD ./common/install_mkl.sh install_mkl.sh -RUN bash ./install_mkl.sh && rm install_mkl.sh - -FROM base as magma -ARG BASE_CUDA_VERSION=10.2 -# Install magma -ADD ./common/install_magma.sh install_magma.sh -RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh - -FROM base as jni -# Install java jni header -ADD ./common/install_jni.sh install_jni.sh -ADD ./java/jni.h jni.h -RUN bash ./install_jni.sh && rm install_jni.sh - -FROM base as libpng -# Install libpng -ADD ./common/install_libpng.sh install_libpng.sh -RUN bash ./install_libpng.sh && rm install_libpng.sh - -FROM ${GPU_IMAGE} as common -ARG DEVTOOLSET_VERSION=11 -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -RUN yum -y install epel-release -RUN yum -y update -RUN yum install -y \ - autoconf \ - automake \ - bison \ - bzip2 \ - curl \ - diffutils \ - file \ - git \ - make \ - patch \ - perl \ - unzip \ - util-linux \ - wget \ - which \ - xz \ - gcc-toolset-${DEVTOOLSET_VERSION}-toolchain \ - glibc-langpack-en - -RUN yum install -y \ - https://repo.ius.io/ius-release-el7.rpm \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -RUN yum swap -y git git236-core -# git236+ would refuse to run git commands in repos owned by other users -# Which causes version check to fail, as pytorch repo is bind-mounted into the image -# Override this behaviour by treating every folder as safe -# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 -RUN git config --global --add safe.directory "*" - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem -# Install LLVM version -COPY --from=openssl /opt/openssl /opt/openssl -COPY --from=base /opt/python /opt/python -COPY --from=base /opt/_internal /opt/_internal -COPY --from=base /usr/local/bin/auditwheel /usr/local/bin/auditwheel -COPY --from=intel /opt/intel /opt/intel -COPY --from=base /usr/local/bin/patchelf /usr/local/bin/patchelf -COPY --from=libpng /usr/local/bin/png* /usr/local/bin/ -COPY --from=libpng /usr/local/bin/libpng* /usr/local/bin/ -COPY --from=libpng /usr/local/include/png* /usr/local/include/ -COPY --from=libpng /usr/local/include/libpng* /usr/local/include/ -COPY --from=libpng /usr/local/lib/libpng* /usr/local/lib/ -COPY --from=libpng /usr/local/lib/pkgconfig /usr/local/lib/pkgconfig -COPY --from=jni /usr/local/include/jni.h /usr/local/include/jni.h - -FROM common as cpu_final -ARG BASE_CUDA_VERSION=11.8 -ARG DEVTOOLSET_VERSION=11 -# Ensure the expected devtoolset is used -ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - -# cmake-3.18.4 from pip -RUN yum install -y python3-pip && \ - python3 -mpip install cmake==3.18.4 && \ - ln -s /usr/local/bin/cmake /usr/bin/cmake3 - -FROM cpu_final as cuda_final -RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=cuda /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=magma /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} - -FROM common as rocm_final -ARG ROCM_VERSION=3.7 -# Install ROCm -ADD ./common/install_rocm.sh install_rocm.sh -RUN bash ./install_rocm.sh ${ROCM_VERSION} && rm install_rocm.sh -# cmake is already installed inside the rocm base image, but both 2 and 3 exist -# cmake3 is needed for the later MIOpen custom build, so that step is last. -RUN yum install -y cmake3 && \ - rm -f /usr/bin/cmake && \ - ln -s /usr/bin/cmake3 /usr/bin/cmake -ADD ./common/install_miopen.sh install_miopen.sh -RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh - -FROM cpu_final as xpu_final -# cmake-3.28.4 from pip -RUN python3 -m pip install --upgrade pip && \ - python3 -mpip install cmake==3.28.4 -ADD ./common/install_xpu.sh install_xpu.sh -RUN bash ./install_xpu.sh && rm install_xpu.sh -RUN pushd /opt/_internal && tar -xJf static-libs-for-embedding-only.tar.xz && popd diff --git a/manywheel/Dockerfile_2_28_aarch64 b/manywheel/Dockerfile_2_28_aarch64 deleted file mode 100644 index 222d261ef..000000000 --- a/manywheel/Dockerfile_2_28_aarch64 +++ /dev/null @@ -1,56 +0,0 @@ -FROM quay.io/pypa/manylinux_2_28_aarch64 as base - -# Graviton needs GCC 10 or above for the build. GCC12 is the default version in almalinux-8. -ARG GCCTOOLSET_VERSION=11 - -# Language variabes -ENV LC_ALL=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US.UTF-8 - -# Installed needed OS packages. This is to support all -# the binary builds (torch, vision, audio, text, data) -RUN yum -y install epel-release -RUN yum -y update -RUN yum install -y \ - autoconf \ - automake \ - bison \ - bzip2 \ - curl \ - diffutils \ - file \ - git \ - less \ - libffi-devel \ - libgomp \ - make \ - openssl-devel \ - patch \ - perl \ - unzip \ - util-linux \ - wget \ - which \ - xz \ - yasm \ - zstd \ - gcc-toolset-${GCCTOOLSET_VERSION}-toolchain - -# Ensure the expected devtoolset is used -ENV PATH=/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - -# git236+ would refuse to run git commands in repos owned by other users -# Which causes version check to fail, as pytorch repo is bind-mounted into the image -# Override this behaviour by treating every folder as safe -# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 -RUN git config --global --add safe.directory "*" - -FROM base as final - -# remove unncessary python versions -RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2 -RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4 -RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6 -RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6 diff --git a/manywheel/Dockerfile_aarch64 b/manywheel/Dockerfile_aarch64 deleted file mode 100644 index b0716d158..000000000 --- a/manywheel/Dockerfile_aarch64 +++ /dev/null @@ -1,93 +0,0 @@ -FROM quay.io/pypa/manylinux2014_aarch64 as base - - -# Graviton needs GCC 10 for the build -ARG DEVTOOLSET_VERSION=10 - -# Language variabes -ENV LC_ALL=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US.UTF-8 - -# Installed needed OS packages. This is to support all -# the binary builds (torch, vision, audio, text, data) -RUN yum -y install epel-release -RUN yum -y update -RUN yum install -y \ - autoconf \ - automake \ - bison \ - bzip2 \ - curl \ - diffutils \ - file \ - git \ - make \ - patch \ - perl \ - unzip \ - util-linux \ - wget \ - which \ - xz \ - yasm \ - less \ - zstd \ - libgomp \ - devtoolset-${DEVTOOLSET_VERSION}-gcc \ - devtoolset-${DEVTOOLSET_VERSION}-gcc-c++ \ - devtoolset-${DEVTOOLSET_VERSION}-gcc-gfortran \ - devtoolset-${DEVTOOLSET_VERSION}-binutils - -# Ensure the expected devtoolset is used -ENV PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/devtoolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - - -# git236+ would refuse to run git commands in repos owned by other users -# Which causes version check to fail, as pytorch repo is bind-mounted into the image -# Override this behaviour by treating every folder as safe -# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 -RUN git config --global --add safe.directory "*" - - -############################################################################### -# libglfortran.a hack -# -# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 is not compiled with -fPIC. -# This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get -# ubuntu's libgfortran.a which is compiled with -fPIC -# NOTE: Need a better way to get this library as Ubuntu's package can be removed by the vender, or changed -############################################################################### -RUN cd ~/ \ - && curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.5.0-1ubuntu1_arm64.deb \ - && ar x ~/libgfortran-10-dev.deb \ - && tar --use-compress-program=unzstd -xvf data.tar.zst -C ~/ \ - && cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/ - -# install cmake -RUN yum install -y cmake3 && \ - ln -s /usr/bin/cmake3 /usr/bin/cmake - -FROM base as openssl -# Install openssl (this must precede `build python` step) -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -FROM base as openblas -# Install openblas -ADD ./common/install_openblas.sh install_openblas.sh -RUN bash ./install_openblas.sh && rm install_openblas.sh - -FROM openssl as final -# remove unncessary python versions -RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2 -RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4 -RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6 -RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6 -COPY --from=openblas /opt/OpenBLAS/ /opt/OpenBLAS/ -ENV LD_LIBRARY_PATH=/opt/OpenBLAS/lib:$LD_LIBRARY_PATH \ No newline at end of file diff --git a/manywheel/Dockerfile_cuda_aarch64 b/manywheel/Dockerfile_cuda_aarch64 deleted file mode 100644 index d243b06b8..000000000 --- a/manywheel/Dockerfile_cuda_aarch64 +++ /dev/null @@ -1,90 +0,0 @@ -FROM quay.io/pypa/manylinux_2_28_aarch64 as base - -# Cuda ARM build needs gcc 11 -ARG DEVTOOLSET_VERSION=11 - -# Language variables -ENV LC_ALL=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US.UTF-8 - -# Installed needed OS packages. This is to support all -# the binary builds (torch, vision, audio, text, data) -RUN yum -y install epel-release -RUN yum -y update -RUN yum install -y \ - autoconf \ - automake \ - bison \ - bzip2 \ - curl \ - diffutils \ - file \ - git \ - make \ - patch \ - perl \ - unzip \ - util-linux \ - wget \ - which \ - xz \ - yasm \ - less \ - zstd \ - libgomp \ - gcc-toolset-${DEVTOOLSET_VERSION}-toolchain - -# Ensure the expected devtoolset is used -ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH - -# git236+ would refuse to run git commands in repos owned by other users -# Which causes version check to fail, as pytorch repo is bind-mounted into the image -# Override this behaviour by treating every folder as safe -# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 -RUN git config --global --add safe.directory "*" - - -FROM base as openssl -# Install openssl (this must precede `build python` step) -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -FROM openssl as final -# remove unncessary python versions -RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2 -RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4 -RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6 -RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6 - -FROM base as cuda -ARG BASE_CUDA_VERSION -# Install CUDA -ADD ./common/install_cuda_aarch64.sh install_cuda_aarch64.sh -RUN bash ./install_cuda_aarch64.sh ${BASE_CUDA_VERSION} && rm install_cuda_aarch64.sh - -FROM base as magma -ARG BASE_CUDA_VERSION -# Install magma -ADD ./common/install_magma.sh install_magma.sh -RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh - -FROM base as openblas -# Install openblas -ADD ./common/install_openblas.sh install_openblas.sh -RUN bash ./install_openblas.sh && rm install_openblas.sh - -FROM final as cuda_final -ARG BASE_CUDA_VERSION -RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=cuda /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=magma /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} -COPY --from=openblas /opt/OpenBLAS/ /opt/OpenBLAS/ -RUN ln -sf /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda -ENV PATH=/usr/local/cuda/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/OpenBLAS/lib:$LD_LIBRARY_PATH \ No newline at end of file diff --git a/manywheel/Dockerfile_cxx11-abi b/manywheel/Dockerfile_cxx11-abi deleted file mode 100644 index 4aa625ca7..000000000 --- a/manywheel/Dockerfile_cxx11-abi +++ /dev/null @@ -1,71 +0,0 @@ -FROM centos:8 as base - -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV PATH /opt/rh/gcc-toolset-11/root/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -# change to a valid repo -RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-*.repo -# enable to install ninja-build -RUN sed -i 's|enabled=0|enabled=1|g' /etc/yum.repos.d/CentOS-Linux-PowerTools.repo - -RUN yum -y update -RUN yum install -y wget curl perl util-linux xz bzip2 git patch which zlib-devel -RUN yum install -y autoconf automake make cmake gdb gcc-toolset-11-gcc-c++ - - -FROM base as openssl -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh - -# Install python -FROM base as python -RUN yum install -y openssl-devel zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel libpcap-devel xz-devel libffi-devel -ADD common/install_cpython.sh install_cpython.sh -RUN bash ./install_cpython.sh && rm install_cpython.sh - -FROM base as conda -ADD ./common/install_conda.sh install_conda.sh -RUN bash ./install_conda.sh && rm install_conda.sh -RUN /opt/conda/bin/conda install -y cmake - -FROM base as intel -# Install MKL -COPY --from=python /opt/python /opt/python -COPY --from=python /opt/_internal /opt/_internal -COPY --from=conda /opt/conda /opt/conda -ENV PATH=/opt/conda/bin:$PATH -ADD ./common/install_mkl.sh install_mkl.sh -RUN bash ./install_mkl.sh && rm install_mkl.sh - -FROM base as patchelf -ADD ./common/install_patchelf.sh install_patchelf.sh -RUN bash ./install_patchelf.sh && rm install_patchelf.sh -RUN cp $(which patchelf) /patchelf - -FROM base as jni -ADD ./common/install_jni.sh install_jni.sh -ADD ./java/jni.h jni.h -RUN bash ./install_jni.sh && rm install_jni.sh - -FROM base as libpng -ADD ./common/install_libpng.sh install_libpng.sh -RUN bash ./install_libpng.sh && rm install_libpng.sh - -FROM base as final -COPY --from=openssl /opt/openssl /opt/openssl -COPY --from=python /opt/python /opt/python -COPY --from=python /opt/_internal /opt/_internal -COPY --from=intel /opt/intel /opt/intel -COPY --from=conda /opt/conda /opt/conda -COPY --from=patchelf /usr/local/bin/patchelf /usr/local/bin/patchelf -COPY --from=jni /usr/local/include/jni.h /usr/local/include/jni.h -COPY --from=libpng /usr/local/bin/png* /usr/local/bin/ -COPY --from=libpng /usr/local/bin/libpng* /usr/local/bin/ -COPY --from=libpng /usr/local/include/png* /usr/local/include/ -COPY --from=libpng /usr/local/include/libpng* /usr/local/include/ -COPY --from=libpng /usr/local/lib/libpng* /usr/local/lib/ -COPY --from=libpng /usr/local/lib/pkgconfig /usr/local/lib/pkgconfig - -RUN yum install -y ninja-build diff --git a/manywheel/Dockerfile_s390x b/manywheel/Dockerfile_s390x deleted file mode 100644 index 5125e3830..000000000 --- a/manywheel/Dockerfile_s390x +++ /dev/null @@ -1,73 +0,0 @@ -FROM --platform=linux/s390x docker.io/ubuntu:24.04 as base - -# Language variables -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 -ENV LANGUAGE=C.UTF-8 - -# Installed needed OS packages. This is to support all -# the binary builds (torch, vision, audio, text, data) -RUN apt update ; apt upgrade -y -RUN apt install -y \ - build-essential \ - autoconf \ - automake \ - bzip2 \ - curl \ - diffutils \ - file \ - git \ - make \ - patch \ - perl \ - unzip \ - util-linux \ - wget \ - which \ - xz-utils \ - less \ - zstd \ - cmake \ - python3 \ - python3-dev \ - python3-setuptools \ - python3-yaml \ - python3-typing-extensions \ - libblas-dev \ - libopenblas-dev \ - liblapack-dev \ - libatlas-base-dev - -# git236+ would refuse to run git commands in repos owned by other users -# Which causes version check to fail, as pytorch repo is bind-mounted into the image -# Override this behaviour by treating every folder as safe -# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 -RUN git config --global --add safe.directory "*" - -FROM base as openssl -# Install openssl (this must precede `build python` step) -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -ADD ./common/install_openssl.sh install_openssl.sh -RUN bash ./install_openssl.sh && rm install_openssl.sh -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -# EPEL for cmake -FROM base as patchelf -# Install patchelf -ADD ./common/install_patchelf.sh install_patchelf.sh -RUN bash ./install_patchelf.sh && rm install_patchelf.sh -RUN cp $(which patchelf) /patchelf - -FROM patchelf as python -# build python -COPY manywheel/build_scripts /build_scripts -ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh -RUN bash build_scripts/build.sh && rm -r build_scripts - -FROM openssl as final -COPY --from=python /opt/python /opt/python -COPY --from=python /opt/_internal /opt/_internal -COPY --from=python /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel -COPY --from=patchelf /usr/local/bin/patchelf /usr/local/bin/patchelf diff --git a/manywheel/LICENSE b/manywheel/LICENSE deleted file mode 100644 index 7d8f7841a..000000000 --- a/manywheel/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 manylinux - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/manywheel/README.md b/manywheel/README.md deleted file mode 100644 index 69a8b7f49..000000000 --- a/manywheel/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# pytorch/builder/manywheel - -## Building docker images - -To build all docker images you can use the convenience script: - -```bash -# Build without pushing -manywheel/build_all_docker.sh -# Build with pushing -WITH_PUSH=true manywheel/build_all_docker.sh -``` - -To build a specific docker image use: -```bash -# GPU_ARCH_TYPE can be ["cuda", "rocm", "cpu"] -# GPU_ARCH_VERSION is GPU_ARCH_TYPE dependent, see manywheel/build_all_docker.sh for examples -GPU_ARCH_TYPE=cuda GPU_ARCH_VERSION=11.1 manywheel/build_docker.sh -# Build with pushing -WITH_PUSH=true GPU_ARCH_TYPE=cuda GPU_ARCH_VERSION=11.1 manywheel/build_docker.sh -``` - -**DISCLAIMER for WITH_PUSH**: -If you'd like to use WITH_PUSH, you must set it to exactly `true`, not `1` nor `ON` nor even `TRUE`, as our scripts -check for exact string equality to enable push functionality. The reason for this rigidity is due to the how we -calculate WITH_PUSH in our GHA workflow YAMLs. Currently, we usually enable push based on the workflow trigger, which -when we query with an expression like `${{ github.event_name == 'push' }}` returns either `true` or `false`. Thus, we -adapt our scripts to fit with this model. \ No newline at end of file diff --git a/manywheel/build.sh b/manywheel/build.sh deleted file mode 100755 index e79083ee0..000000000 --- a/manywheel/build.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -case "${GPU_ARCH_TYPE:-BLANK}" in - BLANK) - # Legacy behavior for CircleCI - bash "${SCRIPTPATH}/build_cuda.sh" - ;; - cuda) - bash "${SCRIPTPATH}/build_cuda.sh" - ;; - rocm) - bash "${SCRIPTPATH}/build_rocm.sh" - ;; - cpu | cpu-cxx11-abi | cpu-s390x | xpu) - bash "${SCRIPTPATH}/build_cpu.sh" - ;; - *) - echo "Un-recognized GPU_ARCH_TYPE '${GPU_ARCH_TYPE}', exiting..." - exit 1 - ;; -esac diff --git a/manywheel/build_all_docker.sh b/manywheel/build_all_docker.sh deleted file mode 100644 index bb7f95ed8..000000000 --- a/manywheel/build_all_docker.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -TOPDIR=$(git rev-parse --show-toplevel) - -GPU_ARCH_TYPE=cpu "${TOPDIR}/manywheel/build_docker.sh" -MANYLINUX_VERSION=2014 GPU_ARCH_TYPE=cpu "${TOPDIR}/manywheel/build_docker.sh" - -GPU_ARCH_TYPE=cpu-aarch64 "${TOPDIR}/manywheel/build_docker.sh" - -GPU_ARCH_TYPE=cpu-s390x "${TOPDIR}/manywheel/build_docker.sh" - -GPU_ARCH_TYPE=cpu-cxx11-abi "${TOPDIR}/manywheel/build_docker.sh" - -for cuda_version in 12.1 11.8; do - GPU_ARCH_TYPE=cuda GPU_ARCH_VERSION="${cuda_version}" "${TOPDIR}/manywheel/build_docker.sh" - MANYLINUX_VERSION=2014 GPU_ARCH_TYPE=cuda GPU_ARCH_VERSION="${cuda_version}" "${TOPDIR}/manywheel/build_docker.sh" -done - -for rocm_version in 6.0 6.1; do - GPU_ARCH_TYPE=rocm GPU_ARCH_VERSION="${rocm_version}" "${TOPDIR}/manywheel/build_docker.sh" - MANYLINUX_VERSION=2014 GPU_ARCH_TYPE=rocm GPU_ARCH_VERSION="${rocm_version}" "${TOPDIR}/manywheel/build_docker.sh" -done - -GPU_ARCH_TYPE=xpu "${TOPDIR}/manywheel/build_docker.sh" diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh deleted file mode 100644 index 08ca92466..000000000 --- a/manywheel/build_common.sh +++ /dev/null @@ -1,501 +0,0 @@ -#!/usr/bin/env bash -# meant to be called only from the neighboring build.sh and build_cpu.sh scripts - -set -ex -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - - -# Require only one python installation -if [[ -z "$DESIRED_PYTHON" ]]; then - echo "Need to set DESIRED_PYTHON env variable" - exit 1 -fi -if [[ -n "$BUILD_PYTHONLESS" && -z "$LIBTORCH_VARIANT" ]]; then - echo "BUILD_PYTHONLESS is set, so need LIBTORCH_VARIANT to also be set" - echo "LIBTORCH_VARIANT should be one of shared-with-deps shared-without-deps static-with-deps static-without-deps" - exit 1 -fi - -# Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -# TODO move this into the Docker images -OS_NAME=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -if [[ "$OS_NAME" == *"CentOS Linux"* ]]; then - retry yum install -q -y zip openssl -elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then - retry yum install -q -y zip openssl -elif [[ "$OS_NAME" == *"Red Hat Enterprise Linux"* ]]; then - retry dnf install -q -y zip openssl -elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then - # TODO: Remove this once nvidia package repos are back online - # Comment out nvidia repositories to prevent them from getting apt-get updated, see https://github.com/pytorch/pytorch/issues/74968 - # shellcheck disable=SC2046 - sed -i 's/.*nvidia.*/# &/' $(find /etc/apt/ -type f -name "*.list") - - retry apt-get update - retry apt-get -y install zip openssl -fi - -# We use the package name to test the package by passing this to 'pip install' -# This is the env variable that setup.py uses to name the package. Note that -# pip 'normalizes' the name first by changing all - to _ -if [[ -z "$TORCH_PACKAGE_NAME" ]]; then - TORCH_PACKAGE_NAME='torch' -fi - -if [[ -z "$TORCH_NO_PYTHON_PACKAGE_NAME" ]]; then - TORCH_NO_PYTHON_PACKAGE_NAME='torch_no_python' -fi - -TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '-' '_')" -TORCH_NO_PYTHON_PACKAGE_NAME="$(echo $TORCH_NO_PYTHON_PACKAGE_NAME | tr '-' '_')" -echo "Expecting the built wheels to all be called '$TORCH_PACKAGE_NAME' or '$TORCH_NO_PYTHON_PACKAGE_NAME'" - -# Version: setup.py uses $PYTORCH_BUILD_VERSION.post$PYTORCH_BUILD_NUMBER if -# PYTORCH_BUILD_NUMBER > 1 -build_version="$PYTORCH_BUILD_VERSION" -build_number="$PYTORCH_BUILD_NUMBER" -if [[ -n "$OVERRIDE_PACKAGE_VERSION" ]]; then - # This will be the *exact* version, since build_number<1 - build_version="$OVERRIDE_PACKAGE_VERSION" - build_number=0 -fi -if [[ -z "$build_version" ]]; then - build_version=1.0.0 -fi -if [[ -z "$build_number" ]]; then - build_number=1 -fi -export PYTORCH_BUILD_VERSION=$build_version -export PYTORCH_BUILD_NUMBER=$build_number - -export CMAKE_LIBRARY_PATH="/opt/intel/lib:/lib:$CMAKE_LIBRARY_PATH" -export CMAKE_INCLUDE_PATH="/opt/intel/include:$CMAKE_INCLUDE_PATH" - -if [[ -e /opt/openssl ]]; then - export OPENSSL_ROOT_DIR=/opt/openssl - export CMAKE_INCLUDE_PATH="/opt/openssl/include":$CMAKE_INCLUDE_PATH -fi - -# If given a python version like 3.6m or 2.7mu, convert this to the format we -# expect. The binary CI jobs pass in python versions like this; they also only -# ever pass one python version, so we assume that DESIRED_PYTHON is not a list -# in this case -if [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then - python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)" - DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}" -fi - -if [[ ${python_nodot} -ge 310 ]]; then - py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:2}" -else - py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:1}" -fi - - -pydir="/opt/python/$DESIRED_PYTHON" -export PATH="$pydir/bin:$PATH" -echo "Will build for Python version: ${DESIRED_PYTHON} with ${python_installation}" - -mkdir -p /tmp/$WHEELHOUSE_DIR - -export PATCHELF_BIN=/usr/local/bin/patchelf -patchelf_version=$($PATCHELF_BIN --version) -echo "patchelf version: " $patchelf_version -if [[ "$patchelf_version" == "patchelf 0.9" ]]; then - echo "Your patchelf version is too old. Please use version >= 0.10." - exit 1 -fi - -######################################################## -# Compile wheels as well as libtorch -####################################################### -if [[ -z "$PYTORCH_ROOT" ]]; then - echo "Need to set PYTORCH_ROOT env variable" - exit 1 -fi -pushd "$PYTORCH_ROOT" -python setup.py clean -retry pip install -qr requirements.txt -case ${DESIRED_PYTHON} in - cp38*) - retry pip install -q numpy==1.15 - ;; - cp31*) - retry pip install -q --pre numpy==2.0.0rc1 - ;; - # Should catch 3.9+ - *) - retry pip install -q --pre numpy==2.0.0rc1 - ;; -esac - -if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then - export _GLIBCXX_USE_CXX11_ABI=1 -else - export _GLIBCXX_USE_CXX11_ABI=0 -fi - -if [[ "$DESIRED_CUDA" == *"rocm"* ]]; then - echo "Calling build_amd.py at $(date)" - python tools/amd_build/build_amd.py -fi - -# This value comes from binary_linux_build.sh (and should only be set to true -# for master / release branches) -BUILD_DEBUG_INFO=${BUILD_DEBUG_INFO:=0} - -if [[ $BUILD_DEBUG_INFO == "1" ]]; then - echo "Building wheel and debug info" -else - echo "BUILD_DEBUG_INFO was not set, skipping debug info" -fi - -if [[ "$DISABLE_RCCL" = 1 ]]; then - echo "Disabling NCCL/RCCL in pyTorch" - USE_RCCL=0 - USE_NCCL=0 - USE_KINETO=0 -else - USE_RCCL=1 - USE_NCCL=1 - USE_KINETO=1 -fi - -echo "Calling setup.py bdist at $(date)" - -if [[ "$USE_SPLIT_BUILD" == "true" ]]; then - echo "Calling setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)" - time EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \ - BUILD_LIBTORCH_WHL=1 BUILD_PYTHON_ONLY=0 \ - BUILD_LIBTORCH_CPU_WITH_DEBUG=$BUILD_DEBUG_INFO \ - USE_NCCL=${USE_NCCL} USE_RCCL=${USE_RCCL} USE_KINETO=${USE_KINETO} \ - python setup.py bdist_wheel -d /tmp/$WHEELHOUSE_DIR - echo "Finished setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)" - echo "Calling setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)" - time EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \ - BUILD_LIBTORCH_WHL=0 BUILD_PYTHON_ONLY=1 \ - BUILD_LIBTORCH_CPU_WITH_DEBUG=$BUILD_DEBUG_INFO \ - USE_NCCL=${USE_NCCL} USE_RCCL=${USE_RCCL} USE_KINETO=${USE_KINETO} \ - python setup.py bdist_wheel -d /tmp/$WHEELHOUSE_DIR --cmake - echo "Finished setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)" -else - time CMAKE_ARGS=${CMAKE_ARGS[@]} \ - EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \ - BUILD_LIBTORCH_CPU_WITH_DEBUG=$BUILD_DEBUG_INFO \ - USE_NCCL=${USE_NCCL} USE_RCCL=${USE_RCCL} USE_KINETO=${USE_KINETO} \ - python setup.py bdist_wheel -d /tmp/$WHEELHOUSE_DIR -fi -echo "Finished setup.py bdist at $(date)" - -# Build libtorch packages -if [[ -n "$BUILD_PYTHONLESS" ]]; then - # Now build pythonless libtorch - # Note - just use whichever python we happen to be on - python setup.py clean - - if [[ $LIBTORCH_VARIANT = *"static"* ]]; then - STATIC_CMAKE_FLAG="-DTORCH_STATIC=1" - fi - - mkdir -p build - pushd build - echo "Calling tools/build_libtorch.py at $(date)" - time CMAKE_ARGS=${CMAKE_ARGS[@]} \ - EXTRA_CAFFE2_CMAKE_FLAGS="${EXTRA_CAFFE2_CMAKE_FLAGS[@]} $STATIC_CMAKE_FLAG" \ - python ../tools/build_libtorch.py - echo "Finished tools/build_libtorch.py at $(date)" - popd - - mkdir -p libtorch/{lib,bin,include,share} - cp -r build/build/lib libtorch/ - - # for now, the headers for the libtorch package will just be copied in - # from one of the wheels (this is from when this script built multiple - # wheels at once) - ANY_WHEEL=$(ls /tmp/$WHEELHOUSE_DIR/torch*.whl | head -n1) - unzip -d any_wheel $ANY_WHEEL - if [[ -d any_wheel/torch/include ]]; then - cp -r any_wheel/torch/include libtorch/ - else - cp -r any_wheel/torch/lib/include libtorch/ - fi - cp -r any_wheel/torch/share/cmake libtorch/share/ - rm -rf any_wheel - - echo $PYTORCH_BUILD_VERSION > libtorch/build-version - echo "$(pushd $PYTORCH_ROOT && git rev-parse HEAD)" > libtorch/build-hash - - mkdir -p /tmp/$LIBTORCH_HOUSE_DIR - - if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then - LIBTORCH_ABI="cxx11-abi-" - else - LIBTORCH_ABI= - fi - - zip -rq /tmp/$LIBTORCH_HOUSE_DIR/libtorch-$LIBTORCH_ABI$LIBTORCH_VARIANT-$PYTORCH_BUILD_VERSION.zip libtorch - cp /tmp/$LIBTORCH_HOUSE_DIR/libtorch-$LIBTORCH_ABI$LIBTORCH_VARIANT-$PYTORCH_BUILD_VERSION.zip \ - /tmp/$LIBTORCH_HOUSE_DIR/libtorch-$LIBTORCH_ABI$LIBTORCH_VARIANT-latest.zip -fi - -popd - -####################################################################### -# ADD DEPENDENCIES INTO THE WHEEL -# -# auditwheel repair doesn't work correctly and is buggy -# so manually do the work of copying dependency libs and patchelfing -# and fixing RECORDS entries correctly -###################################################################### - -fname_with_sha256() { - HASH=$(sha256sum $1 | cut -c1-8) - DIRNAME=$(dirname $1) - BASENAME=$(basename $1) - # Do not rename nvrtc-builtins.so as they are dynamically loaded - # by libnvrtc.so - # Similarly don't mangle libcudnn and libcublas library names - if [[ $BASENAME == "libnvrtc-builtins.s"* || $BASENAME == "libcudnn"* || $BASENAME == "libcublas"* ]]; then - echo $1 - else - INITNAME=$(echo $BASENAME | cut -f1 -d".") - ENDNAME=$(echo $BASENAME | cut -f 2- -d".") - echo "$DIRNAME/$INITNAME-$HASH.$ENDNAME" - fi -} - -fname_without_so_number() { - LINKNAME=$(echo $1 | sed -e 's/\.so.*/.so/g') - echo "$LINKNAME" -} - -make_wheel_record() { - FPATH=$1 - if echo $FPATH | grep RECORD >/dev/null 2>&1; then - # if the RECORD file, then - echo "$FPATH,," - else - HASH=$(openssl dgst -sha256 -binary $FPATH | openssl base64 | sed -e 's/+/-/g' | sed -e 's/\//_/g' | sed -e 's/=//g') - FSIZE=$(ls -nl $FPATH | awk '{print $5}') - echo "$FPATH,sha256=$HASH,$FSIZE" - fi -} - -replace_needed_sofiles() { - find $1 -name '*.so*' | while read sofile; do - origname=$2 - patchedname=$3 - if [[ "$origname" != "$patchedname" ]] || [[ "$DESIRED_CUDA" == *"rocm"* ]]; then - set +e - origname=$($PATCHELF_BIN --print-needed $sofile | grep "$origname.*") - ERRCODE=$? - set -e - if [ "$ERRCODE" -eq "0" ]; then - echo "patching $sofile entry $origname to $patchedname" - $PATCHELF_BIN --replace-needed $origname $patchedname $sofile - fi - fi - done -} - -echo 'Built this wheel:' -ls /tmp/$WHEELHOUSE_DIR -mkdir -p "/$WHEELHOUSE_DIR" -mv /tmp/$WHEELHOUSE_DIR/torch*linux*.whl /$WHEELHOUSE_DIR/ - -if [[ "$USE_SPLIT_BUILD" == "true" ]]; then - mv /tmp/$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/ || true -fi - -if [[ -n "$BUILD_PYTHONLESS" ]]; then - mkdir -p /$LIBTORCH_HOUSE_DIR - mv /tmp/$LIBTORCH_HOUSE_DIR/*.zip /$LIBTORCH_HOUSE_DIR - rm -rf /tmp/$LIBTORCH_HOUSE_DIR -fi -rm -rf /tmp/$WHEELHOUSE_DIR -rm -rf /tmp_dir -mkdir /tmp_dir -pushd /tmp_dir - -for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.whl /$LIBTORCH_HOUSE_DIR/libtorch*.zip; do - - # if the glob didn't match anything - if [[ ! -e $pkg ]]; then - continue - fi - - rm -rf tmp - mkdir -p tmp - cd tmp - cp $pkg . - - unzip -q $(basename $pkg) - rm -f $(basename $pkg) - - if [[ -d torch ]]; then - PREFIX=torch - else - PREFIX=libtorch - fi - - if [[ $pkg != *"without-deps"* ]]; then - # copy over needed dependent .so files over and tag them with their hash - patched=() - for filepath in "${DEPS_LIST[@]}"; do - filename=$(basename $filepath) - destpath=$PREFIX/lib/$filename - if [[ "$filepath" != "$destpath" ]]; then - cp $filepath $destpath - fi - - # ROCm workaround for roctracer dlopens - if [[ "$DESIRED_CUDA" == *"rocm"* ]]; then - patchedpath=$(fname_without_so_number $destpath) - else - patchedpath=$(fname_with_sha256 $destpath) - fi - patchedname=$(basename $patchedpath) - if [[ "$destpath" != "$patchedpath" ]]; then - mv $destpath $patchedpath - fi - patched+=("$patchedname") - echo "Copied $filepath to $patchedpath" - done - - echo "patching to fix the so names to the hashed names" - for ((i=0;i<${#DEPS_LIST[@]};++i)); do - replace_needed_sofiles $PREFIX ${DEPS_SONAME[i]} ${patched[i]} - # do the same for caffe2, if it exists - if [[ -d caffe2 ]]; then - replace_needed_sofiles caffe2 ${DEPS_SONAME[i]} ${patched[i]} - fi - done - - # copy over needed auxiliary files - for ((i=0;i<${#DEPS_AUX_SRCLIST[@]};++i)); do - srcpath=${DEPS_AUX_SRCLIST[i]} - dstpath=$PREFIX/${DEPS_AUX_DSTLIST[i]} - mkdir -p $(dirname $dstpath) - cp $srcpath $dstpath - done - fi - - # set RPATH of _C.so and similar to $ORIGIN, $ORIGIN/lib - find $PREFIX -maxdepth 1 -type f -name "*.so*" | while read sofile; do - echo "Setting rpath of $sofile to ${C_SO_RPATH:-'$ORIGIN:$ORIGIN/lib'}" - $PATCHELF_BIN --set-rpath ${C_SO_RPATH:-'$ORIGIN:$ORIGIN/lib'} ${FORCE_RPATH:-} $sofile - $PATCHELF_BIN --print-rpath $sofile - done - - # set RPATH of lib/ files to $ORIGIN - find $PREFIX/lib -maxdepth 1 -type f -name "*.so*" | while read sofile; do - echo "Setting rpath of $sofile to ${LIB_SO_RPATH:-'$ORIGIN'}" - $PATCHELF_BIN --set-rpath ${LIB_SO_RPATH:-'$ORIGIN'} ${FORCE_RPATH:-} $sofile - $PATCHELF_BIN --print-rpath $sofile - done - - # regenerate the RECORD file with new hashes - record_file=$(echo $(basename $pkg) | sed -e 's/-cp.*$/.dist-info\/RECORD/g') - if [[ -e $record_file ]]; then - echo "Generating new record file $record_file" - : > "$record_file" - # generate records for folders in wheel - find * -type f | while read fname; do - make_wheel_record "$fname" >>"$record_file" - done - fi - - if [[ $BUILD_DEBUG_INFO == "1" ]]; then - pushd "$PREFIX/lib" - - # Duplicate library into debug lib - cp libtorch_cpu.so libtorch_cpu.so.dbg - - # Keep debug symbols on debug lib - strip --only-keep-debug libtorch_cpu.so.dbg - - # Remove debug info from release lib - strip --strip-debug libtorch_cpu.so - - objcopy libtorch_cpu.so --add-gnu-debuglink=libtorch_cpu.so.dbg - - # Zip up debug info - mkdir -p /tmp/debug - mv libtorch_cpu.so.dbg /tmp/debug/libtorch_cpu.so.dbg - CRC32=$(objcopy --dump-section .gnu_debuglink=>(tail -c4 | od -t x4 -An | xargs echo) libtorch_cpu.so) - - pushd /tmp - PKG_NAME=$(basename "$pkg" | sed 's/\.whl$//g') - zip /tmp/debug-whl-libtorch-"$PKG_NAME"-"$CRC32".zip /tmp/debug/libtorch_cpu.so.dbg - cp /tmp/debug-whl-libtorch-"$PKG_NAME"-"$CRC32".zip "$PYTORCH_FINAL_PACKAGE_DIR" - popd - - popd - fi - - # zip up the wheel back - zip -rq $(basename $pkg) $PREIX* - - # replace original wheel - rm -f $pkg - mv $(basename $pkg) $pkg - cd .. - rm -rf tmp -done - -# Copy wheels to host machine for persistence before testing -if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then - mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true - if [[ -n "$BUILD_PYTHONLESS" ]]; then - cp /$LIBTORCH_HOUSE_DIR/libtorch*.zip "$PYTORCH_FINAL_PACKAGE_DIR" - else - cp /$WHEELHOUSE_DIR/torch*.whl "$PYTORCH_FINAL_PACKAGE_DIR" - fi -fi - -# remove stuff before testing -rm -rf /opt/rh -if ls /usr/local/cuda* >/dev/null 2>&1; then - rm -rf /usr/local/cuda* -fi - - -# Test that all the wheels work -if [[ -z "$BUILD_PYTHONLESS" ]]; then - export OMP_NUM_THREADS=4 # on NUMA machines this takes too long - pushd $PYTORCH_ROOT/test - - # Install the wheel for this Python version - if [[ "$USE_SPLIT_BUILD" == "true" ]]; then - pip uninstall -y "$TORCH_NO_PYTHON_PACKAGE_NAME" || true - fi - - pip uninstall -y "$TORCH_PACKAGE_NAME" - - if [[ "$USE_SPLIT_BUILD" == "true" ]]; then - pip install "$TORCH_NO_PYTHON_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v - fi - - pip install "$TORCH_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v - - # Print info on the libraries installed in this wheel - # Rather than adjust find command to skip non-library files with an embedded *.so* in their name, - # since this is only for reporting purposes, we add the || true to the ldd command. - installed_libraries=($(find "$pydir/lib/python${py_majmin}/site-packages/torch/" -name '*.so*')) - echo "The wheel installed all of the libraries: ${installed_libraries[@]}" - for installed_lib in "${installed_libraries[@]}"; do - ldd "$installed_lib" || true - done - - # Run the tests - echo "$(date) :: Running tests" - pushd "$PYTORCH_ROOT" - LD_LIBRARY_PATH=/usr/local/nvidia/lib64 \ - "${SOURCE_DIR}/../run_tests.sh" manywheel "${py_majmin}" "$DESIRED_CUDA" - popd - echo "$(date) :: Finished tests" -fi diff --git a/manywheel/build_cpu.sh b/manywheel/build_cpu.sh deleted file mode 100755 index 3bb2affea..000000000 --- a/manywheel/build_cpu.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -GPU_ARCH_TYPE=${GPU_ARCH_TYPE:-cpu} - -export TH_BINARY_BUILD=1 -export USE_CUDA=0 - -# Keep an array of cmake variables to add to -if [[ -z "$CMAKE_ARGS" ]]; then - # These are passed to tools/build_pytorch_libs.sh::build() - CMAKE_ARGS=() -fi -if [[ -z "$EXTRA_CAFFE2_CMAKE_FLAGS" ]]; then - # These are passed to tools/build_pytorch_libs.sh::build_caffe2() - EXTRA_CAFFE2_CMAKE_FLAGS=() -fi - -DIR_SUFFIX=cpu -if [[ "$GPU_ARCH_TYPE" == "xpu" ]]; then - DIR_SUFFIX=xpu - source /opt/intel/oneapi/pytorch-gpu-dev-0.5/oneapi-vars.sh -fi - -WHEELHOUSE_DIR="wheelhouse$DIR_SUFFIX" -LIBTORCH_HOUSE_DIR="libtorch_house$DIR_SUFFIX" -if [[ -z "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then - if [[ -z "$BUILD_PYTHONLESS" ]]; then - PYTORCH_FINAL_PACKAGE_DIR="/remote/wheelhouse$DIR_SUFFIX" - else - PYTORCH_FINAL_PACKAGE_DIR="/remote/libtorch_house$DIR_SUFFIX" - fi -fi -mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true - -OS_NAME=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -if [[ "$OS_NAME" == *"CentOS Linux"* ]]; then - LIBGOMP_PATH="/usr/lib64/libgomp.so.1" -elif [[ "$OS_NAME" == *"Red Hat Enterprise Linux"* ]]; then - LIBGOMP_PATH="/usr/lib64/libgomp.so.1" -elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then - LIBGOMP_PATH="/usr/lib64/libgomp.so.1" -elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then - if [[ "$(uname -m)" == "s390x" ]]; then - LIBGOMP_PATH="/usr/lib/s390x-linux-gnu/libgomp.so.1" - else - LIBGOMP_PATH="/usr/lib/x86_64-linux-gnu/libgomp.so.1" - fi -fi - -DEPS_LIST=( - "$LIBGOMP_PATH" -) - -DEPS_SONAME=( - "libgomp.so.1" -) - -rm -rf /usr/local/cuda* - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -if [[ -z "$BUILD_PYTHONLESS" ]]; then - BUILD_SCRIPT=build_common.sh -else - BUILD_SCRIPT=build_libtorch.sh -fi -source ${SOURCE_DIR}/${BUILD_SCRIPT} diff --git a/manywheel/build_cuda.sh b/manywheel/build_cuda.sh deleted file mode 100644 index 1d8e8b295..000000000 --- a/manywheel/build_cuda.sh +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P ))" - -export TORCH_NVCC_FLAGS="-Xfatbin -compress-all" -export NCCL_ROOT_DIR=/usr/local/cuda -export TH_BINARY_BUILD=1 -export USE_STATIC_CUDNN=1 -export USE_STATIC_NCCL=1 -export ATEN_STATIC_CUDA=1 -export USE_CUDA_STATIC_LINK=1 -export INSTALL_TEST=0 # dont install test binaries into site-packages -export USE_CUPTI_SO=0 -export USE_CUSPARSELT=${USE_CUSPARSELT:-1} # Enable if not disabled by libtorch build - -# Keep an array of cmake variables to add to -if [[ -z "$CMAKE_ARGS" ]]; then - # These are passed to tools/build_pytorch_libs.sh::build() - CMAKE_ARGS=() -fi -if [[ -z "$EXTRA_CAFFE2_CMAKE_FLAGS" ]]; then - # These are passed to tools/build_pytorch_libs.sh::build_caffe2() - EXTRA_CAFFE2_CMAKE_FLAGS=() -fi - -# Determine CUDA version and architectures to build for -# -# NOTE: We should first check `DESIRED_CUDA` when determining `CUDA_VERSION`, -# because in some cases a single Docker image can have multiple CUDA versions -# on it, and `nvcc --version` might not show the CUDA version we want. -if [[ -n "$DESIRED_CUDA" ]]; then - # If the DESIRED_CUDA already matches the format that we expect - if [[ ${DESIRED_CUDA} =~ ^[0-9]+\.[0-9]+$ ]]; then - CUDA_VERSION=${DESIRED_CUDA} - else - # cu90, cu92, cu100, cu101 - if [[ ${#DESIRED_CUDA} -eq 4 ]]; then - CUDA_VERSION="${DESIRED_CUDA:2:1}.${DESIRED_CUDA:3:1}" - elif [[ ${#DESIRED_CUDA} -eq 5 ]]; then - CUDA_VERSION="${DESIRED_CUDA:2:2}.${DESIRED_CUDA:4:1}" - fi - fi - echo "Using CUDA $CUDA_VERSION as determined by DESIRED_CUDA" - - # There really has to be a better way to do this - eli - # Possibly limiting builds to specific cuda versions be delimiting images would be a choice - if [[ "$OS_NAME" == *"Ubuntu"* ]]; then - echo "Switching to CUDA version ${DESIRED_CUDA}" - /builder/conda/switch_cuda_version.sh "${DESIRED_CUDA}" - fi -else - CUDA_VERSION=$(nvcc --version|grep release|cut -f5 -d" "|cut -f1 -d",") - echo "CUDA $CUDA_VERSION Detected" -fi - -cuda_version_nodot=$(echo $CUDA_VERSION | tr -d '.') - -TORCH_CUDA_ARCH_LIST="5.0;6.0;7.0;7.5;8.0;8.6" -case ${CUDA_VERSION} in - 12.4) - if [[ "$GPU_ARCH_TYPE" = "cuda-aarch64" ]]; then - TORCH_CUDA_ARCH_LIST="9.0" - else - TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST};9.0" - fi - EXTRA_CAFFE2_CMAKE_FLAGS+=("-DATEN_NO_TEST=ON") - ;; - 12.1) - TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST};9.0" - EXTRA_CAFFE2_CMAKE_FLAGS+=("-DATEN_NO_TEST=ON") - ;; - 11.8) - TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST};3.7;9.0" - EXTRA_CAFFE2_CMAKE_FLAGS+=("-DATEN_NO_TEST=ON") - ;; - 11.[67]) - TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST};3.7" - EXTRA_CAFFE2_CMAKE_FLAGS+=("-DATEN_NO_TEST=ON") - ;; - *) - echo "unknown cuda version $CUDA_VERSION" - exit 1 - ;; -esac - -if [[ -n "$OVERRIDE_TORCH_CUDA_ARCH_LIST" ]]; then - TORCH_CUDA_ARCH_LIST="$OVERRIDE_TORCH_CUDA_ARCH_LIST" - - # Prune CUDA again with new arch list. Unfortunately, we need to re-install CUDA to prune it again - override_gencode="" - for arch in ${TORCH_CUDA_ARCH_LIST//;/ } ; do - arch_code=$(echo "$arch" | tr -d .) - override_gencode="${override_gencode}-gencode arch=compute_$arch_code,code=sm_$arch_code " - done - - export OVERRIDE_GENCODE=$override_gencode - bash "$(dirname "$SCRIPTPATH")"/common/install_cuda.sh "${CUDA_VERSION}" -fi - -export TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST} -echo "${TORCH_CUDA_ARCH_LIST}" - -# Package directories -WHEELHOUSE_DIR="wheelhouse$cuda_version_nodot" -LIBTORCH_HOUSE_DIR="libtorch_house$cuda_version_nodot" -if [[ -z "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then - if [[ -z "$BUILD_PYTHONLESS" ]]; then - PYTORCH_FINAL_PACKAGE_DIR="/remote/wheelhouse$cuda_version_nodot" - else - PYTORCH_FINAL_PACKAGE_DIR="/remote/libtorch_house$cuda_version_nodot" - fi -fi -mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true - -OS_NAME=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -if [[ "$OS_NAME" == *"CentOS Linux"* ]]; then - LIBGOMP_PATH="/usr/lib64/libgomp.so.1" -elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then - LIBGOMP_PATH="/usr/lib64/libgomp.so.1" -elif [[ "$OS_NAME" == *"Red Hat Enterprise Linux"* ]]; then - LIBGOMP_PATH="/usr/lib64/libgomp.so.1" -elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then - LIBGOMP_PATH="/usr/lib/x86_64-linux-gnu/libgomp.so.1" -fi - -DEPS_LIST=( - "$LIBGOMP_PATH" -) -DEPS_SONAME=( - "libgomp.so.1" -) - -if [[ $USE_CUSPARSELT == "1" ]]; then - DEPS_SONAME+=( - "libcusparseLt.so.0" - ) - DEPS_LIST+=( - "/usr/local/cuda/lib64/libcusparseLt.so.0" - ) -fi - -if [[ $CUDA_VERSION == "12.1" || $CUDA_VERSION == "12.4" ]]; then - export USE_STATIC_CUDNN=0 - # Try parallelizing nvcc as well - export TORCH_NVCC_FLAGS="-Xfatbin -compress-all --threads 2" - - if [[ -z "$PYTORCH_EXTRA_INSTALL_REQUIREMENTS" ]]; then - echo "Bundling with cudnn and cublas." - DEPS_LIST+=( - "/usr/local/cuda/lib64/libcudnn_adv.so.9" - "/usr/local/cuda/lib64/libcudnn_cnn.so.9" - "/usr/local/cuda/lib64/libcudnn_graph.so.9" - "/usr/local/cuda/lib64/libcudnn_ops.so.9" - "/usr/local/cuda/lib64/libcudnn_engines_runtime_compiled.so.9" - "/usr/local/cuda/lib64/libcudnn_engines_precompiled.so.9" - "/usr/local/cuda/lib64/libcudnn_heuristic.so.9" - "/usr/local/cuda/lib64/libcudnn.so.9" - "/usr/local/cuda/lib64/libcublas.so.12" - "/usr/local/cuda/lib64/libcublasLt.so.12" - "/usr/local/cuda/lib64/libcudart.so.12" - "/usr/local/cuda/lib64/libnvToolsExt.so.1" - "/usr/local/cuda/lib64/libnvrtc.so.12" - "/usr/local/cuda/lib64/libnvrtc-builtins.so" - ) - DEPS_SONAME+=( - "libcudnn_adv.so.9" - "libcudnn_cnn.so.9" - "libcudnn_graph.so.9" - "libcudnn_ops.so.9" - "libcudnn_engines_runtime_compiled.so.9" - "libcudnn_engines_precompiled.so.9" - "libcudnn_heuristic.so.9" - "libcudnn.so.9" - "libcublas.so.12" - "libcublasLt.so.12" - "libcudart.so.12" - "libnvToolsExt.so.1" - "libnvrtc.so.12" - "libnvrtc-builtins.so" - ) - else - echo "Using nvidia libs from pypi." - CUDA_RPATHS=( - '$ORIGIN/../../nvidia/cublas/lib' - '$ORIGIN/../../nvidia/cuda_cupti/lib' - '$ORIGIN/../../nvidia/cuda_nvrtc/lib' - '$ORIGIN/../../nvidia/cuda_runtime/lib' - '$ORIGIN/../../nvidia/cudnn/lib' - '$ORIGIN/../../nvidia/cufft/lib' - '$ORIGIN/../../nvidia/curand/lib' - '$ORIGIN/../../nvidia/cusolver/lib' - '$ORIGIN/../../nvidia/cusparse/lib' - '$ORIGIN/../../nvidia/nccl/lib' - '$ORIGIN/../../nvidia/nvtx/lib' - ) - CUDA_RPATHS=$(IFS=: ; echo "${CUDA_RPATHS[*]}") - export C_SO_RPATH=$CUDA_RPATHS':$ORIGIN:$ORIGIN/lib' - export LIB_SO_RPATH=$CUDA_RPATHS':$ORIGIN' - export FORCE_RPATH="--force-rpath" - export USE_STATIC_NCCL=0 - export USE_SYSTEM_NCCL=1 - export ATEN_STATIC_CUDA=0 - export USE_CUDA_STATIC_LINK=0 - export USE_CUPTI_SO=1 - export NCCL_INCLUDE_DIR="/usr/local/cuda/include/" - export NCCL_LIB_DIR="/usr/local/cuda/lib64/" - fi -elif [[ $CUDA_VERSION == "11.8" ]]; then - export USE_STATIC_CUDNN=0 - # Try parallelizing nvcc as well - export TORCH_NVCC_FLAGS="-Xfatbin -compress-all --threads 2" - # Bundle ptxas into the wheel, see https://github.com/pytorch/pytorch/pull/119750 - export BUILD_BUNDLE_PTXAS=1 - - if [[ -z "$PYTORCH_EXTRA_INSTALL_REQUIREMENTS" ]]; then - echo "Bundling with cudnn and cublas." - DEPS_LIST+=( - "/usr/local/cuda/lib64/libcudnn_adv.so.9" - "/usr/local/cuda/lib64/libcudnn_cnn.so.9" - "/usr/local/cuda/lib64/libcudnn_graph.so.9" - "/usr/local/cuda/lib64/libcudnn_ops.so.9" - "/usr/local/cuda/lib64/libcudnn_engines_runtime_compiled.so.9" - "/usr/local/cuda/lib64/libcudnn_engines_precompiled.so.9" - "/usr/local/cuda/lib64/libcudnn_heuristic.so.9" - "/usr/local/cuda/lib64/libcudnn.so.9" - "/usr/local/cuda/lib64/libcublas.so.11" - "/usr/local/cuda/lib64/libcublasLt.so.11" - "/usr/local/cuda/lib64/libcudart.so.11.0" - "/usr/local/cuda/lib64/libnvToolsExt.so.1" - "/usr/local/cuda/lib64/libnvrtc.so.11.2" # this is not a mistake, it links to more specific cuda version - "/usr/local/cuda/lib64/libnvrtc-builtins.so.11.8" - ) - DEPS_SONAME+=( - "libcudnn_adv.so.9" - "libcudnn_cnn.so.9" - "libcudnn_graph.so.9" - "libcudnn_ops.so.9" - "libcudnn_engines_runtime_compiled.so.9" - "libcudnn_engines_precompiled.so.9" - "libcudnn_heuristic.so.9" - "libcudnn.so.9" - "libcublas.so.11" - "libcublasLt.so.11" - "libcudart.so.11.0" - "libnvToolsExt.so.1" - "libnvrtc.so.11.2" - "libnvrtc-builtins.so.11.8" - ) - else - echo "Using nvidia libs from pypi." - CUDA_RPATHS=( - '$ORIGIN/../../nvidia/cublas/lib' - '$ORIGIN/../../nvidia/cuda_cupti/lib' - '$ORIGIN/../../nvidia/cuda_nvrtc/lib' - '$ORIGIN/../../nvidia/cuda_runtime/lib' - '$ORIGIN/../../nvidia/cudnn/lib' - '$ORIGIN/../../nvidia/cufft/lib' - '$ORIGIN/../../nvidia/curand/lib' - '$ORIGIN/../../nvidia/cusolver/lib' - '$ORIGIN/../../nvidia/cusparse/lib' - '$ORIGIN/../../nvidia/nccl/lib' - '$ORIGIN/../../nvidia/nvtx/lib' - ) - CUDA_RPATHS=$(IFS=: ; echo "${CUDA_RPATHS[*]}") - export C_SO_RPATH=$CUDA_RPATHS':$ORIGIN:$ORIGIN/lib' - export LIB_SO_RPATH=$CUDA_RPATHS':$ORIGIN' - export FORCE_RPATH="--force-rpath" - export USE_STATIC_NCCL=0 - export USE_SYSTEM_NCCL=1 - export ATEN_STATIC_CUDA=0 - export USE_CUDA_STATIC_LINK=0 - export USE_CUPTI_SO=1 - export NCCL_INCLUDE_DIR="/usr/local/cuda/include/" - export NCCL_LIB_DIR="/usr/local/cuda/lib64/" - fi -else - echo "Unknown cuda version $CUDA_VERSION" - exit 1 -fi - -# builder/test.sh requires DESIRED_CUDA to know what tests to exclude -export DESIRED_CUDA="$cuda_version_nodot" - -# Switch `/usr/local/cuda` to the desired CUDA version -rm -rf /usr/local/cuda || true -ln -s "/usr/local/cuda-${CUDA_VERSION}" /usr/local/cuda - -# Switch `/usr/local/magma` to the desired CUDA version -rm -rf /usr/local/magma || true -ln -s /usr/local/cuda-${CUDA_VERSION}/magma /usr/local/magma - -export CUDA_VERSION=$(ls /usr/local/cuda/lib64/libcudart.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev) # 10.0.130 -export CUDA_VERSION_SHORT=$(ls /usr/local/cuda/lib64/libcudart.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev | cut -f1,2 -d".") # 10.0 -export CUDNN_VERSION=$(ls /usr/local/cuda/lib64/libcudnn.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev) - -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -if [[ -z "$BUILD_PYTHONLESS" ]]; then - BUILD_SCRIPT=build_common.sh -else - BUILD_SCRIPT=build_libtorch.sh -fi -source $SCRIPTPATH/${BUILD_SCRIPT} diff --git a/manywheel/build_docker.sh b/manywheel/build_docker.sh deleted file mode 100755 index 1e7f9ab1b..000000000 --- a/manywheel/build_docker.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -TOPDIR=$(git rev-parse --show-toplevel) - -DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}" - -GPU_ARCH_TYPE=${GPU_ARCH_TYPE:-cpu} -GPU_ARCH_VERSION=${GPU_ARCH_VERSION:-} -MANY_LINUX_VERSION=${MANY_LINUX_VERSION:-} -DOCKERFILE_SUFFIX=${DOCKERFILE_SUFFIX:-} -WITH_PUSH=${WITH_PUSH:-} - -case ${GPU_ARCH_TYPE} in - cpu) - TARGET=cpu_final - DOCKER_TAG=cpu - GPU_IMAGE=centos:7 - DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9" - ;; - cpu-manylinux_2_28) - TARGET=cpu_final - DOCKER_TAG=cpu - GPU_IMAGE=amd64/almalinux:8 - DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11" - MANY_LINUX_VERSION="2_28" - ;; - cpu-aarch64) - TARGET=final - DOCKER_TAG=cpu-aarch64 - GPU_IMAGE=arm64v8/centos:7 - DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=10" - MANY_LINUX_VERSION="aarch64" - ;; - cpu-aarch64-2_28) - TARGET=final - DOCKER_TAG=cpu-aarch64 - GPU_IMAGE=arm64v8/almalinux:8 - DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11" - MANY_LINUX_VERSION="2_28_aarch64" - ;; - cpu-cxx11-abi) - TARGET=final - DOCKER_TAG=cpu-cxx11-abi - GPU_IMAGE="" - DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9" - MANY_LINUX_VERSION="cxx11-abi" - ;; - cpu-s390x) - TARGET=final - DOCKER_TAG=cpu-s390x - GPU_IMAGE=redhat/ubi9 - DOCKER_GPU_BUILD_ARG="" - MANY_LINUX_VERSION="s390x" - ;; - cuda) - TARGET=cuda_final - DOCKER_TAG=cuda${GPU_ARCH_VERSION} - # Keep this up to date with the minimum version of CUDA we currently support - GPU_IMAGE=centos:7 - DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=9" - ;; - cuda-manylinux_2_28) - TARGET=cuda_final - DOCKER_TAG=cuda${GPU_ARCH_VERSION} - GPU_IMAGE=amd64/almalinux:8 - DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11" - MANY_LINUX_VERSION="2_28" - ;; - cuda-aarch64) - TARGET=cuda_final - DOCKER_TAG=cuda${GPU_ARCH_VERSION} - GPU_IMAGE=arm64v8/centos:7 - DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11" - MANY_LINUX_VERSION="aarch64" - DOCKERFILE_SUFFIX="_cuda_aarch64" - ;; - rocm) - TARGET=rocm_final - DOCKER_TAG=rocm${GPU_ARCH_VERSION} - GPU_IMAGE=rocm/dev-centos-7:${GPU_ARCH_VERSION}-complete - PYTORCH_ROCM_ARCH="gfx900;gfx906;gfx908;gfx90a;gfx1030;gfx1100" - ROCM_REGEX="([0-9]+)\.([0-9]+)[\.]?([0-9]*)" - if [[ $GPU_ARCH_VERSION =~ $ROCM_REGEX ]]; then - ROCM_VERSION_INT=$((${BASH_REMATCH[1]}*10000 + ${BASH_REMATCH[2]}*100 + ${BASH_REMATCH[3]:-0})) - else - echo "ERROR: rocm regex failed" - exit 1 - fi - if [[ $ROCM_VERSION_INT -ge 60000 ]]; then - PYTORCH_ROCM_ARCH+=";gfx942" - fi - DOCKER_GPU_BUILD_ARG="--build-arg ROCM_VERSION=${GPU_ARCH_VERSION} --build-arg PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} --build-arg DEVTOOLSET_VERSION=9" - ;; - xpu) - TARGET=xpu_final - DOCKER_TAG=xpu - GPU_IMAGE=amd64/almalinux:8 - DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11" - MANY_LINUX_VERSION="2_28" - ;; - *) - echo "ERROR: Unrecognized GPU_ARCH_TYPE: ${GPU_ARCH_TYPE}" - exit 1 - ;; -esac - -IMAGES='' -DOCKER_NAME=manylinux${MANY_LINUX_VERSION} -DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/${DOCKER_NAME}-builder:${DOCKER_TAG} -if [[ -n ${MANY_LINUX_VERSION} && -z ${DOCKERFILE_SUFFIX} ]]; then - DOCKERFILE_SUFFIX=_${MANY_LINUX_VERSION} -fi -( - set -x - DOCKER_BUILDKIT=1 docker build \ - -t "${DOCKER_IMAGE}" \ - ${DOCKER_GPU_BUILD_ARG} \ - --build-arg "GPU_IMAGE=${GPU_IMAGE}" \ - --target "${TARGET}" \ - -f "${TOPDIR}/manywheel/Dockerfile${DOCKERFILE_SUFFIX}" \ - "${TOPDIR}" -) - -GITHUB_REF=${GITHUB_REF:-$(git symbolic-ref -q HEAD || git describe --tags --exact-match)} -GIT_BRANCH_NAME=${GITHUB_REF##*/} -GIT_COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse HEAD)} -DOCKER_IMAGE_BRANCH_TAG=${DOCKER_IMAGE}-${GIT_BRANCH_NAME} -DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE}-${GIT_COMMIT_SHA} - -( - set -x - if [[ -n ${GITHUB_REF} ]]; then - docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_BRANCH_TAG} - docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_SHA_TAG} - fi -) - -if [[ "${WITH_PUSH}" == true ]]; then - ( - set -x - docker push "${DOCKER_IMAGE}" - if [[ -n ${GITHUB_REF} ]]; then - docker push "${DOCKER_IMAGE_BRANCH_TAG}" - docker push "${DOCKER_IMAGE_SHA_TAG}" - fi - ) -fi diff --git a/manywheel/build_libtorch.sh b/manywheel/build_libtorch.sh deleted file mode 100644 index c3c0391a8..000000000 --- a/manywheel/build_libtorch.sh +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env bash -# meant to be called only from the neighboring build.sh and build_cpu.sh scripts - -set -e pipefail -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Require only one python installation -if [[ -z "$DESIRED_PYTHON" ]]; then - echo "Need to set DESIRED_PYTHON env variable" - exit 1 -fi -if [[ -n "$BUILD_PYTHONLESS" && -z "$LIBTORCH_VARIANT" ]]; then - echo "BUILD_PYTHONLESS is set, so need LIBTORCH_VARIANT to also be set" - echo "LIBTORCH_VARIANT should be one of shared-with-deps shared-without-deps static-with-deps static-without-deps" - exit 1 -fi - -# Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -# TODO move this into the Docker images -OS_NAME=`awk -F= '/^NAME/{print $2}' /etc/os-release` -if [[ "$OS_NAME" == *"CentOS Linux"* ]]; then - retry yum install -q -y zip openssl -elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then - retry yum install -q -y zip openssl -elif [[ "$OS_NAME" == *"Red Hat Enterprise Linux"* ]]; then - retry dnf install -q -y zip openssl -elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then - # TODO: Remove this once nvidia package repos are back online - # Comment out nvidia repositories to prevent them from getting apt-get updated, see https://github.com/pytorch/pytorch/issues/74968 - # shellcheck disable=SC2046 - sed -i 's/.*nvidia.*/# &/' $(find /etc/apt/ -type f -name "*.list") - retry apt-get update - retry apt-get -y install zip openssl -fi - -# Version: setup.py uses $PYTORCH_BUILD_VERSION.post$PYTORCH_BUILD_NUMBER if -# PYTORCH_BUILD_NUMBER > 1 -build_version="$PYTORCH_BUILD_VERSION" -build_number="$PYTORCH_BUILD_NUMBER" -if [[ -n "$OVERRIDE_PACKAGE_VERSION" ]]; then - # This will be the *exact* version, since build_number<1 - build_version="$OVERRIDE_PACKAGE_VERSION" - build_number=0 -fi -if [[ -z "$build_version" ]]; then - build_version=1.0.0 -fi -if [[ -z "$build_number" ]]; then - build_number=1 -fi -export PYTORCH_BUILD_VERSION=$build_version -export PYTORCH_BUILD_NUMBER=$build_number - -export CMAKE_LIBRARY_PATH="/opt/intel/lib:/lib:$CMAKE_LIBRARY_PATH" -export CMAKE_INCLUDE_PATH="/opt/intel/include:$CMAKE_INCLUDE_PATH" - -# set OPENSSL_ROOT_DIR=/opt/openssl if it exists -if [[ -e /opt/openssl ]]; then - export OPENSSL_ROOT_DIR=/opt/openssl - export CMAKE_INCLUDE_PATH="/opt/openssl/include":$CMAKE_INCLUDE_PATH -fi - -# If given a python version like 3.6m or 2.7mu, convert this to the format we -# expect. The binary CI jobs pass in python versions like this; they also only -# ever pass one python version, so we assume that DESIRED_PYTHON is not a list -# in this case -if [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then - python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)" - DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}" -fi -pydir="/opt/python/$DESIRED_PYTHON" -export PATH="$pydir/bin:$PATH" - -export PATCHELF_BIN=/usr/local/bin/patchelf -patchelf_version=`$PATCHELF_BIN --version` -echo "patchelf version: " $patchelf_version -if [[ "$patchelf_version" == "patchelf 0.9" ]]; then - echo "Your patchelf version is too old. Please use version >= 0.10." - exit 1 -fi - -######################################################## -# Compile wheels as well as libtorch -####################################################### -if [[ -z "$PYTORCH_ROOT" ]]; then - echo "Need to set PYTORCH_ROOT env variable" - exit 1 -fi -pushd "$PYTORCH_ROOT" -python setup.py clean -retry pip install -qr requirements.txt -if [[ "$DESIRED_PYTHON" == "cp38-cp38" ]]; then - retry pip install -q numpy==1.15 -else - retry pip install -q numpy==1.11 -fi - -if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then - export _GLIBCXX_USE_CXX11_ABI=1 -else - export _GLIBCXX_USE_CXX11_ABI=0 -fi - -if [[ "$DESIRED_CUDA" == *"rocm"* ]]; then - echo "Calling build_amd.py at $(date)" - python tools/amd_build/build_amd.py - # TODO remove this work-around once pytorch sources are updated - export ROCclr_DIR=/opt/rocm/rocclr/lib/cmake/rocclr -fi - -echo "Calling setup.py install at $(date)" - -if [[ $LIBTORCH_VARIANT = *"static"* ]]; then - STATIC_CMAKE_FLAG="-DTORCH_STATIC=1" -fi - -( - set -x - - mkdir -p build - - time CMAKE_ARGS=${CMAKE_ARGS[@]} \ - EXTRA_CAFFE2_CMAKE_FLAGS="${EXTRA_CAFFE2_CMAKE_FLAGS[@]} $STATIC_CMAKE_FLAG" \ - # TODO: Remove this flag once https://github.com/pytorch/pytorch/issues/55952 is closed - CFLAGS='-Wno-deprecated-declarations' \ - BUILD_LIBTORCH_CPU_WITH_DEBUG=1 \ - python setup.py install - - mkdir -p libtorch/{lib,bin,include,share} - - # Make debug folder separate so it doesn't get zipped up with the rest of - # libtorch - mkdir debug - - # Copy over all lib files - cp -rv build/lib/* libtorch/lib/ - cp -rv build/lib*/torch/lib/* libtorch/lib/ - - # Copy over all include files - cp -rv build/include/* libtorch/include/ - cp -rv build/lib*/torch/include/* libtorch/include/ - - # Copy over all of the cmake files - cp -rv build/lib*/torch/share/* libtorch/share/ - - # Split libtorch into debug / release version - cp libtorch/lib/libtorch_cpu.so libtorch/lib/libtorch_cpu.so.dbg - - # Keep debug symbols on debug lib - strip --only-keep-debug libtorch/lib/libtorch_cpu.so.dbg - - # Remove debug info from release lib - strip --strip-debug libtorch/lib/libtorch_cpu.so - - # Add a debug link to the release lib to the debug lib (debuggers will then - # search for symbols in a file called libtorch_cpu.so.dbg in some - # predetermined locations) and embed a CRC32 of the debug library into the .so - cd libtorch/lib - - objcopy libtorch_cpu.so --add-gnu-debuglink=libtorch_cpu.so.dbg - cd ../.. - - # Move the debug symbols to its own directory so it doesn't get processed / - # zipped with all the other libraries - mv libtorch/lib/libtorch_cpu.so.dbg debug/libtorch_cpu.so.dbg - - echo "${PYTORCH_BUILD_VERSION}" > libtorch/build-version - echo "$(pushd $PYTORCH_ROOT && git rev-parse HEAD)" > libtorch/build-hash - -) - -if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then - LIBTORCH_ABI="cxx11-abi-" -else - LIBTORCH_ABI= -fi - -( - set -x - - mkdir -p /tmp/$LIBTORCH_HOUSE_DIR - - # objcopy installs a CRC32 into libtorch_cpu above so, so add that to the name here - CRC32=$(objcopy --dump-section .gnu_debuglink=>(tail -c4 | od -t x4 -An | xargs echo) libtorch/lib/libtorch_cpu.so) - - # Zip debug symbols - zip /tmp/$LIBTORCH_HOUSE_DIR/debug-libtorch-$LIBTORCH_ABI$LIBTORCH_VARIANT-$PYTORCH_BUILD_VERSION-$CRC32.zip debug/libtorch_cpu.so.dbg - - # Zip and copy libtorch - zip -rq /tmp/$LIBTORCH_HOUSE_DIR/libtorch-$LIBTORCH_ABI$LIBTORCH_VARIANT-$PYTORCH_BUILD_VERSION.zip libtorch - cp /tmp/$LIBTORCH_HOUSE_DIR/libtorch-$LIBTORCH_ABI$LIBTORCH_VARIANT-$PYTORCH_BUILD_VERSION.zip \ - /tmp/$LIBTORCH_HOUSE_DIR/libtorch-$LIBTORCH_ABI$LIBTORCH_VARIANT-latest.zip -) - - -popd - -####################################################################### -# ADD DEPENDENCIES INTO THE WHEEL -# -# auditwheel repair doesn't work correctly and is buggy -# so manually do the work of copying dependency libs and patchelfing -# and fixing RECORDS entries correctly -###################################################################### - -fname_with_sha256() { - HASH=$(sha256sum $1 | cut -c1-8) - DIRNAME=$(dirname $1) - BASENAME=$(basename $1) - if [[ $BASENAME == "libnvrtc-builtins.so" || $BASENAME == "libcudnn"* ]]; then - echo $1 - else - INITNAME=$(echo $BASENAME | cut -f1 -d".") - ENDNAME=$(echo $BASENAME | cut -f 2- -d".") - echo "$DIRNAME/$INITNAME-$HASH.$ENDNAME" - fi -} - -fname_without_so_number() { - LINKNAME=$(echo $1 | sed -e 's/\.so.*/.so/g') - echo "$LINKNAME" -} - -make_wheel_record() { - FPATH=$1 - if echo $FPATH | grep RECORD >/dev/null 2>&1; then - # if the RECORD file, then - echo "$FPATH,," - else - HASH=$(openssl dgst -sha256 -binary $FPATH | openssl base64 | sed -e 's/+/-/g' | sed -e 's/\//_/g' | sed -e 's/=//g') - FSIZE=$(ls -nl $FPATH | awk '{print $5}') - echo "$FPATH,sha256=$HASH,$FSIZE" - fi -} - -echo 'Built this package:' -( - set -x - mkdir -p /$LIBTORCH_HOUSE_DIR - mv /tmp/$LIBTORCH_HOUSE_DIR/*.zip /$LIBTORCH_HOUSE_DIR - rm -rf /tmp/$LIBTORCH_HOUSE_DIR -) -TMP_DIR=$(mktemp -d) -trap "rm -rf ${TMP_DIR}" EXIT -pushd "${TMP_DIR}" - -for pkg in /$LIBTORCH_HOUSE_DIR/libtorch*.zip; do - - # if the glob didn't match anything - if [[ ! -e $pkg ]]; then - continue - fi - - rm -rf tmp - mkdir -p tmp - cd tmp - cp $pkg . - - unzip -q $(basename $pkg) - rm -f $(basename $pkg) - - PREFIX=libtorch - - if [[ $pkg != *"without-deps"* ]]; then - # copy over needed dependent .so files over and tag them with their hash - patched=() - for filepath in "${DEPS_LIST[@]}"; do - filename=$(basename $filepath) - destpath=$PREFIX/lib/$filename - if [[ "$filepath" != "$destpath" ]]; then - cp $filepath $destpath - fi - - if [[ "$DESIRED_CUDA" == *"rocm"* ]]; then - patchedpath=$(fname_without_so_number $destpath) - else - patchedpath=$(fname_with_sha256 $destpath) - fi - patchedname=$(basename $patchedpath) - if [[ "$destpath" != "$patchedpath" ]]; then - mv $destpath $patchedpath - fi - patched+=("$patchedname") - echo "Copied $filepath to $patchedpath" - done - - echo "patching to fix the so names to the hashed names" - for ((i=0;i<${#DEPS_LIST[@]};++i)); do - find $PREFIX -name '*.so*' | while read sofile; do - origname=${DEPS_SONAME[i]} - patchedname=${patched[i]} - if [[ "$origname" != "$patchedname" ]] || [[ "$DESIRED_CUDA" == *"rocm"* ]]; then - set +e - origname=$($PATCHELF_BIN --print-needed $sofile | grep "$origname.*") - ERRCODE=$? - set -e - if [ "$ERRCODE" -eq "0" ]; then - echo "patching $sofile entry $origname to $patchedname" - $PATCHELF_BIN --replace-needed $origname $patchedname $sofile - fi - fi - done - done - - # copy over needed auxiliary files - for ((i=0;i<${#DEPS_AUX_SRCLIST[@]};++i)); do - srcpath=${DEPS_AUX_SRCLIST[i]} - dstpath=$PREFIX/${DEPS_AUX_DSTLIST[i]} - mkdir -p $(dirname $dstpath) - cp $srcpath $dstpath - done - fi - - # set RPATH of _C.so and similar to $ORIGIN, $ORIGIN/lib - find $PREFIX -maxdepth 1 -type f -name "*.so*" | while read sofile; do - echo "Setting rpath of $sofile to " '$ORIGIN:$ORIGIN/lib' - $PATCHELF_BIN --set-rpath '$ORIGIN:$ORIGIN/lib' $sofile - $PATCHELF_BIN --print-rpath $sofile - done - - # set RPATH of lib/ files to $ORIGIN - find $PREFIX/lib -maxdepth 1 -type f -name "*.so*" | while read sofile; do - echo "Setting rpath of $sofile to " '$ORIGIN' - $PATCHELF_BIN --set-rpath '$ORIGIN' $sofile - $PATCHELF_BIN --print-rpath $sofile - done - - # regenerate the RECORD file with new hashes - record_file=`echo $(basename $pkg) | sed -e 's/-cp.*$/.dist-info\/RECORD/g'` - if [[ -e $record_file ]]; then - echo "Generating new record file $record_file" - rm -f $record_file - # generate records for folders in wheel - find * -type f | while read fname; do - echo $(make_wheel_record $fname) >>$record_file - done - fi - - # zip up the wheel back - zip -rq $(basename $pkg) $PREFIX* - - # replace original wheel - rm -f $pkg - mv $(basename $pkg) $pkg - cd .. - rm -rf tmp -done - -# Copy wheels to host machine for persistence before testing -if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then - cp /$LIBTORCH_HOUSE_DIR/libtorch*.zip "$PYTORCH_FINAL_PACKAGE_DIR" - cp /$LIBTORCH_HOUSE_DIR/debug-libtorch*.zip "$PYTORCH_FINAL_PACKAGE_DIR" -fi diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh deleted file mode 100755 index 77942898d..000000000 --- a/manywheel/build_rocm.sh +++ /dev/null @@ -1,288 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -export ROCM_HOME=/opt/rocm -export MAGMA_HOME=$ROCM_HOME/magma -# TODO: libtorch_cpu.so is broken when building with Debug info -export BUILD_DEBUG_INFO=0 - -# TODO Are these all used/needed? -export TH_BINARY_BUILD=1 -export USE_STATIC_CUDNN=1 -export USE_STATIC_NCCL=1 -export ATEN_STATIC_CUDA=1 -export USE_CUDA_STATIC_LINK=1 -export INSTALL_TEST=0 # dont install test binaries into site-packages -# Set RPATH instead of RUNPATH when using patchelf to avoid LD_LIBRARY_PATH override -export FORCE_RPATH="--force-rpath" - -# Keep an array of cmake variables to add to -if [[ -z "$CMAKE_ARGS" ]]; then - # These are passed to tools/build_pytorch_libs.sh::build() - CMAKE_ARGS=() -fi -if [[ -z "$EXTRA_CAFFE2_CMAKE_FLAGS" ]]; then - # These are passed to tools/build_pytorch_libs.sh::build_caffe2() - EXTRA_CAFFE2_CMAKE_FLAGS=() -fi - -# Determine ROCm version and architectures to build for -# -# NOTE: We should first check `DESIRED_CUDA` when determining `ROCM_VERSION` -if [[ -n "$DESIRED_CUDA" ]]; then - if ! echo "${DESIRED_CUDA}"| grep "^rocm" >/dev/null 2>/dev/null; then - export DESIRED_CUDA="rocm${DESIRED_CUDA}" - fi - # rocm3.7, rocm3.5.1 - ROCM_VERSION="$DESIRED_CUDA" - echo "Using $ROCM_VERSION as determined by DESIRED_CUDA" -else - echo "Must set DESIRED_CUDA" - exit 1 -fi - -# Package directories -WHEELHOUSE_DIR="wheelhouse$ROCM_VERSION" -LIBTORCH_HOUSE_DIR="libtorch_house$ROCM_VERSION" -if [[ -z "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then - if [[ -z "$BUILD_PYTHONLESS" ]]; then - PYTORCH_FINAL_PACKAGE_DIR="/remote/wheelhouse$ROCM_VERSION" - else - PYTORCH_FINAL_PACKAGE_DIR="/remote/libtorch_house$ROCM_VERSION" - fi -fi -mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true - -# To make version comparison easier, create an integer representation. -ROCM_VERSION_CLEAN=$(echo ${ROCM_VERSION} | sed s/rocm//) -save_IFS="$IFS" -IFS=. ROCM_VERSION_ARRAY=(${ROCM_VERSION_CLEAN}) -IFS="$save_IFS" -if [[ ${#ROCM_VERSION_ARRAY[@]} == 2 ]]; then - ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} - ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} - ROCM_VERSION_PATCH=0 -elif [[ ${#ROCM_VERSION_ARRAY[@]} == 3 ]]; then - ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} - ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} - ROCM_VERSION_PATCH=${ROCM_VERSION_ARRAY[2]} -else - echo "Unhandled ROCM_VERSION ${ROCM_VERSION}" - exit 1 -fi -ROCM_INT=$(($ROCM_VERSION_MAJOR * 10000 + $ROCM_VERSION_MINOR * 100 + $ROCM_VERSION_PATCH)) - -# Required ROCm libraries -ROCM_SO_FILES=( - "libMIOpen.so" - "libamdhip64.so" - "libhipblas.so" - "libhipfft.so" - "libhiprand.so" - "libhipsolver.so" - "libhipsparse.so" - "libhsa-runtime64.so" - "libamd_comgr.so" - "libmagma.so" - "librccl.so" - "librocblas.so" - "librocfft.so" - "librocm_smi64.so" - "librocrand.so" - "librocsolver.so" - "librocsparse.so" - "libroctracer64.so" - "libroctx64.so" -) - -if [[ $ROCM_INT -ge 50600 ]]; then - ROCM_SO_FILES+=("libhipblaslt.so") -fi - -if [[ $ROCM_INT -lt 50500 ]]; then - ROCM_SO_FILES+=("librocfft-device-0.so") - ROCM_SO_FILES+=("librocfft-device-1.so") - ROCM_SO_FILES+=("librocfft-device-2.so") - ROCM_SO_FILES+=("librocfft-device-3.so") -fi - -if [[ $ROCM_INT -ge 50400 ]]; then - ROCM_SO_FILES+=("libhiprtc.so") -fi - -if [[ $ROCM_INT -ge 60100 ]]; then - ROCM_SO_FILES+=("librocprofiler-register.so") -fi - -OS_NAME=`awk -F= '/^NAME/{print $2}' /etc/os-release` -if [[ "$OS_NAME" == *"CentOS Linux"* ]]; then - LIBGOMP_PATH="/usr/lib64/libgomp.so.1" - LIBNUMA_PATH="/usr/lib64/libnuma.so.1" - LIBELF_PATH="/usr/lib64/libelf.so.1" - LIBTINFO_PATH="/usr/lib64/libtinfo.so.5" - LIBDRM_PATH="/opt/amdgpu/lib64/libdrm.so.2" - LIBDRM_AMDGPU_PATH="/opt/amdgpu/lib64/libdrm_amdgpu.so.1" - if [[ $ROCM_INT -ge 60100 ]]; then - # Below libs are direct dependencies of libhipsolver - LIBSUITESPARSE_CONFIG_PATH="/lib64/libsuitesparseconfig.so.4" - LIBCHOLMOD_PATH="/lib64/libcholmod.so.2" - # Below libs are direct dependencies of libcholmod - LIBAMD_PATH="/lib64/libamd.so.2" - LIBCAMD_PATH="/lib64/libcamd.so.2" - LIBCCOLAMD_PATH="/lib64/libccolamd.so.2" - LIBCOLAMD_PATH="/lib64/libcolamd.so.2" - LIBSATLAS_PATH="/lib64/atlas/libsatlas.so.3" - # Below libs are direct dependencies of libsatlas - LIBGFORTRAN_PATH="/lib64/libgfortran.so.3" - LIBQUADMATH_PATH="/lib64/libquadmath.so.0" - fi - MAYBE_LIB64=lib64 -elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then - LIBGOMP_PATH="/usr/lib/x86_64-linux-gnu/libgomp.so.1" - LIBNUMA_PATH="/usr/lib/x86_64-linux-gnu/libnuma.so.1" - LIBELF_PATH="/usr/lib/x86_64-linux-gnu/libelf.so.1" - if [[ $ROCM_INT -ge 50300 ]]; then - LIBTINFO_PATH="/lib/x86_64-linux-gnu/libtinfo.so.6" - else - LIBTINFO_PATH="/lib/x86_64-linux-gnu/libtinfo.so.5" - fi - LIBDRM_PATH="/usr/lib/x86_64-linux-gnu/libdrm.so.2" - LIBDRM_AMDGPU_PATH="/usr/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1" - if [[ $ROCM_INT -ge 60100 ]]; then - # Below libs are direct dependencies of libhipsolver - LIBCHOLMOD_PATH="/lib/x86_64-linux-gnu/libcholmod.so.3" - # Below libs are direct dependencies of libcholmod - LIBSUITESPARSE_CONFIG_PATH="/lib/x86_64-linux-gnu/libsuitesparseconfig.so.5" - LIBAMD_PATH="/lib/x86_64-linux-gnu/libamd.so.2" - LIBCAMD_PATH="/lib/x86_64-linux-gnu/libcamd.so.2" - LIBCCOLAMD_PATH="/lib/x86_64-linux-gnu/libccolamd.so.2" - LIBCOLAMD_PATH="/lib/x86_64-linux-gnu/libcolamd.so.2" - LIBMETIS_PATH="/lib/x86_64-linux-gnu/libmetis.so.5" - LIBLAPACK_PATH="/lib/x86_64-linux-gnu/liblapack.so.3" - LIBBLAS_PATH="/lib/x86_64-linux-gnu/libblas.so.3" - # Below libs are direct dependencies of libblas - LIBGFORTRAN_PATH="/lib/x86_64-linux-gnu/libgfortran.so.5" - LIBQUADMATH_PATH="/lib/x86_64-linux-gnu/libquadmath.so.0" - fi - MAYBE_LIB64=lib -fi -OS_SO_PATHS=($LIBGOMP_PATH $LIBNUMA_PATH\ - $LIBELF_PATH $LIBTINFO_PATH\ - $LIBDRM_PATH $LIBDRM_AMDGPU_PATH\ - $LIBSUITESPARSE_CONFIG_PATH\ - $LIBCHOLMOD_PATH $LIBAMD_PATH\ - $LIBCAMD_PATH $LIBCCOLAMD_PATH\ - $LIBCOLAMD_PATH $LIBSATLAS_PATH\ - $LIBGFORTRAN_PATH $LIBQUADMATH_PATH\ - $LIBMETIS_PATH $LIBLAPACK_PATH\ - $LIBBLAS_PATH) -OS_SO_FILES=() -for lib in "${OS_SO_PATHS[@]}" -do - file_name="${lib##*/}" # Substring removal of path to get filename - OS_SO_FILES[${#OS_SO_FILES[@]}]=$file_name # Append lib to array -done - -# PyTorch-version specific -# AOTriton dependency only for PyTorch >= 2.4 -if (( $(echo "${PYTORCH_VERSION} 2.4" | awk '{print ($1 >= $2)}') )); then - ROCM_SO_FILES+=("libaotriton_v2.so") -fi - -# rocBLAS library files -if [[ $ROCM_INT -ge 50200 ]]; then - ROCBLAS_LIB_SRC=$ROCM_HOME/lib/rocblas/library - ROCBLAS_LIB_DST=lib/rocblas/library -else - ROCBLAS_LIB_SRC=$ROCM_HOME/rocblas/lib/library - ROCBLAS_LIB_DST=lib/library -fi -ARCH=$(echo $PYTORCH_ROCM_ARCH | sed 's/;/|/g') # Replace ; seperated arch list to bar for grep -ARCH_SPECIFIC_FILES=$(ls $ROCBLAS_LIB_SRC | grep -E $ARCH) -OTHER_FILES=$(ls $ROCBLAS_LIB_SRC | grep -v gfx) -ROCBLAS_LIB_FILES=($ARCH_SPECIFIC_FILES $OTHER_FILES) - -# hipblaslt library files -HIPBLASLT_LIB_SRC=$ROCM_HOME/lib/hipblaslt/library -HIPBLASLT_LIB_DST=lib/hipblaslt/library -ARCH_SPECIFIC_FILES=$(ls $HIPBLASLT_LIB_SRC | grep -E $ARCH) -OTHER_FILES=$(ls $HIPBLASLT_LIB_SRC | grep -v gfx) -HIPBLASLT_LIB_FILES=($ARCH_SPECIFIC_FILES $OTHER_FILES) - -# ROCm library files -ROCM_SO_PATHS=() -for lib in "${ROCM_SO_FILES[@]}" -do - file_path=($(find $ROCM_HOME/lib/ -name "$lib")) # First search in lib - if [[ -z $file_path ]]; then - if [ -d "$ROCM_HOME/lib64/" ]; then - file_path=($(find $ROCM_HOME/lib64/ -name "$lib")) # Then search in lib64 - fi - fi - if [[ -z $file_path ]]; then - file_path=($(find $ROCM_HOME/ -name "$lib")) # Then search in ROCM_HOME - fi - if [[ -z $file_path ]]; then - echo "Error: Library file $lib is not found." >&2 - exit 1 - fi - ROCM_SO_PATHS[${#ROCM_SO_PATHS[@]}]="$file_path" # Append lib to array -done - -DEPS_LIST=( - ${ROCM_SO_PATHS[*]} - ${OS_SO_PATHS[*]} -) - -DEPS_SONAME=( - ${ROCM_SO_FILES[*]} - ${OS_SO_FILES[*]} -) - -DEPS_AUX_SRCLIST=( - "${ROCBLAS_LIB_FILES[@]/#/$ROCBLAS_LIB_SRC/}" - "${HIPBLASLT_LIB_FILES[@]/#/$HIPBLASLT_LIB_SRC/}" - "/opt/amdgpu/share/libdrm/amdgpu.ids" -) - -DEPS_AUX_DSTLIST=( - "${ROCBLAS_LIB_FILES[@]/#/$ROCBLAS_LIB_DST/}" - "${HIPBLASLT_LIB_FILES[@]/#/$HIPBLASLT_LIB_DST/}" - "share/libdrm/amdgpu.ids" -) - -if [[ $ROCM_INT -ge 50500 ]]; then - # MIOpen library files - MIOPEN_SHARE_SRC=$ROCM_HOME/share/miopen/db - MIOPEN_SHARE_DST=share/miopen/db - MIOPEN_SHARE_FILES=($(ls $MIOPEN_SHARE_SRC | grep -E $ARCH)) - - DEPS_AUX_SRCLIST+=(${MIOPEN_SHARE_FILES[@]/#/$MIOPEN_SHARE_SRC/}) - DEPS_AUX_DSTLIST+=(${MIOPEN_SHARE_FILES[@]/#/$MIOPEN_SHARE_DST/}) -fi - -if [[ $ROCM_INT -ge 50600 ]]; then - # RCCL library files - if [[ $ROCM_INT -ge 50700 ]]; then - RCCL_SHARE_SRC=$ROCM_HOME/share/rccl/msccl-algorithms - RCCL_SHARE_DST=share/rccl/msccl-algorithms - else - RCCL_SHARE_SRC=$ROCM_HOME/lib/msccl-algorithms - RCCL_SHARE_DST=lib/msccl-algorithms - fi - RCCL_SHARE_FILES=($(ls $RCCL_SHARE_SRC)) - - DEPS_AUX_SRCLIST+=(${RCCL_SHARE_FILES[@]/#/$RCCL_SHARE_SRC/}) - DEPS_AUX_DSTLIST+=(${RCCL_SHARE_FILES[@]/#/$RCCL_SHARE_DST/}) -fi - -echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" - -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -if [[ -z "$BUILD_PYTHONLESS" ]]; then - BUILD_SCRIPT=build_common.sh -else - BUILD_SCRIPT=build_libtorch.sh -fi -source $SCRIPTPATH/${BUILD_SCRIPT} diff --git a/manywheel/build_scripts/build.sh b/manywheel/build_scripts/build.sh deleted file mode 100644 index d7369f800..000000000 --- a/manywheel/build_scripts/build.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# Top-level build script called from Dockerfile - -# Stop at any error, show all commands -set -ex - -# openssl version to build, with expected sha256 hash of .tar.gz -# archive -OPENSSL_ROOT=openssl-1.1.1l -OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 -DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc -PATCHELF_HASH=d9afdff4baeacfbc64861454f368b7f2c15c44d245293f7587bbf726bfe722fb -CURL_ROOT=curl-7.73.0 -CURL_HASH=cf34fe0b07b800f1c01a499a6e8b2af548f6d0e044dca4a29d88a4bee146d131 -AUTOCONF_ROOT=autoconf-2.69 -AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 - -# Get build utilities -MY_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MY_DIR/build_utils.sh - -if [ "$(uname -m)" != "s390x" ] ; then - # Dependencies for compiling Python that we want to remove from - # the final image after compiling Python - PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel" - - # Libraries that are allowed as part of the manylinux1 profile - MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel" - - # Development tools and libraries - yum -y install bzip2 make git patch unzip bison yasm diffutils \ - automake which file cmake28 \ - kernel-devel-`uname -r` \ - ${PYTHON_COMPILE_DEPS} -else - # Dependencies for compiling Python that we want to remove from - # the final image after compiling Python - PYTHON_COMPILE_DEPS="zlib1g-dev libbz2-dev libncurses-dev libsqlite3-dev libdb-dev libpcap-dev liblzma-dev libffi-dev" - - # Libraries that are allowed as part of the manylinux1 profile - MANYLINUX1_DEPS="libglib2.0-dev libX11-dev libncurses-dev" - - # Development tools and libraries - apt install -y bzip2 make git patch unzip diffutils \ - automake which file cmake \ - linux-headers-virtual \ - ${PYTHON_COMPILE_DEPS} -fi - -# Install newest autoconf -build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH -autoconf --version - -# Compile the latest Python releases. -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. We delete openssl afterwards.) -build_openssl $OPENSSL_ROOT $OPENSSL_HASH -/build_scripts/install_cpython.sh - -PY39_BIN=/opt/python/cp39-cp39/bin - -# Our openssl doesn't know how to find the system CA trust store -# (https://github.com/pypa/manylinux/issues/53) -# And it's not clear how up-to-date that is anyway -# So let's just use the same one pip and everyone uses -$PY39_BIN/pip install certifi -ln -s $($PY39_BIN/python -c 'import certifi; print(certifi.where())') \ - /opt/_internal/certs.pem -# If you modify this line you also have to modify the versions in the -# Dockerfiles: -export SSL_CERT_FILE=/opt/_internal/certs.pem - -# Install newest curl -build_curl $CURL_ROOT $CURL_HASH -rm -rf /usr/local/include/curl /usr/local/lib/libcurl* /usr/local/lib/pkgconfig/libcurl.pc -hash -r -curl --version -curl-config --features - -# Install patchelf (latest with unreleased bug fixes) -curl -sLOk https://nixos.org/releases/patchelf/patchelf-0.10/patchelf-0.10.tar.gz -# check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH -tar -xzf patchelf-0.10.tar.gz -(cd patchelf-0.10 && ./configure && make && make install) -rm -rf patchelf-0.10.tar.gz patchelf-0.10 - -# Install latest pypi release of auditwheel -$PY39_BIN/pip install auditwheel -ln -s $PY39_BIN/auditwheel /usr/local/bin/auditwheel - -# Clean up development headers and other unnecessary stuff for -# final image -if [ "$(uname -m)" != "s390x" ] ; then - yum -y erase wireless-tools gtk2 libX11 hicolor-icon-theme \ - avahi freetype bitstream-vera-fonts \ - ${PYTHON_COMPILE_DEPS} || true > /dev/null 2>&1 - yum -y install ${MANYLINUX1_DEPS} - yum -y clean all > /dev/null 2>&1 - yum list installed -else - apt purge -y ${PYTHON_COMPILE_DEPS} || true > /dev/null 2>&1 -fi -# we don't need libpython*.a, and they're many megabytes -find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f -# Strip what we can -- and ignore errors, because this just attempts to strip -# *everything*, including non-ELF files: -find /opt/_internal -type f -print0 \ - | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -# We do not need the Python test suites, or indeed the precompiled .pyc and -# .pyo files. Partially cribbed from: -# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile -find /opt/_internal \ - \( -type d -a -name test -o -name tests \) \ - -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ - -print0 | xargs -0 rm -f - -for PYTHON in /opt/python/*/bin/python; do - # Smoke test to make sure that our Pythons work, and do indeed detect as - # being manylinux compatible: - $PYTHON $MY_DIR/manylinux1-check.py - # Make sure that SSL cert checking works - $PYTHON $MY_DIR/ssl-check.py -done - -# Fix libc headers to remain compatible with C99 compilers. -find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + - -# Now we can delete our built SSL -rm -rf /usr/local/ssl diff --git a/manywheel/build_scripts/build_utils.sh b/manywheel/build_scripts/build_utils.sh deleted file mode 100755 index 548f36c6e..000000000 --- a/manywheel/build_scripts/build_utils.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# Helper utilities for build - -OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/old/1.1.1/ -CURL_DOWNLOAD_URL=https://curl.askapache.com/download - -AUTOCONF_DOWNLOAD_URL=https://ftp.gnu.org/gnu/autoconf - - -function check_var { - if [ -z "$1" ]; then - echo "required variable not defined" - exit 1 - fi -} - - -function do_openssl_build { - ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null - make > /dev/null - make install > /dev/null -} - - -function check_sha256sum { - local fname=$1 - check_var ${fname} - local sha256=$2 - check_var ${sha256} - - echo "${sha256} ${fname}" > ${fname}.sha256 - sha256sum -c ${fname}.sha256 - rm -f ${fname}.sha256 -} - - -function build_openssl { - local openssl_fname=$1 - check_var ${openssl_fname} - local openssl_sha256=$2 - check_var ${openssl_sha256} - check_var ${OPENSSL_DOWNLOAD_URL} - curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz - check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256} - tar -xzf ${openssl_fname}.tar.gz - (cd ${openssl_fname} && do_openssl_build) - rm -rf ${openssl_fname} ${openssl_fname}.tar.gz -} - - -function do_curl_build { - LIBS=-ldl ./configure --with-ssl --disable-shared > /dev/null - make > /dev/null - make install > /dev/null -} - - -function build_curl { - local curl_fname=$1 - check_var ${curl_fname} - local curl_sha256=$2 - check_var ${curl_sha256} - check_var ${CURL_DOWNLOAD_URL} - curl -sLO ${CURL_DOWNLOAD_URL}/${curl_fname}.tar.bz2 - check_sha256sum ${curl_fname}.tar.bz2 ${curl_sha256} - tar -jxf ${curl_fname}.tar.bz2 - (cd ${curl_fname} && do_curl_build) - rm -rf ${curl_fname} ${curl_fname}.tar.bz2 -} - - -function do_standard_install { - ./configure > /dev/null - make > /dev/null - make install > /dev/null -} - - -function build_autoconf { - local autoconf_fname=$1 - check_var ${autoconf_fname} - local autoconf_sha256=$2 - check_var ${autoconf_sha256} - check_var ${AUTOCONF_DOWNLOAD_URL} - curl -sLO ${AUTOCONF_DOWNLOAD_URL}/${autoconf_fname}.tar.gz - check_sha256sum ${autoconf_fname}.tar.gz ${autoconf_sha256} - tar -zxf ${autoconf_fname}.tar.gz - (cd ${autoconf_fname} && do_standard_install) - rm -rf ${autoconf_fname} ${autoconf_fname}.tar.gz -} diff --git a/manywheel/build_scripts/manylinux1-check.py b/manywheel/build_scripts/manylinux1-check.py deleted file mode 100644 index 7cb62e0c0..000000000 --- a/manywheel/build_scripts/manylinux1-check.py +++ /dev/null @@ -1,53 +0,0 @@ -# Logic copied from PEP 513 - -def is_manylinux1_compatible(): - # Only Linux, and only x86-64 / i686 - from distutils.util import get_platform - if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x"]: - return False - - # Check for presence of _manylinux module - try: - import _manylinux - return bool(_manylinux.manylinux1_compatible) - except (ImportError, AttributeError): - # Fall through to heuristic check below - pass - - # Check glibc version. CentOS 5 uses glibc 2.5. - return have_compatible_glibc(2, 5) - -def have_compatible_glibc(major, minimum_minor): - import ctypes - - process_namespace = ctypes.CDLL(None) - try: - gnu_get_libc_version = process_namespace.gnu_get_libc_version - except AttributeError: - # Symbol doesn't exist -> therefore, we are not linked to - # glibc. - return False - - # Call gnu_get_libc_version, which returns a string like "2.5". - gnu_get_libc_version.restype = ctypes.c_char_p - version_str = gnu_get_libc_version() - # py2 / py3 compatibility: - if not isinstance(version_str, str): - version_str = version_str.decode("ascii") - - # Parse string and check against requested version. - version = [int(piece) for piece in version_str.split(".")] - assert len(version) == 2 - if major != version[0]: - return False - if minimum_minor > version[1]: - return False - return True - -import sys -if is_manylinux1_compatible(): - print("%s is manylinux1 compatible" % (sys.executable,)) - sys.exit(0) -else: - print("%s is NOT manylinux1 compatible" % (sys.executable,)) - sys.exit(1) diff --git a/manywheel/build_scripts/ssl-check.py b/manywheel/build_scripts/ssl-check.py deleted file mode 100644 index b91927173..000000000 --- a/manywheel/build_scripts/ssl-check.py +++ /dev/null @@ -1,33 +0,0 @@ -# cf. https://github.com/pypa/manylinux/issues/53 - -GOOD_SSL = "https://google.com" -BAD_SSL = "https://self-signed.badssl.com" - -import sys - -print("Testing SSL certificate checking for Python:", sys.version) - -if (sys.version_info[:2] < (2, 7) - or sys.version_info[:2] < (3, 4)): - print("This version never checks SSL certs; skipping tests") - sys.exit(0) - -if sys.version_info[0] >= 3: - from urllib.request import urlopen - EXC = OSError -else: - from urllib import urlopen - EXC = IOError - -print("Connecting to %s should work" % (GOOD_SSL,)) -urlopen(GOOD_SSL) -print("...it did, yay.") - -print("Connecting to %s should fail" % (BAD_SSL,)) -try: - urlopen(BAD_SSL) - # If we get here then we failed: - print("...it DIDN'T!!!!!11!!1one!") - sys.exit(1) -except EXC: - print("...it did, yay.") \ No newline at end of file diff --git a/manywheel/test_wheel.sh b/manywheel/test_wheel.sh deleted file mode 100755 index ada7d93f0..000000000 --- a/manywheel/test_wheel.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -yum install -y wget git - -rm -rf /usr/local/cuda* - -# Install Anaconda -if ! ls /py -then - echo "Miniconda needs to be installed" - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh - bash ~/miniconda.sh -b -p /py -else - echo "Miniconda is already installed" -fi - -export PATH="/py/bin:$PATH" - -# Anaconda token -if ls /remote/token -then - source /remote/token -fi - -conda install -y conda-build anaconda-client -