From b121fe690f79c83976a592820d3a7eb8b933724b Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Fri, 10 May 2024 12:50:47 -0400 Subject: [PATCH] Add manylinux_2_28 image (#1816) * Add manylinux_2_28 image --- .github/workflows/build-manywheel-images.yml | 15 ++ manywheel/Dockerfile_2_28 | 143 +++++++++++++++++++ manywheel/build_docker.sh | 8 ++ 3 files changed, 166 insertions(+) create mode 100644 manywheel/Dockerfile_2_28 diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml index a599635f8..f308edcac 100644 --- a/.github/workflows/build-manywheel-images.yml +++ b/.github/workflows/build-manywheel-images.yml @@ -107,6 +107,21 @@ jobs: - name: Build Docker Image run: | manywheel/build_docker.sh + build-docker-cpu-manylinux_2_28: + runs-on: ubuntu-22.04 + env: + GPU_ARCH_TYPE: cpu-manylinux_2_28 + steps: + - name: Checkout PyTorch + uses: actions/checkout@v3 + - name: Authenticate if WITH_PUSH + run: | + if [[ "${WITH_PUSH}" == true ]]; then + echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin + fi + - name: Build Docker Image + run: | + manywheel/build_docker.sh build-docker-cpu-aarch64: runs-on: linux.arm64.2xlarge env: diff --git a/manywheel/Dockerfile_2_28 b/manywheel/Dockerfile_2_28 new file mode 100644 index 000000000..6566f115d --- /dev/null +++ b/manywheel/Dockerfile_2_28 @@ -0,0 +1,143 @@ +# syntax = docker/dockerfile:experimental +ARG ROCM_VERSION=3.7 +ARG BASE_CUDA_VERSION=11.8 +ARG GPU_IMAGE=amd64/almalinux:8 +FROM quay.io/pypa/manylinux_2_28_x86_64 as base + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +ARG DEVTOOLSET_VERSION=11 +RUN yum install -y wget curl perl util-linux xz bzip2 git patch which perl zlib-devel yum-utils gcc-toolset-${DEVTOOLSET_VERSION}-toolchain +ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH + +# cmake-3.18.4 from pip +RUN yum install -y python3-pip && \ + python3 -mpip install cmake==3.18.4 && \ + ln -s /usr/local/bin/cmake /usr/bin/cmake + +FROM base as openssl +# Install openssl (this must precede `build python` step) +# (In order to have a proper SSL module, Python is compiled +# against a recent openssl [see env vars above], which is linked +# statically. We delete openssl afterwards.) +ADD ./common/install_openssl.sh install_openssl.sh +RUN bash ./install_openssl.sh && rm install_openssl.sh + + +# remove unncessary python versions +RUN rm -rf /opt/python/cp26-cp26m /opt/_internal/cpython-2.6.9-ucs2 +RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4 +RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6 +RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6 + +FROM base as cuda +ARG BASE_CUDA_VERSION=11.8 +# Install CUDA +ADD ./common/install_cuda.sh install_cuda.sh +RUN bash ./install_cuda.sh ${BASE_CUDA_VERSION} && rm install_cuda.sh + +FROM base as intel +# MKL +ADD ./common/install_mkl.sh install_mkl.sh +RUN bash ./install_mkl.sh && rm install_mkl.sh + +FROM base as magma +ARG BASE_CUDA_VERSION=10.2 +# Install magma +ADD ./common/install_magma.sh install_magma.sh +RUN bash ./install_magma.sh ${BASE_CUDA_VERSION} && rm install_magma.sh + +FROM base as jni +# Install java jni header +ADD ./common/install_jni.sh install_jni.sh +ADD ./java/jni.h jni.h +RUN bash ./install_jni.sh && rm install_jni.sh + +FROM base as libpng +# Install libpng +ADD ./common/install_libpng.sh install_libpng.sh +RUN bash ./install_libpng.sh && rm install_libpng.sh + +FROM ${GPU_IMAGE} as common +ARG DEVTOOLSET_VERSION=11 +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +RUN yum -y install epel-release +RUN yum -y update +RUN yum install -y \ + autoconf \ + automake \ + bison \ + bzip2 \ + curl \ + diffutils \ + file \ + git \ + make \ + patch \ + perl \ + unzip \ + util-linux \ + wget \ + which \ + xz \ + gcc-toolset-${DEVTOOLSET_VERSION}-toolchain + +RUN yum install -y \ + https://repo.ius.io/ius-release-el7.rpm \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +RUN yum swap -y git git236-core +# git236+ would refuse to run git commands in repos owned by other users +# Which causes version check to fail, as pytorch repo is bind-mounted into the image +# Override this behaviour by treating every folder as safe +# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327 +RUN git config --global --add safe.directory "*" + +ENV SSL_CERT_FILE=/opt/_internal/certs.pem +# Install LLVM version +COPY --from=openssl /opt/openssl /opt/openssl +COPY --from=base /opt/python /opt/python +COPY --from=base /opt/_internal /opt/_internal +COPY --from=base /usr/local/bin/auditwheel /usr/local/bin/auditwheel +COPY --from=intel /opt/intel /opt/intel +COPY --from=base /usr/local/bin/patchelf /usr/local/bin/patchelf +COPY --from=libpng /usr/local/bin/png* /usr/local/bin/ +COPY --from=libpng /usr/local/bin/libpng* /usr/local/bin/ +COPY --from=libpng /usr/local/include/png* /usr/local/include/ +COPY --from=libpng /usr/local/include/libpng* /usr/local/include/ +COPY --from=libpng /usr/local/lib/libpng* /usr/local/lib/ +COPY --from=libpng /usr/local/lib/pkgconfig /usr/local/lib/pkgconfig +COPY --from=jni /usr/local/include/jni.h /usr/local/include/jni.h + +FROM common as cpu_final +ARG BASE_CUDA_VERSION=11.8 +ARG DEVTOOLSET_VERSION=11 +# Ensure the expected devtoolset is used +ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH +# cmake +RUN yum install -y cmake3 && \ + ln -s /usr/bin/cmake3 /usr/bin/cmake + + +FROM cpu_final as cuda_final +RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION} +COPY --from=cuda /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} +COPY --from=magma /usr/local/cuda-${BASE_CUDA_VERSION} /usr/local/cuda-${BASE_CUDA_VERSION} + +FROM common as rocm_final +ARG ROCM_VERSION=3.7 +# Install ROCm +ADD ./common/install_rocm.sh install_rocm.sh +RUN bash ./install_rocm.sh ${ROCM_VERSION} && rm install_rocm.sh +# cmake is already installed inside the rocm base image, but both 2 and 3 exist +# cmake3 is needed for the later MIOpen custom build, so that step is last. +RUN yum install -y cmake3 && \ + rm -f /usr/bin/cmake && \ + ln -s /usr/bin/cmake3 /usr/bin/cmake +ADD ./common/install_miopen.sh install_miopen.sh +RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh diff --git a/manywheel/build_docker.sh b/manywheel/build_docker.sh index 4d3816588..05f3dad81 100755 --- a/manywheel/build_docker.sh +++ b/manywheel/build_docker.sh @@ -20,6 +20,14 @@ case ${GPU_ARCH_TYPE} in GPU_IMAGE=centos:7 DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9" ;; + cpu-manylinux_2_28) + TARGET=cpu_final + DOCKER_TAG=cpu + LEGACY_DOCKER_IMAGE=${DOCKER_REGISTRY}/pytorch/manylinux_2_28-cpu + GPU_IMAGE=amd64/almalinux:8 + DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11" + MANY_LINUX_VERSION="2_28" + ;; cpu-aarch64) TARGET=final DOCKER_TAG=cpu-aarch64