build: RHEL8 Python Backend

build.py

@@ -78,6 +78,7 @@
         "2024.0.0",  # Standalone OpenVINO
         "3.2.6",  # DCGM version
         "0.5.3.post1",  # vLLM version
+        "3.12.1",  # RHEL Python version
     )
 }
 
@@ -950,7 +951,6 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap):
             libb64-devel \\
             gperftools-devel \\
             patchelf \\
-            python3.11-devel \\
             python3-pip \\
             python3-setuptools \\
             rapidjson-devel \\
@@ -963,6 +963,10 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap):
             libxml2-devel \\
             numactl-devel \\
             wget
+"""
+    # Requires openssl-devel to be installed first for pyenv build to be successful
+    df += change_default_python_version_rhel(TRITON_VERSION_MAP[FLAGS.version][7])
+    df += """
 
 RUN pip3 install --upgrade pip \\
       && pip3 install --upgrade \\
@@ -1389,7 +1393,29 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
 
     # Add dependencies needed for python backend
     if "python" in backends:
-        df += """
+        if target_platform() == "rhel":
+            df += """
+# python3, python3-pip and some pip installs required for the python backend
+RUN yum install -y \\
+        libarchive-devel \\
+        python3-pip \\
+        openssl-devel \\
+        readline-devel
+"""
+            # Requires openssl-devel to be installed first for pyenv build to be successful
+            df += change_default_python_version_rhel(
+                TRITON_VERSION_MAP[FLAGS.version][7]
+            )
+            df += """
+RUN pip3 install --upgrade pip \\
+    && pip3 install --upgrade \\
+        wheel \\
+        setuptools \\
+        \"numpy<2\" \\
+        virtualenv
+"""
+        else:
+            df += """
 # python3, python3-pip and some pip installs required for the python backend
 RUN apt-get update \\
       && apt-get install -y --no-install-recommends \\
@@ -1514,6 +1540,37 @@ def add_cpu_libs_to_linux_dockerfile(backends, target_machine):
     return df
 
 
+def change_default_python_version_rhel(version):
+    # Example version 3.12.1 --> 3.12
+    major_minor_version = ".".join(version.split(".")[:2])
+    df = """
+# RHEL image has several python versions. It's important
+# to set the correct version, otherwise, packages that are
+# pip installed will not be found during testing.
+ENV PYVER {}
+ENV PYTHONPATH /opt/python/v
+RUN ln -sf /opt/python/cp${{PYVER/./}}* ${{PYTHONPATH}}
+ENV PYBIN ${{PYTHONPATH}}/bin
+ENV PYTHON_BIN_PATH ${{PYBIN}}/python${{PYVER}}
+ENV PATH ${{PYBIN}}:${{PATH}}
+""".format(
+        major_minor_version
+    )
+    df += """
+# The python library version available for install via 'yum install python3.X-devel' does not
+# match the version of python inside the RHEL base container. This means that python packages
+# installed within the container will not be picked up by the python backend stub process pybind
+# bindings. It must instead must be installed via pyenv.
+RUN curl https://pyenv.run | bash
+ENV PATH="/root/.pyenv/bin:$PATH"
+RUN eval "$(pyenv init -)"
+RUN CONFIGURE_OPTS=\"--with-openssl=/usr/lib64\" && pyenv install {} \\
+    && cp /root/.pyenv/versions/{}/lib/libpython3* /usr/lib64/""".format(
+        version, version
+    )
+    return df
+
+
 def create_dockerfile_windows(
     ddir, dockerfile_name, argmap, backends, repoagents, caches
 ):
@@ -1956,6 +2013,19 @@ def backend_build(
     cmake_script.mkdir(os.path.join(install_dir, "backends"))
     cmake_script.rmdir(os.path.join(install_dir, "backends", be))
 
+    # The python library version available for install via 'yum install python3.X-devel' does not
+    # match the version of python inside the RHEL base container. This means that python packages
+    # installed within the container will not be picked up by the python backend stub process pybind
+    # bindings. It must instead must be installed via pyenv. We package it here for better usability.
+    if target_platform() == "rhel" and be == "python":
+        major_minor_version = ".".join(
+            (TRITON_VERSION_MAP[FLAGS.version][7]).split(".")[:2]
+        )
+        version_matched_files = "/usr/lib64/libpython" + major_minor_version + "*"
+        cmake_script.cp(
+            version_matched_files, os.path.join(repo_install_dir, "backends", be)
+        )
+
     cmake_script.cpdir(
         os.path.join(repo_install_dir, "backends", be),
         os.path.join(install_dir, "backends"),