diff --git a/aarch64_linux/aarch64_ci_setup.sh b/aarch64_linux/aarch64_ci_setup.sh
index 87ed70a72d..3114815d53 100755
--- a/aarch64_linux/aarch64_ci_setup.sh
+++ b/aarch64_linux/aarch64_ci_setup.sh
@@ -2,35 +2,38 @@
 set -eux -o pipefail
 
 # This script is used to prepare the Docker container for aarch64_ci_wheel_build.py python script
-# as we need to install conda and setup the python version for the build.
-
-CONDA_PYTHON_EXE=/opt/conda/bin/python
-CONDA_EXE=/opt/conda/bin/conda
-CONDA_ENV_NAME=aarch64_env
-PATH=/opt/conda/bin:$PATH
-LD_LIBRARY_PATH=/opt/conda/envs/${CONDA_ENV_NAME}/lib/:/opt/conda/lib:$LD_LIBRARY_PATH
-
-###############################################################################
-# Install conda
-# disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path"
-# when using Python version, less than the conda latest
-###############################################################################
-echo 'Installing conda-forge'
-curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh
-chmod +x /mambaforge.sh
-/mambaforge.sh -b -p /opt/conda
-rm /mambaforge.sh
-source /opt/conda/etc/profile.d/conda.sh
-conda config --set ssl_verify False
-conda create -y -c conda-forge -n "${CONDA_ENV_NAME}" python=${DESIRED_PYTHON}
-conda activate "${CONDA_ENV_NAME}"
+# as we need to setup the required python version and install few tools.
 
+dnf update
+
+if [ "$DESIRED_PYTHON" == "3.10" ]; then
+   dnf install -y epel-release
+   dnf install -y gcc openssl-devel bzip2-devel libffi-devel zlib-devel wget make
+   wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz
+   tar xzf Python-3.10.14.tgz
+   cd Python-3.10.14
+   ./configure --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions
+   make -j 8
+   make altinstall
+   rm ../Python-3.10.14.tgz
+elif [ "$DESIRED_PYTHON" == "3.8" ] || [ "$DESIRED_PYTHON" == "3.9" ] || [ "$DESIRED_PYTHON" == "3.11" ]; then
+    dnf install -y python$DESIRED_PYTHON
+else
+   echo "unsupported python version passed. 3.8, 3.9, 3.10 or 3.11 are the only supported versions"
+   exit 1
+fi
+
+/usr/local/bin/python$DESIRED_PYTHON -m venv appenv
+
+source appenv/bin/activate
+
+python3 --version
+
+dnf install -y python3-devel python3-pip
+
+pip3 install dataclasses typing-extensions scons pyyaml pygit2 ninja patchelf Cython
 if [[ "$DESIRED_PYTHON"  == "3.8" ]]; then
     pip install -q numpy==1.24.4
 else
     pip install -q --pre numpy==2.0.0rc1
 fi
-conda install -y -c conda-forge pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 openblas==0.3.25=*openmp* ninja==1.11.1 scons==4.5.2
-
-python --version
-conda --version
diff --git a/aarch64_linux/aarch64_wheel_ci_build.py b/aarch64_linux/aarch64_wheel_ci_build.py
index c474184edd..eef3ef10c0 100755
--- a/aarch64_linux/aarch64_wheel_ci_build.py
+++ b/aarch64_linux/aarch64_wheel_ci_build.py
@@ -15,6 +15,21 @@ def list_dir(path: str) -> List[str]:
     return check_output(["ls", "-1", path]).decode().split("\n")
 
 
+def build_OpenBLAS() -> None:
+    '''
+    Building OpenBLAS, because the package in many linux is old
+    '''
+    print('Building OpenBLAS')
+    openblas_build_flags=["NUM_THREADS=128", "USE_OPENMP=1", "DYNAMIC_ARCH=1", "TARGET=ARMV8", "CFLAGS=-O3"]
+    openblas_checkout_dir="OpenBLAS"
+
+    check_call(["git", "clone", "https://github.com/OpenMathLib/OpenBLAS.git", "-b", "v0.3.25",
+                "--depth", "1", "--shallow-submodules"])
+
+    check_call(["make", "-j8"]  + openblas_build_flags,
+               cwd=openblas_checkout_dir)
+    check_call(["make", "-j8"] + openblas_build_flags + ["install"], cwd=openblas_checkout_dir)
+
 def build_ArmComputeLibrary() -> None:
     '''
     Using ArmComputeLibrary for aarch64 PyTorch
@@ -27,7 +42,7 @@ def build_ArmComputeLibrary() -> None:
     os.makedirs(acl_install_dir)
     check_call(["git", "clone", "https://github.com/ARM-software/ComputeLibrary.git", "-b", "v23.08",
                 "--depth", "1", "--shallow-submodules"])
-    check_call(["scons", "Werror=1", "-j8", f"build_dir=/{acl_install_dir}/build"] + acl_build_flags,
+    check_call(["scons", "Werror=0", "-j8", f"build_dir=/{acl_install_dir}/build"] + acl_build_flags,
                cwd=acl_checkout_dir)
     for d in ["arm_compute", "include", "utils", "support", "src"]:
         shutil.copytree(f"{acl_checkout_dir}/{d}", f"{acl_install_dir}/{d}")
@@ -95,6 +110,7 @@ def parse_arguments():
     elif branch.startswith(("v1.", "v2.")):
         build_vars += f"BUILD_TEST=0 PYTORCH_BUILD_VERSION={branch[1:branch.find('-')]} PYTORCH_BUILD_NUMBER=1 "
 
+    build_OpenBLAS()
     if enable_mkldnn:
         build_ArmComputeLibrary()
         print("build pytorch with mkldnn+acl backend")
@@ -102,7 +118,9 @@ def parse_arguments():
                       "ACL_ROOT_DIR=/acl " \
                       "LD_LIBRARY_PATH=/pytorch/build/lib:/acl/build:$LD_LIBRARY_PATH " \
                       "ACL_INCLUDE_DIR=/acl/build " \
-                      "ACL_LIBRARY=/acl/build "
+                      "ACL_LIBRARY=/acl/build " \
+                      "BLAS=OpenBLAS " \
+                      "OpenBLAS_HOME=/OpenBLAS "
     else:
         print("build pytorch without mkldnn backend")