-
Notifications
You must be signed in to change notification settings - Fork 35
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
6 changed files
with
294 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| name: Multipy runtime nightly release with cuda | ||
|
|
||
| on: | ||
| schedule: | ||
| - cron: '0 2 * * *' # run at 2 AM UTC | ||
| pull_request: | ||
| push: | ||
| branches: | ||
| - main | ||
|
|
||
| jobs: | ||
| unittest: | ||
| strategy: | ||
| matrix: | ||
| python-version: [3.8] | ||
| platform: [linux.4xlarge.nvidia.gpu] | ||
| abi: [0,1] | ||
| fail-fast: false | ||
| runs-on: ${{ matrix.platform }} | ||
| steps: | ||
| - name: Update pip | ||
| run: | | ||
| sudo yum update -y | ||
| sudo yum -y install git python3-pip | ||
| sudo pip3 install --upgrade pip | ||
| - name: Setup conda | ||
| run: | | ||
| wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh | ||
| bash ~/miniconda.sh -b -p $HOME/miniconda | ||
| - name: setup Path | ||
| run: | | ||
| echo /usr/local/cuda-11.3/bin >> $GITHUB_PATH | ||
| echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH | ||
| echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH | ||
| - name: Checkout MultiPy | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| submodules: true | ||
|
|
||
| - name: Setup SSH (Click me for login details) | ||
| uses: ./.github/actions/setup-ssh | ||
| with: | ||
| github-secret: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Install C++ toolchain | ||
| run: | | ||
| sudo yum -y install clang llvm | ||
| export CC=clang | ||
| export CXX=clang++ | ||
| sudo yum -y install xz-devel bzip2-devel libnsl2-devel readline-devel expat-devel gdbm-devel glibc-devel gmp-devel libffi-devel libGL-devel libX11-devel ncurses-devel openssl-devel sqlite-devel tcl-devel tix-devel tk-devel | ||
| sudo yum -y install lzma | ||
| sudo yum -y install uuid | ||
| sudo yum -y install openmpi-devel | ||
| sudo yum -y install zlib-devel | ||
| - name: create conda env | ||
| run: | | ||
| conda create --name multipy_runtime_env python=${{ matrix.python-version }} | ||
| conda info | ||
| - name: Install python/pytorch dependencies | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| conda run -n multipy_runtime_env python -m pip install astunparse numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_extensions future six requests dataclasses pytest | ||
| - name: gen examples | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| # A minor hack to get the CI working as conda doesn't have torch, | ||
| # fortunately we can remove this once we have a dynamically linked torch | ||
| cd multipy/runtime/example | ||
| conda create --name example_env python=${{ matrix.python-version }} | ||
| conda run -n example_env python -m pip install torch torchvision torchaudio pathlib | ||
| conda run -n example_env python generate_examples.py | ||
| - name: Build pytorch with ABI=${{ matrix.abi }} | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| export GLIBCXX_USE_CXX11_ABI=${{ matrix.abi }} | ||
| export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=${{ matrix.abi }}" | ||
| export TORCH_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${{ matrix.abi }}" | ||
| cd multipy/runtime/third-party/pytorch | ||
| export USE_DEPLOY=1 | ||
| export USE_CUDA=1 | ||
| conda run -n multipy_runtime_env python setup.py develop | ||
| - name: Build multipy runtime with ABI=${{ matrix.abi }} | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime | ||
| mkdir build | ||
| cd build | ||
| conda run -n multipy_runtime_env cmake -DABI_EQUALS_1=${{ matrix.abi }} .. | ||
| conda run -n multipy_runtime_env cmake --build . --config Release | ||
| - name: install files | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime/build | ||
| conda run -n multipy_runtime_env cmake --install . --prefix "." | ||
| - name: Run unit tests with ABI=${{ matrix.abi }} | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime/build | ||
| ./test_deploy | ||
| - name: Run unit tests with ABI=${{ matrix.abi }} with gpu | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime/build | ||
| ./test_deploy_gpu | ||
| - name: create tarball [click me to get a list of files for the nightly release] | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime/build/dist | ||
| tar -czvf multipy_runtime.tar.gz multipy/ | ||
| - name: Update nightly release | ||
| uses: pyTooling/Actions/releaser@main | ||
| with: | ||
| tag: nightly-runtime-cuda-abi-${{ matrix.abi }} | ||
| rm: true | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| files: | | ||
| multipy/runtime/build/dist/multipy_runtime.tar.gz |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| name: Multipy runtime tests with cuda | ||
|
|
||
| on: | ||
| pull_request: | ||
| push: | ||
| branches: | ||
| - main | ||
|
|
||
| jobs: | ||
| unittest: | ||
| strategy: | ||
| matrix: | ||
| python-version: [3.7, 3.8, 3.9] | ||
| platform: [linux.4xlarge.nvidia.gpu] | ||
| abi: [0,1] | ||
| fail-fast: false | ||
| runs-on: ${{ matrix.platform }} | ||
| steps: | ||
| - name: Update pip | ||
| run: | | ||
| sudo yum update -y | ||
| sudo yum -y install git python3-pip | ||
| sudo pip3 install --upgrade pip | ||
| - name: Setup conda | ||
| run: | | ||
| wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh | ||
| bash ~/miniconda.sh -b -p $HOME/miniconda | ||
| - name: Install CUDA 11.3 | ||
| shell: bash | ||
| run: | | ||
| sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm | ||
| sudo yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo | ||
| sudo yum clean expire-cache | ||
| sudo yum install -y nvidia-driver-latest-dkms | ||
| sudo yum install -y cuda-11-3 | ||
| sudo yum install -y cuda-drivers | ||
| sudo yum install -y libcudnn8-devel | ||
| - name: setup Path | ||
| run: | | ||
| echo /usr/local/cuda-11.3/bin >> $GITHUB_PATH | ||
| echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH | ||
| echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH | ||
| - name: Checkout MultiPy | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| submodules: true | ||
|
|
||
| - name: Setup SSH (Click me for login details) | ||
| uses: ./.github/actions/setup-ssh | ||
| with: | ||
| github-secret: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Install C++ toolchain | ||
| run: | | ||
| sudo yum -y install clang llvm | ||
| export CC=clang | ||
| export CXX=clang++ | ||
| sudo yum -y install xz-devel bzip2-devel libnsl2-devel readline-devel expat-devel gdbm-devel glibc-devel gmp-devel libffi-devel libGL-devel libX11-devel ncurses-devel openssl-devel sqlite-devel tcl-devel tix-devel tk-devel | ||
| sudo yum -y install lzma | ||
| sudo yum -y install uuid | ||
| sudo yum -y install openmpi-devel | ||
| sudo yum -y install zlib-devel | ||
| - name: create conda env | ||
| run: | | ||
| conda create --name multipy_runtime_env python=${{ matrix.python-version }} | ||
| conda info | ||
| - name: Install python/pytorch dependencies | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| conda run -n multipy_runtime_env python -m pip install astunparse numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_extensions future six requests dataclasses pytest | ||
| - name: gen examples | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| # A minor hack to get the CI working as conda doesn't have torch, | ||
| # fortunately we can remove this once we have a dynamically linked torch | ||
| cd multipy/runtime/example | ||
| conda create --name example_env python=${{ matrix.python-version }} | ||
| conda run -n example_env python -m pip install torch torchvision torchaudio pathlib | ||
| conda run -n example_env python generate_examples.py | ||
| - name: Build pytorch with ABI=${{ matrix.abi }} | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| export GLIBCXX_USE_CXX11_ABI=${{ matrix.abi }} | ||
| export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=${{ matrix.abi }}" | ||
| export TORCH_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${{ matrix.abi }}" | ||
| cd multipy/runtime/third-party/pytorch | ||
| export USE_DEPLOY=1 | ||
| export USE_CUDA=1 | ||
| conda run -n multipy_runtime_env python setup.py develop | ||
| - name: Build multipy runtime with ABI=${{ matrix.abi }} | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime | ||
| mkdir build | ||
| cd build | ||
| conda run -n multipy_runtime_env cmake -DABI_EQUALS_1=${{ matrix.abi }} .. | ||
| conda run -n multipy_runtime_env cmake --build . --config Release | ||
| - name: install files | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime/build | ||
| conda run -n multipy_runtime_env cmake --install . --prefix "." | ||
| - name: Run unit tests with ABI=${{ matrix.abi }} | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime/build | ||
| ./test_deploy | ||
| - name: Run unit tests with ABI=${{ matrix.abi }} with gpu | ||
| shell: bash -l {0} | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: | | ||
| cd multipy/runtime/build | ||
| ./test_deploy_gpu |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters