From 06d6b28b4a60cad8aa18c3f2dca584f13beca669 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Mar 2023 11:25:25 -0800 Subject: [PATCH 1/6] Try remove setuptools --- packaging/torchdata/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/torchdata/meta.yaml b/packaging/torchdata/meta.yaml index c1619c86f..b443ed638 100644 --- a/packaging/torchdata/meta.yaml +++ b/packaging/torchdata/meta.yaml @@ -14,7 +14,7 @@ requirements: # - {{ compiler('c') }} # [win] # - {{ compiler('cxx') }} # [win] - python - - setuptools + # - setuptools - cpuonly - curl # [not win] - openssl # [unix] From f4edfe74c7b89a250c303398dad185f7b7aa47aa Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Mar 2023 11:42:17 -0800 Subject: [PATCH 2/6] Try to repro issue --- .github/workflows/_build_test_upload.yml | 444 +---------------------- 1 file changed, 3 insertions(+), 441 deletions(-) diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index 06e1a7f8d..0cbe4ad80 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -1,34 +1,7 @@ name: Build, Test and Upload Wheel on: - workflow_call: - inputs: - branch: - required: true - type: string - pre_dev_release: - required: true - type: boolean - pytorch_version: - required: true - type: string - do-upload: - required: false - default: true - type: boolean - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: - required: true - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: - required: true - PYPI_TOKEN: - required: false - CONDA_PYTORCHBOT_TOKEN: - required: false - CONDA_TEST_PYTORCHBOT_TOKEN: - required: false - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: - required: false + pull_request: jobs: get_release_type: @@ -38,262 +11,11 @@ jobs: steps: - name: Get Release Type run: | - if [[ "${{ inputs.branch }}" == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then - RELEASE_TYPE=official - elif [[ "${{ inputs.branch }}" == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then - RELEASE_TYPE=test - else - if [[ "${{ github.base_ref }}" == release/* ]]; then - RELEASE_TYPE=test - else - RELEASE_TYPE=nightly - fi - fi + RELEASE_TYPE=test echo "Release Type: $RELEASE_TYPE" echo "type=$RELEASE_TYPE" >> $GITHUB_OUTPUT id: get_release_type - wheel_build_test: - needs: get_release_type - runs-on: ${{ matrix.os }} - container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - - macos-m1-12 - python-version: - - 3.8 - - 3.9 - - "3.10" - - "3.11" - - pure - exclude: - - os: macos-latest - python-version: pure - - os: windows-latest - python-version: pure - - os: macos-m1-12 - python-version: pure - steps: - - name: Checkout Source Repository - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Setup Python ${{ matrix.python-version }} for Windows - if: ${{ startsWith( matrix.os, 'windows' ) }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Add temp runner environment variables - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}" - - name: Install Conda on MacOS - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - mkdir -p "${MINICONDA_INSTALL_PATH_MACOS}" - if ${{ startsWith( matrix.os, 'macos-m1' ) }}; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - else - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - fi - bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}" - rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> "$GITHUB_PATH" - - name: Setup Python ${{ matrix.python-version }} on MacOS - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - conda init bash - conda create -y --name wheel_build_env python=${{ matrix.python-version }} - - name: Setup msbuild on Windows - if: startsWith( matrix.os, 'windows' ) - uses: microsoft/setup-msbuild@v1.1 - - name: Set up Visual Studio shell - if: startsWith( matrix.os, 'windows' ) - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - name: Install Build Dependency - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version != 'pure' }}; then - source packaging/manylinux/python_helper.sh - yum -y install ninja-build zlib-static - # Docker path is /__w by default - export WORKSPACE="/__w" - # Install static OpenSSL/libcrypto library - ./packaging/manylinux/install_openssl_curl.sh - fi - else - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - fi - pip install cmake ninja - echo "/home/runner/.local/bin" >> $GITHUB_PATH - fi - - name: Install PyTorch and Build TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} - BUILD_S3: ${{ matrix.python-version != 'pure' }} - run: | - set -ex - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - # Use Python 3.8 for pure python build - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - # Docker path is /__w by default - export WORKSPACE="/__w" - # See: https://github.com/actions/checkout/issues/760 - git config --global --add safe.directory "$WORKSPACE/data/data" - if ${{ matrix.python-version != 'pure' }}; then - # AWSSDK uses $CMAKE_PREFIX_PATH to find openssl - export OPENSSL_ROOT_DIR="$WORKSPACE/ssl" - export CURL_ROOT_DIR="$WORKSPACE/curl" - export CMAKE_PREFIX_PATH="$OPENSSL_ROOT_DIR:$CURL_ROOT_DIR:$CMAKE_PREFIX_PATH" - export STATIC_DEPS=TRUE - fi - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - fi - packaging/build_wheel.sh - - name: Validate TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - pip3 install auditwheel - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - fi - pip3 install pkginfo - for pkg in dist/torchdata*.whl; do - echo "PkgInfo of $pkg:" - pkginfo $pkg - if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then - auditwheel show $pkg - auditwheel repair $pkg --plat manylinux2014_x86_64 -w wheelhouse - fi - done - if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then - mv dist temp - mv wheelhouse dist - fi - - name: Install TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - fi - pip3 install dist/torchdata*.whl - - name: Run Smoke Tests - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env - fi - if ${{ matrix.python-version == 'pure' }}; then - python test/smoke_test/smoke_test.py --no-s3 - else - python test/smoke_test/smoke_test.py - fi - - name: Upload Wheels to Github - if: always() - uses: actions/upload-artifact@v3 - with: - name: torchdata-artifact - path: dist/torchdata*.whl - - wheel_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, wheel_build_test] - runs-on: ubuntu-latest - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: torchdata-artifact - - name: Determine if Wheel Uploading is needed - run: | - upload=false - for txt in torchdata*.whl; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Wheels - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh torchdata*.whl - - name: Upload Wheels to S3 Storage - if: steps.trigger_upload.outputs.value == 'true' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - run: | - if [[ ${{ inputs.branch }} == 'main' ]]; then - S3_PATH=s3://pytorch/whl/nightly/ - else - S3_PATH=s3://pytorch/whl/test/ - fi - pip3 install awscli - set -x - for pkg in torchdata*.whl; do - aws s3 cp "$pkg" "$S3_PATH" --acl public-read - done - - name: Upload Official Wheels to PYPI - if: | - steps.trigger_upload.outputs.value == 'true' && - needs.get_release_type.outputs.type == 'official' - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: | - pip3 install twine - python -m twine upload \ - --username __token__ \ - --password "$PYPI_TOKEN" \ - torchdata*.whl - conda_build_test: needs: get_release_type runs-on: ${{ matrix.os }} @@ -302,20 +24,14 @@ jobs: fail-fast: false matrix: os: - - macos-latest - - ubuntu-latest - - windows-latest - macos-m1-12 python-version: - 3.8 - - 3.9 - "3.10" - - "3.11" steps: - name: Checkout Source Repository uses: actions/checkout@v3 with: - ref: ${{ inputs.branch }} submodules: recursive - name: Determine if build AWSSDK shell: bash -l {0} @@ -369,7 +85,7 @@ jobs: shell: bash -l {0} env: PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} + PYTORCH_VERSION: "2.0.0" BUILD_S3: ${{ steps.build_s3.outputs.value }} run: | set -ex @@ -391,157 +107,3 @@ jobs: with: name: torchdata-artifact path: conda-bld/*/torchdata-*.tar.bz2 - - conda_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, conda_build_test] - runs-on: ubuntu-latest - container: continuumio/miniconda3 - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: torchdata-artifact - - name: Determine if Conda Uploading is needed - run: | - upload=false - for pkg in ./*/torchdata-*.tar.bz2; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Conda Package - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh ./*/torchdata-*.tar.bz2 - - name: Upload Packages to Conda - if: steps.trigger_upload.outputs.value == 'true' - shell: bash -l {0} - env: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} - CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} - run: | - conda create -y --name conda_upload_env - conda activate conda_upload_env - - conda install -yq anaconda-client - conda install -c conda-forge -yq jq - - if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then - CONDA_CHANNEL=pytorch - CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN} - elif [[ ${{ needs.get_release_type.outputs.type }} == 'test' ]]; then - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_TEST_PYTORCHBOT_TOKEN} - else - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_NIGHTLY_PYTORCHBOT_TOKEN} - fi - - if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then - # Loop over all platforms [win-64, osx-64, linux-64] - for subdir in $(find . -type f -name '*torchdata*.tar.bz2' | sed -r 's|/[^/]+$||' | uniq | cut -f2 -d/) ; do - version="" - # Find existing conda packages on pytorch nightly - for val in $(conda search --json torchdata --channel=$CONDA_CHANNEL --subdir=$subdir | jq -r '.[][] | .version, .build'); do - if [[ -z $version ]]; then - version=$val - else - build=$val - # Check if a new conda package built based on the same Python version and platform exists - for new_package in ./$subdir/torchdata-*-$build.tar.bz2; do - if [[ -f "$new_package" ]]; then - echo "Removing $CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - anaconda -t "${CONDA_TOKEN}" remove -f "$CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - break - fi - done - version="" - build="" - fi - done - # Upload new conda packages per subdir - anaconda -t "${CONDA_TOKEN}" upload ./$subdir/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - done - else - anaconda -t "${CONDA_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - fi - - build_docs: - if: | - always() && inputs.branch != '' && - ( needs.wheel_upload.outputs.upload == 'true' || needs.conda_upload.outputs.upload == 'true' ) - needs: [get_release_type, wheel_upload, conda_upload] - runs-on: ubuntu-latest - steps: - - name: Setup Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Install Dependencies - run: | - echo `python3 --version` - sudo apt-get install -y python-setuptools - python3 -m pip install --upgrade pip - python3 -m pip install setuptools - python3 -m pip install matplotlib - sudo apt-get install -y yarn - - name: Install PyTorch & TorchData - run: | - pip3 install numpy - # Add version requirement to PyTorch except nightly release - if [[ -z "${{ inputs.pytorch_version }}" ]]; then - PYTORCH_VERSION=torch - else - PYTORCH_VERSION=torch==${{ inputs.pytorch_version }} - fi - - PIP_CHANNEL=${{ needs.get_release_type.outputs.type }} - if [[ $PIP_CHANNEL == 'official' ]]; then - pip3 install "$PYTORCH_VERSION" -f https://download.pytorch.org/whl/torch_stable.html - else - pip3 install --pre "$PYTORCH_VERSION" -f "https://download.pytorch.org/whl/$PIP_CHANNEL/torch_$PIP_CHANNEL.html" - fi - - pip3 install -r requirements.txt - pip3 install . - - name: Check env - run: echo `which spinx-build` - - name: Build the docset - run: | - cd ./docs - sudo apt-get install -y graphviz - pip3 install -r requirements.txt - make html - cd .. - - name: Export Target Folder - run: | - TARGET_FOLDER=${{ inputs.branch }} - if [[ $TARGET_FOLDER == release/* ]]; then - TARGET_FOLDER=${TARGET_FOLDER:8} - elif [[ $TARGET_FOLDER == tags/* ]]; then - TARGET_FOLDER=${TARGET_FOLDER:5} - elif [[ $TARGET_FOLDER == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then - if [[ $TARGET_FOLDER == v*.*.* ]]; then - TARGET_FOLDER=${TARGET_FOLDER%.*} - fi - TARGET_FOLDER=${TARGET_FOLDER:1} - fi - echo "value=$TARGET_FOLDER" >> $GITHUB_OUTPUT - id: target_folder - - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs/build/html # The folder the action should deploy. - target-folder: ${{ steps.target_folder.outputs.value }} # The destination folder the action should deploy to. From 9e784c20337ee0f2d05d49c82c67853ba141e654 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Mar 2023 11:57:42 -0800 Subject: [PATCH 3/6] Move setuptools to host dep --- packaging/torchdata/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/torchdata/meta.yaml b/packaging/torchdata/meta.yaml index b443ed638..4b492fe4b 100644 --- a/packaging/torchdata/meta.yaml +++ b/packaging/torchdata/meta.yaml @@ -14,12 +14,13 @@ requirements: # - {{ compiler('c') }} # [win] # - {{ compiler('cxx') }} # [win] - python - # - setuptools - cpuonly - curl # [not win] - openssl # [unix] - zlib # [unix] {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} + host: + - setuptools run: - python - urllib3>=1.25 From 641c04a741ff6998a61d9aed7719c2a5f2cb32a4 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Mar 2023 12:12:46 -0800 Subject: [PATCH 4/6] try_fix --- packaging/torchdata/meta.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/torchdata/meta.yaml b/packaging/torchdata/meta.yaml index 4b492fe4b..d2a890066 100644 --- a/packaging/torchdata/meta.yaml +++ b/packaging/torchdata/meta.yaml @@ -20,7 +20,10 @@ requirements: - zlib # [unix] {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} host: + - python - setuptools + - pkg-config # [not win] + - cmake run: - python - urllib3>=1.25 From 31b08625038ec3ecd9a5dd33202296280678436f Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Mar 2023 12:24:19 -0800 Subject: [PATCH 5/6] testing --- packaging/torchdata/meta.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/packaging/torchdata/meta.yaml b/packaging/torchdata/meta.yaml index d2a890066..4b492fe4b 100644 --- a/packaging/torchdata/meta.yaml +++ b/packaging/torchdata/meta.yaml @@ -20,10 +20,7 @@ requirements: - zlib # [unix] {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} host: - - python - setuptools - - pkg-config # [not win] - - cmake run: - python - urllib3>=1.25 From 4cf8f96dfbeecc13066bfe48de030580bdf78e62 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 2 Mar 2023 12:28:05 -0800 Subject: [PATCH 6/6] try fix --- packaging/torchdata/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packaging/torchdata/meta.yaml b/packaging/torchdata/meta.yaml index 4b492fe4b..ccc075773 100644 --- a/packaging/torchdata/meta.yaml +++ b/packaging/torchdata/meta.yaml @@ -19,8 +19,6 @@ requirements: - openssl # [unix] - zlib # [unix] {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} - host: - - setuptools run: - python - urllib3>=1.25