-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'mac_wheels_ci' of github.com:scotts/torchcodec into mac…
…_wheels_ci
- Loading branch information
Showing
2 changed files
with
76 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -54,16 +54,6 @@ jobs: | |
build-platform: "python-build-package" | ||
build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ${CONDA_RUN} python3 -m build --wheel -vvv --no-isolation" | ||
|
||
validate-binaries: | ||
uses: pytorch/test-infra/.github/workflows/validate-domain-library.yml@main | ||
with: | ||
package_type: "wheel" | ||
os: "macos-arm64" | ||
channel: "nightly" | ||
repository: "pytorch/torchcodec" | ||
smoke_test: "source ./packaging/validate_wheel.sh" | ||
install_torch: true | ||
|
||
install-and-test: | ||
runs-on: macos-m1-stable | ||
strategy: | ||
|
@@ -78,61 +68,96 @@ jobs: | |
with: | ||
name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ | ||
path: pytorch/torchcodec/dist/ | ||
- name: Setup miniconda | ||
uses: pytorch/test-infra/.github/actions/setup-miniconda@macbuildwheel | ||
## start copying from test-infra/.github/actions/setup-binary-builds/action.yaml, line 84 | ||
- name: Setup miniconda (for pytorch_pkg_helpers) | ||
uses: conda-incubator/[email protected] | ||
with: | ||
miniconda-version: "py39_4.12.0" | ||
python-version: ${{ matrix.python-version }} | ||
- name: Update pip | ||
- name: Clean conda environment | ||
shell: bash -l {0} | ||
run: | | ||
set -euxo pipefail | ||
conda info | grep -i 'base environment' | ||
conda clean --all --quiet --yes | ||
- name: Reset channel priority | ||
shell: bash -l {0} | ||
run: | | ||
python3 -m pip install --upgrade pip | ||
set -euxo pipefail | ||
conda config --set channel_priority false | ||
- name: Generate file from pytorch_pkg_helpers | ||
working-directory: ${{ inputs.repository }} | ||
shell: bash -l {0} | ||
run: | | ||
set -euxo pipefail | ||
CONDA_ENV="${RUNNER_TEMP}/pytorch_pkg_helpers_${GITHUB_RUN_ID}" | ||
conda create \ | ||
--yes --quiet \ | ||
--prefix "${CONDA_ENV}" \ | ||
"python=3.9" | ||
CONDA_ENV="${CONDA_ENV}" | ||
CONDA_RUN="conda run -p ${CONDA_ENV}" | ||
${CONDA_RUN} python -m pip install ${GITHUB_WORKSPACE}/test-infra/tools/pkg-helpers | ||
BUILD_ENV_FILE="${RUNNER_TEMP}/build_env_${GITHUB_RUN_ID}" | ||
${CONDA_RUN} python -m pytorch_pkg_helpers > "${BUILD_ENV_FILE}" | ||
cat "${BUILD_ENV_FILE}" | ||
echo "BUILD_ENV_FILE=${BUILD_ENV_FILE}" >> "${GITHUB_ENV}" | ||
- name: Setup conda environment for build | ||
shell: bash -l {0} | ||
env: | ||
PYTHON_VERSION: ${{ matrix.python-version }} | ||
run: | | ||
set -euxo pipefail | ||
CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}" | ||
if [[ "${PACKAGE_TYPE:-}" == "conda" ]]; then | ||
# For conda package host python version is irrelevant | ||
export PYTHON_VERSION=3.9 | ||
export CONDA_BUILD_EXTRA="conda=24.4.0 conda-build=24.3.0 python-libarchive-c=2.9" | ||
else | ||
# For wheel builds we don't need neither conda nor conda-build | ||
export CONDA_BUILD_EXTRA="" | ||
fi | ||
conda create \ | ||
--yes --quiet \ | ||
--prefix "${CONDA_ENV}" \ | ||
"python=${PYTHON_VERSION}" \ | ||
cmake=3.26 \ | ||
ninja=1.10 \ | ||
pkg-config=0.29 \ | ||
${CONDA_BUILD_EXTRA} \ | ||
wheel=0.37 | ||
echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}" | ||
echo "CONDA_RUN=conda run -p ${CONDA_ENV}" >> "${GITHUB_ENV}" | ||
## end copying | ||
- name: Update pip | ||
run: python -m pip install --upgrade pip | ||
- name: Install PyTorch | ||
run: | | ||
python3 -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu | ||
${CONDA_RUN} python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu | ||
- name: Install torchcodec from the wheel | ||
run: | | ||
wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` | ||
echo Installing $wheel_path | ||
python3 -m pip install $wheel_path -vvv | ||
${CONDA_RUN} python -m pip install $wheel_path -vvv | ||
- name: Check out repo | ||
uses: actions/checkout@v3 | ||
- name: Install ffmpeg, post build | ||
run: | | ||
# Ideally we would have checked for that before installing the wheel, | ||
# but we need to checkout the repo to access this file, and we don't | ||
# want to checkout the repo before installing the wheel to avoid any | ||
# side-effect. It's OK. | ||
source packaging/helpers.sh | ||
# below was failing, but when I just try to call "ffmpeg -version" that also fails? | ||
#assert_ffmpeg_not_installed | ||
conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge | ||
ffmpeg -version | ||
- name: Install test dependencies | ||
run: | | ||
python3 -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu | ||
# Ideally we would find a way to get those dependencies from pyproject.toml | ||
python3 -m pip install numpy pytest pillow | ||
${CONDA_RUN} python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu | ||
${CONDA_RUN} python -m pip install numpy pytest pillow | ||
- name: Delete the src/ folder just for fun | ||
run: | | ||
# The only reason we checked-out the repo is to get access to the | ||
# tests. We don't care about the rest. Out of precaution, we delete | ||
# the src/ folder to be extra sure that we're running the code from | ||
# the installed wheel rather than from the source. | ||
# This is just to be extra cautious and very overkill because a) | ||
# there's no way the `torchcodec` package from src/ can be found from | ||
# the PythonPath: the main point of `src/` is precisely to protect | ||
# against that and b) if we ever were to execute code from | ||
# `src/torchcodec`, it would fail loudly because the built .so files | ||
# aren't present there. | ||
rm -r src/ | ||
ls | ||
- name: Smoke test | ||
run: | | ||
python3 test/decoders/manual_smoke_test.py | ||
${CONDA_RUN} python test/decoders/manual_smoke_test.py | ||
- name: Run Python tests | ||
run: | | ||
pytest test | ||
${CONDA_RUN} pytest test |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters