From 22f1e4a75197c5de1c8a94350a4bcc136e3148b9 Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Fri, 4 Oct 2024 12:17:56 -0400 Subject: [PATCH] [DO NOT LAND] Try to debug Mac CI failure ghstack-source-id: 1b9a713e3b4cc7bf9d36c77543e2f2f2b342f67b Pull Request resolved: https://github.com/pytorch/executorch/pull/5844 --- .github/workflows/macos-job.yml | 200 ++++++++++++++++++++++++++++++++ .github/workflows/pull.yml | 28 +++++ 2 files changed, 228 insertions(+) create mode 100644 .github/workflows/macos-job.yml diff --git a/.github/workflows/macos-job.yml b/.github/workflows/macos-job.yml new file mode 100644 index 00000000000..410223a8d25 --- /dev/null +++ b/.github/workflows/macos-job.yml @@ -0,0 +1,200 @@ +name: Set up conda environment for testing + +description: Clean workspace and check out PyTorch + +inputs: + python-version: + description: If set to any value, dont use sudo to clean the workspace + required: false + type: string + default: "3.9" + miniconda-version: + description: Miniconda version to install + required: false + type: string + default: "23.1.0-1" + environment-file: + description: Environment file to install dependencies from + required: false + type: string + default: "" + pip-requirements-file: + description: An optional pip requirements file to be installed in the conda environment + required: false + type: string + default: "" + +runs: + using: composite + steps: + # Use the same trick from https://github.com/marketplace/actions/setup-miniconda + # to refresh the cache daily. This is kind of optional though + - name: Get date + id: get-date + shell: bash + run: | + echo "today=$(/bin/date -u '+%Y%m%d')d" >> "${GITHUB_OUTPUT}" + + - name: Setup miniconda cache + id: miniconda-cache + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/miniconda + key: miniconda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }} + + - name: Install miniconda (${{ inputs.miniconda-version }}) + if: steps.miniconda-cache.outputs.cache-hit != 'true' + env: + MINICONDA_VERSION: ${{ inputs.miniconda-version }} + shell: bash -l {0} + run: | + MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda" + mkdir -p "${MINICONDA_INSTALL_PATH}" + case ${RUNNER_OS}-${RUNNER_ARCH} in + Linux-X64) + MINICONDA_ARCH="Linux-x86_64" + ;; + macOS-ARM64) + MINICONDA_ARCH="MacOSX-arm64" + ;; + macOS-X64) + MINICONDA_ARCH="MacOSX-x86_64" + ;; + *) + echo "::error::Platform ${RUNNER_OS}-${RUNNER_ARCH} currently unsupported using this action" + exit 1 + ;; + esac + MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py39_${MINICONDA_VERSION}-${MINICONDA_ARCH}.sh" + curl -fsSL "${MINICONDA_URL}" -o "${MINICONDA_INSTALL_PATH}/miniconda.sh" + bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}" + rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh" + + - name: Update GitHub path to include miniconda install + shell: bash + run: | + set -x + + MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda" + echo "${MINICONDA_INSTALL_PATH}/bin" >> $GITHUB_PATH + # NB: GITHUB_PATH has a lower priority than PATH, so also set the path + # here to make sure that the correct conda is used + echo "PATH=${MINICONDA_INSTALL_PATH}/bin:${PATH}" >> $GITHUB_ENV + + # When the environment-file or pip-requirements-file inputs are not set or are set to invalid paths, the hashFiles + # function will return an empty string without failing the step. This works out nicely and we can have a various + # combination of cache key such as: + # - Both are missing or invalid: miniconda-env-macOS-ARM64-20221022d-- + # - Both are set: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)-HASH(pip-requirements-file) + # - The first one is missing or invalid: miniconda-env-macOS-ARM64-20221022d--HASH(pip-requirements-file) + # - The second one is missing or invalid: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)- + # + # There is no need to skip or run actions/cache with complicated logic + - name: Setup miniconda env cache + id: miniconda-env-cache + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/conda-python-${{ inputs.python-version }} + key: miniconda-env-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(inputs.environment-file) }}-${{ hashFiles(inputs.pip-requirements-file) }} + + - name: Setup conda environment with python (v${{ inputs.python-version }}) + if: steps.miniconda-env-cache.outcome == 'success' && steps.miniconda-env-cache.outputs.cache-hit != 'true' + shell: bash + env: + PYTHON_VERSION: ${{ inputs.python-version }} + ENV_FILE: ${{ inputs.environment-file }} + PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }} + run: | + set -x + + CONDA_BASE_ENV="${RUNNER_TEMP}/conda-python-${PYTHON_VERSION}" + ENV_FILE_FLAG="" + if [[ -f "${ENV_FILE}" ]]; then + ENV_FILE_FLAG="--file ${ENV_FILE}" + elif [[ -n "${ENV_FILE}" ]]; then + echo "::warning::Specified env file (${ENV_FILE}) not found, not going to include it" + fi + + CONDA_EXTRA_FLAGS="" + if [[ "${PYTHON_VERSION}" == "3.11" ]]; then + CONDA_EXTRA_FLAGS=" -c pytorch-nightly" + fi + + # Print the conda we are using here in case we need debugging information + CONDA_RUNTIME=$(which conda) + "${CONDA_RUNTIME}" --version + + "${CONDA_RUNTIME}" create \ + --yes --quiet \ + --prefix "${CONDA_BASE_ENV}" \ + ${ENV_FILE_FLAG} \ + python="${PYTHON_VERSION}" \ + cmake=3.22 \ + ninja=1.10 \ + pkg-config=0.29 \ + wheel=0.37 \ + ${CONDA_EXTRA_FLAGS} + + if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then + "${CONDA_RUNTIME}" run -p "${CONDA_BASE_ENV}" --no-capture-output python3 -mpip install -r "${PIP_REQUIREMENTS_FILE}" + elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then + echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it" + fi + + - name: Clone the base conda environment and update GitHub env + shell: bash + env: + PYTHON_VERSION: ${{ inputs.python-version }} + CONDA_BASE_ENV: ${{ runner.temp }}/conda-python-${{ inputs.python-version }} + PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }} + run: | + set -x + + # Print the conda we are using here in case we need debugging information + CONDA_RUNTIME=$(which conda) + "${CONDA_RUNTIME}" --version + + CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}" + "${CONDA_RUNTIME}" create \ + --yes --quiet \ + --prefix "${CONDA_ENV}" \ + --clone "${CONDA_BASE_ENV}" + + set +e + # NB: Cloning sometimes doesn't copied pip dependencies (untracked files) over. If this + # happens, let's attempt to install the pip requirements directly on top of the cloned + # environment. This is to make sure that no dependency is missing. + UNTRACKED_FILES_COUNT=$("${CONDA_RUNTIME}" package -p "${CONDA_ENV}" -u | grep -v "^#" | wc -l | xargs) + set -e + + if [[ -z "${UNTRACKED_FILES_COUNT}" ]] || [[ "${UNTRACKED_FILES_COUNT}" == "0" ]]; then + if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then + # NB: Force reinstall and don't use the cache, as the installation would still fail + # when reporting that all requirements have been satisfied + "${CONDA_RUNTIME}" run -p "${CONDA_ENV}" --no-capture-output python3 -mpip install --ignore-installed --no-cache-dir -r "${PIP_REQUIREMENTS_FILE}" + elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then + echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it" + fi + fi + + echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}" + echo "CONDA_RUN=${CONDA_RUNTIME} run -p ${CONDA_ENV} --no-capture-output" >> "${GITHUB_ENV}" + if [[ "${PYTHON_VERSION}" == "3.11" ]]; then + # TODO: Remove me, when more packages will be available on default channel + echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV} -c pytorch-nightly" >> "${GITHUB_ENV}" + else + echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV}" >> "${GITHUB_ENV}" + fi + + - name: Reset channel priority + shell: bash + run: | + CONDA_RUNTIME=$(which conda) + + set -euxo pipefail + "${CONDA_RUNTIME}" config --set channel_priority false + + - name: Setup upterm session + uses: owenthereal/action-upterm@v1 + with: + limit-access-to-actor: true diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index f7d2b627bc5..e9ac4b4692c 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -441,3 +441,31 @@ jobs: # run e2e (export, tokenizer and runner) PYTHON_EXECUTABLE=python bash .ci/scripts/test_phi_3_mini.sh + + debug: + runs-on: macos-13-xlarge + uses: ./.github/workflows/macos_job.yml + with: + runner: macos-m1-stable + python-version: '3.11' + submodules: 'true' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + BUILD_TOOL=cmake + BACKEND=coreml + + bash .ci/scripts/setup-conda.sh + + # Setup MacOS dependencies as there is no Docker support on MacOS atm + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh + echo "Finishing installing coreml." + + # Build and test coreml model + MODELS=(mv3 ic4 resnet50 edsr mobilebert w2l) + for MODEL_NAME in "${MODELS[@]}"; do + echo "::group::Exporting coreml model: $MODEL_NAME" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" + echo "::endgroup::" + done