periodic #277
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: periodic | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # Runs daily at midnight UTC | |
| push: | |
| tags: | |
| - ciflow/periodic/* | |
| workflow_dispatch: | |
| jobs: | |
| gather-models-cpu: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| models: ${{ steps.gather-models-cpu.outputs.models }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'false' | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Extract the list of models to run on CPU | |
| id: gather-models-cpu | |
| run: | | |
| set -eux | |
| PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py --event "periodic" --backend "cpu" | |
| test-cpu-compile: | |
| name: test-cpu-compile (${{ matrix.platform }}, ${{ matrix.model_name }}) | |
| needs: gather-models-cpu | |
| strategy: | |
| matrix: ${{ fromJSON(needs.gather-models-cpu.outputs.models) }} | |
| fail-fast: false | |
| runs-on: ${{ matrix.runner }} | |
| env: | |
| TORCHCHAT_ROOT: ${{ github.workspace }} | |
| REPO_NAME: ${{ matrix.repo_name }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Print machine info | |
| run: | | |
| echo "$(uname -a)" | |
| - name: Install dependencies | |
| run: | | |
| ./install/install_requirements.sh | |
| pip3 list | |
| python3 -c 'import torch;print(f"torch: {torch.__version__, torch.version.git_version}")' | |
| - name: Download checkpoints | |
| run: | | |
| bash ${TORCHCHAT_ROOT}/.ci/scripts/wget_checkpoint.sh ${{ matrix.repo_name }} "${{ matrix.resources }}" | |
| - name: Run validation | |
| run: | | |
| pushd ${TORCHCHAT_ROOT} | |
| bash .ci/scripts/convert_checkpoint.sh ${REPO_NAME} | |
| bash .ci/scripts/validate.sh "./checkpoints/${REPO_NAME}/model.pth" "cpu" "compile" | |
| test-cpu-aoti: | |
| name: test-cpu-aoti (${{ matrix.platform }}, ${{ matrix.model_name }}) | |
| needs: gather-models-cpu | |
| strategy: | |
| matrix: ${{ fromJSON(needs.gather-models-cpu.outputs.models) }} | |
| fail-fast: false | |
| runs-on: ${{ matrix.runner }} | |
| env: | |
| TORCHCHAT_ROOT: ${{ github.workspace }} | |
| REPO_NAME: ${{ matrix.repo_name }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Print machine info | |
| run: | | |
| echo "$(uname -a)" | |
| - name: Install dependencies | |
| run: | | |
| ./install/install_requirements.sh | |
| pip3 list | |
| python3 -c 'import torch;print(f"torch: {torch.__version__, torch.version.git_version}")' | |
| - name: Download checkpoints | |
| run: | | |
| bash ${TORCHCHAT_ROOT}/.ci/scripts/wget_checkpoint.sh ${{ matrix.repo_name }} "${{ matrix.resources }}" | |
| - name: Run validation | |
| run: | | |
| pushd ${TORCHCHAT_ROOT} | |
| bash .ci/scripts/convert_checkpoint.sh ${REPO_NAME} | |
| bash .ci/scripts/validate.sh "./checkpoints/${REPO_NAME}/model.pth" "cpu" "aoti" | |
| gather-models-gpu: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| models: ${{ steps.gather-models-gpu.outputs.models }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'false' | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Extract the list of models to run on GPU | |
| id: gather-models-gpu | |
| run: | | |
| set -eux | |
| PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py --event "periodic" --backend "gpu" | |
| test-gpu: | |
| uses: pytorch/test-infra/.github/workflows/linux_job.yml@main | |
| name: test-gpu (${{ matrix.platform }}, ${{ matrix.model_name }}) | |
| needs: gather-models-gpu | |
| secrets: inherit | |
| strategy: | |
| matrix: ${{ fromJSON(needs.gather-models-gpu.outputs.models) }} | |
| fail-fast: false | |
| with: | |
| secrets-env: "HF_TOKEN_PERIODIC" | |
| runner: ${{ matrix.runner }} | |
| gpu-arch-type: cuda | |
| gpu-arch-version: "12.1" | |
| script: | | |
| echo "::group::Print machine info" | |
| nvidia-smi | |
| echo "::endgroup::" | |
| echo "::group::Install required packages" | |
| ./install/install_requirements.sh cuda | |
| pip3 list | |
| python3 -c 'import torch;print(f"torch: {torch.__version__, torch.version.git_version}")' | |
| echo "::endgroup::" | |
| echo "::group::Download checkpoint" | |
| export REPO_NAME="${{ matrix.repo_name }}" | |
| case "${{ matrix.install_method }}" in | |
| wget) | |
| bash .ci/scripts/wget_checkpoint.sh "${REPO_NAME}" "${{ matrix.resources }}" | |
| ;; | |
| huggingface-cli) | |
| ( | |
| set +x | |
| HF_TOKEN="${SECRET_HF_TOKEN_PERIODIC}" bash .ci/scripts/download_llama.sh | |
| ) | |
| ;; | |
| esac | |
| echo "::endgroup::" | |
| echo "::group::Convert checkpoint" | |
| bash .ci/scripts/convert_checkpoint.sh ${REPO_NAME} | |
| echo "::endgroup::" | |
| echo "::group::Run inference" | |
| bash .ci/scripts/validate.sh "./checkpoints/${REPO_NAME}/model.pth" "cuda" "compile" | |
| bash .ci/scripts/validate.sh "./checkpoints/${REPO_NAME}/model.pth" "cuda" "aoti" | |
| echo "::group::Run eval" | |
| bash .ci/scripts/validate.sh "./checkpoints/${REPO_NAME}/model.pth" "cuda" "eval" | |
| echo "::endgroup::" |