Nightly Publish #46
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Publish | |
| on: | |
| schedule: | |
| - cron: '0 10 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: 'staging/snapshot, default is snapshot' | |
| required: true | |
| default: 'snapshot' | |
| jobs: | |
| build: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ${{ matrix.operating-system }} | |
| strategy: | |
| matrix: | |
| operating-system: [ macos-latest, ubuntu-latest ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 17 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: check disk space | |
| run: df -h | |
| - name: install libomp on macos | |
| if: ${{ runner.os == 'macOS' }} | |
| run: brew install libomp | |
| - name: Build with Gradle | |
| run: ./gradlew -Dnightly=true build :jacoco:testCodeCoverageReport | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: reports | |
| path: | | |
| ${{ github.workspace }}/**/build/reports/**/* | |
| !${{ github.workspace }}/**/build/reports/jacoco/* | |
| test-pytorch: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ${{ matrix.operating-system }} | |
| strategy: | |
| matrix: | |
| operating-system: [ macos-latest, ubuntu-latest, windows-latest ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 17 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Test with Gradle | |
| run: ./gradlew :integration:test "-Dai.djl.default_engine=PyTorch" | |
| test-tensorflow: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ${{ matrix.operating-system }} | |
| strategy: | |
| matrix: | |
| operating-system: [ ubuntu-latest, windows-latest ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 17 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Test with Gradle | |
| run: ./gradlew :integration:test "-Dai.djl.default_engine=TensorFlow" | |
| test-aarch64: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, aarch64 ] | |
| container: | |
| image: amazonlinux:2 | |
| env: | |
| JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.aarch64 | |
| timeout-minutes: 30 | |
| needs: create-runners | |
| steps: | |
| - name: Setup Environment | |
| run: | | |
| yum install -y tar gzip | |
| yum -y install java-17-amazon-corretto-devel | |
| - uses: taiki-e/checkout-action@v1 | |
| - name: Test with Gradle | |
| run: | | |
| ./gradlew :integration:test "-Dai.djl.default_engine=PyTorch" | |
| ./gradlew :integration:clean | |
| # OnnxRuntime 1.17.1 requires GLIBC 2.27 | |
| # ./gradlew :integration:test "-Dai.djl.default_engine=OnnxRuntime" | |
| # ./gradlew :integration:clean | |
| test-cuda-124: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, gpu ] | |
| container: | |
| image: nvidia/cuda:12.4.1-devel-ubuntu22.04 | |
| options: --gpus all --runtime=nvidia | |
| timeout-minutes: 30 | |
| needs: create-runners | |
| steps: | |
| - name: Setup Environment | |
| run: | | |
| apt-get update | |
| DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common wget locales libfontconfig1 | |
| locale-gen en_US.UTF-8 | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 17 | |
| distribution: corretto | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Test with Gradle | |
| run: | | |
| ./gradlew :integration:test -Dai.djl.default_engine=PyTorch | |
| ./gradlew :integration:clean :integration:test -Dai.djl.default_engine=TensorFlow | |
| ./gradlew :engines:pytorch:pytorch-model-zoo:test | |
| ./gradlew :engines:tensorflow:tensorflow-model-zoo:test | |
| # manually install cudnn9, PyTorch doesn't work with cudnn9, but OnnxRuntime requires cudnn9 | |
| apt-get update && apt-get install -y --no-install-recommends libcudnn9-cuda-12 | |
| ./gradlew :engines:onnxruntime:onnxruntime-engine:test | |
| publish: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ubuntu-latest | |
| needs: [ build, test-pytorch, test-tensorflow, test-aarch64, test-cuda-124 ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 17 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Publish to snapshot repository | |
| if: ${{ github.event.inputs.mode == '' || github.event.inputs.mode == 'snapshot' }} | |
| run: | | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.13.1 -Psnapshot | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.1.2 -Psnapshot | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.3.1 -Psnapshot | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.4.0 -Psnapshot | |
| ./gradlew clean engines:ml:xgboost:publish -Pgpu -Psnapshot | |
| ./gradlew clean publish -Psnapshot | |
| cd bom | |
| ./gradlew publish -Psnapshot | |
| env: | |
| ORG_GRADLE_PROJECT_signingKey: ${{ secrets.ORG_GRADLE_PROJECT_signingKey }} | |
| ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.ORG_GRADLE_PROJECT_signingPassword }} | |
| ORG_GRADLE_PROJECT_ossrhUsername: ${{ secrets.ORG_GRADLE_PROJECT_ossrhUsername }} | |
| ORG_GRADLE_PROJECT_ossrhPassword: ${{ secrets.ORG_GRADLE_PROJECT_ossrhPassword }} | |
| - name: Publish to staging repository | |
| if: ${{ github.event.inputs.mode == 'staging' }} | |
| run: | | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.13.1 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.1.2 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.3.1 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.4.0 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:ml:xgboost:publish -Pgpu -P${{ github.event.inputs.mode }} | |
| ./gradlew clean publish -P${{ github.event.inputs.mode }} | |
| cd bom | |
| ./gradlew publish -P${{ github.event.inputs.mode }} | |
| env: | |
| ORG_GRADLE_PROJECT_signingKey: ${{ secrets.ORG_GRADLE_PROJECT_signingKey }} | |
| ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.ORG_GRADLE_PROJECT_signingPassword }} | |
| ORG_GRADLE_PROJECT_ossrhUsername: ${{ secrets.ORG_GRADLE_PROJECT_ossrhUsername }} | |
| ORG_GRADLE_PROJECT_ossrhPassword: ${{ secrets.ORG_GRADLE_PROJECT_ossrhPassword }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| create-runners: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, scheduler ] | |
| steps: | |
| - name: Create new Graviton instance | |
| id: create_aarch64 | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \ | |
| https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \ | |
| --fail \ | |
| | jq '.token' | tr -d '"' ) | |
| ./start_instance.sh action_graviton $token djl | |
| - name: Create new GPU instance 1 | |
| id: create_gpu_1 | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \ | |
| https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \ | |
| --fail \ | |
| | jq '.token' | tr -d '"' ) | |
| ./start_instance.sh action_gpu $token djl | |
| outputs: | |
| aarch64_instance_id: ${{ steps.create_aarch64.outputs.action_graviton_instance_id }} | |
| gpu_1_instance_id: ${{ steps.create_gpu_1.outputs.action_gpu_instance_id }} | |
| stop-runners: | |
| if: ${{ github.repository == 'deepjavalibrary/djl' && always() }} | |
| runs-on: [ self-hosted, scheduler ] | |
| needs: [ create-runners, test-aarch64, test-cuda-124 ] | |
| steps: | |
| - name: Stop all instances | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| instance_id=${{ needs.create-runners.outputs.aarch64_instance_id }} | |
| ./stop_instance.sh $instance_id | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| instance_id=${{ needs.create-runners.outputs.gpu_1_instance_id }} | |
| ./stop_instance.sh $instance_id |