Merge remote-tracking branch 'upstream/main' into libshortfin-windows…

…-build
nod-ai · Oct 2, 2024 · c68920d · c68920d
2 parents 3e85b6e + b0c2399
commit c68920d
Show file tree

Hide file tree

Showing 210 changed files with 8,088 additions and 1,222 deletions.
diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml
@@ -0,0 +1,111 @@
+# Copyright 2024 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: Build packages
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs at 11:00 AM UTC, which is 3:00 AM PST (UTC-8)
+    - cron: '0 11 * * *'
+
+jobs:
+  # Note: metadata generation could happen in a separate trigger/schedule
+  # workflow. For cross platform builds, it's useful to just generate the
+  # metadata on Linux and pass that to later jobs using artifacts.
+  setup_metadata:
+    runs-on: ubuntu-24.04
+    outputs:
+      shark_package_version: ${{ steps.version.outputs.shark_package_version }}
+    steps:
+      # For now the version is just a calendar date + an automatically
+      # incrementing value. We may want different versions for nightly/dev
+      # builds and stable releases published to official places like pypi.
+      - name: Compute version
+        id: version
+        run: |
+          shark_package_version="$(printf '%(%Y%m%d)T.${{ github.run_number }}')"
+          echo "shark_package_version=${shark_package_version}" >> $GITHUB_OUTPUT
+          cat << EOF > ./version_info.json
+          {
+            "package-version": "${shark_package_version}"
+          }
+          EOF
+          cat ./version_info.json
+      - name: Upload version_info.json
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: version_info
+          path: version_info.json
+
+  build_packages:
+    name: "${{ matrix.package }} :: ${{ matrix.platform }} :: ${{ matrix.python-version }}"
+    runs-on: ${{ matrix.runs-on }}
+    needs: [setup_metadata]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Ubuntu packages.
+          - runs-on: ubuntu-24.04
+            platform: linux-x86_64
+            package: shortfin
+            python-version: cp312-cp312
+          - runs-on: ubuntu-24.04
+            platform: linux-x86_64
+            package: shortfin
+            python-version: cp313-cp313
+          - runs-on: ubuntu-24.04
+            platform: linux-x86_64
+            package: shortfin
+            python-version: cp313-cp313t
+
+          # TODO(#130): macOS platform
+          # TODO(#130): Windows platform
+          # TODO(#130): sharktank packages
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        path: "c" # Windows can hit path length limits, so use a short path.
+        submodules: false
+
+    - name: Download version_info.json
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      with:
+        name: version_info
+        path: ./c/shortfin/
+        merge-multiple: true
+
+    - name: Build shortfin (Linux x86_64, ${{ matrix.python-version }})
+      if: "matrix.package == 'shortfin' && matrix.platform == 'linux-x86_64'"
+      env:
+        OUTPUT_DIR: "${{ github.workspace }}/bindist"
+        OVERRIDE_PYTHON_VERSIONS: "${{ matrix.python-version }}"
+      run: |
+        [ -e ./bindist/* ] && rm ./bindist/*
+        ./c/shortfin/build_tools/build_linux_package.sh
+
+    - name: Upload python wheels
+      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      with:
+        if-no-files-found: error
+        name: snapshot-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }}
+        path: bindist
+
+    - name: Release python wheels
+      uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0
+      with:
+        artifacts: bindist/*.whl
+        token: "${{ secrets.RELEASE_PUBLISH_ACCESS_TOKEN }}"
+        tag: "dev-wheels"
+        name: "dev-wheels"
+        body: "Automatic snapshot release of SHARK-Platform python wheels."
+        removeArtifacts: false
+        allowUpdates: true
+        replacesArtifacts: true
+        makeLatest: false
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -55,20 +55,9 @@ jobs:
           pip install --no-compile -r pytorch-cpu-requirements.txt
           pip install --no-compile -f https://iree.dev/pip-release-links.html --src deps \
             -e "git+https://github.com/iree-org/iree-turbine.git#egg=shark-turbine"
-          pip install --no-compile -r requirements.txt -e sharktank/ shortfin/
+          pip install --no-compile -r requirements.txt -e sharktank/
 
       - name: Run sharktank tests
         if: ${{ !cancelled() }}
         run: |
           pytest -n 4 sharktank/
-
-      - name: Run shortfin tests
-        if: ${{ !cancelled() }}
-        run: |
-          pytest -n 4 shortfin/
-
-      # TODO: Enable type checking of sharktank
-      - name: MyPy Type Checking Shortfin
-        if: ${{ !cancelled() }}
-        run: |
-          (cd shortfin && mypy)
diff --git a/.github/workflows/ci_linux_x64-libshortfin.yml b/.github/workflows/ci_linux_x64-libshortfin.yml
@@ -4,7 +4,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-name: CI - libshortfin
+name: CI - shortfin
 
 on:
   workflow_dispatch:
@@ -14,14 +14,22 @@ on:
       - main
     paths:
       - '.github/workflows/ci_linux_x64-libshortfin.yml'
-      - 'libshortfin/**'
+      - 'shortfin/**'
 
 permissions:
   contents: read
 
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
 env:
   IREE_REPO_DIR: ${{ github.workspace }}/iree
-  LIBSHORTFIN_DIR: ${{ github.workspace }}/libshortfin/
+  LIBSHORTFIN_DIR: ${{ github.workspace }}/shortfin/
 
 jobs:
   build-and-test:
@@ -49,8 +57,8 @@ jobs:
         ref: candidate-20240904.1006
 
     - name: Initalize IREE submodules
+      working-directory: ${{ env.IREE_REPO_DIR }}
       run : |
-        cd ${{ env.IREE_REPO_DIR }}
         git submodule update --init --depth 1 -- third_party/benchmark
         git submodule update --init --depth 1 -- third_party/cpuinfo/
         git submodule update --init --depth 1 -- third_party/flatcc
@@ -63,41 +71,45 @@ jobs:
         python-version: "3.12"
         cache: "pip"
     - name: Install Python packages
-      # TODO: Switch to `pip install -r requirements.txt -e libshortfin/`.
+      # TODO: Switch to `pip install -r requirements.txt -e shortfin/`.
+      working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
-        pip install -r ${{ env.LIBSHORTFIN_DIR }}/requirements-tests.txt
-        pip install -r ${{ env.LIBSHORTFIN_DIR }}/requirements-iree-compiler.txt
+        pip install -r requirements-tests.txt
+        pip install -r requirements-iree-compiler.txt
         pip freeze
 
-    - name: Build libshortfin (full)
+    - name: Build shortfin (full)
+      working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
-        mkdir ${{ env.LIBSHORTFIN_DIR }}/build
-        cd ${{ env.LIBSHORTFIN_DIR }}/build
+        mkdir build
         cmake -GNinja \
+          -S. \
+          -Bbuild \
           -DCMAKE_C_COMPILER=clang-18 \
           -DCMAKE_CXX_COMPILER=clang++-18 \
           -DCMAKE_LINKER_TYPE=LLD \
           -DSHORTFIN_BUNDLE_DEPS=ON \
           -DSHORTFIN_IREE_SOURCE_DIR=${{ env.IREE_REPO_DIR }} \
           -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON \
           ..
-        cmake --build . --target all
-        pip install -v -e .
+        cmake --build build --target all
+        pip install -v -e build/
 
-    - name: Test libshortfin (full)
+    - name: Test shortfin (full)
+      working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
-        cd ${{ env.LIBSHORTFIN_DIR }}/build
-        ctest --timeout 30 --output-on-failure
-        cd ${{ env.LIBSHORTFIN_DIR }}
+        ctest --timeout 30 --output-on-failure --test-dir build
         pytest -s
 
-    - name: Build libshortfin (host-only)
+    - name: Build shortfin (host-only)
+      working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
-        mkdir ${{ env.LIBSHORTFIN_DIR }}/build-host-only
-        cd ${{ env.LIBSHORTFIN_DIR }}/build-host-only
+        mkdir build-host-only
         # In this configuration, also build static+dynamic in order to verify
         # that path structurally works.
         cmake -GNinja \
+          -S. \
+          -Bbuild-host-only \
           -DCMAKE_C_COMPILER=clang-18 \
           -DCMAKE_CXX_COMPILER=clang++-18 \
           -DCMAKE_LINKER_TYPE=LLD \
@@ -107,4 +119,4 @@ jobs:
           -DSHORTFIN_BUILD_STATIC=ON \
           -DSHORTFIN_BUILD_DYNAMIC=ON \
           ..
-        cmake --build . --target all
+        cmake --build build-host-only --target all
diff --git a/.github/workflows/ci_linux_x64_asan-libshortfin.yml b/.github/workflows/ci_linux_x64_asan-libshortfin.yml
@@ -4,7 +4,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-name: CI - libshortfin - ASan
+name: CI - shortfin - ASan
 
 on:
   workflow_dispatch:
@@ -14,19 +14,27 @@ on:
       - main
     paths:
       - '.github/workflows/ci_linux_x64_asan-libshortfin.yml'
-      - 'libshortfin/**'
+      - 'shortfin/**'
 
 permissions:
   contents: read
 
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
 env:
   PYENV_ROOT: ${{ github.workspace }}/pyenv
   PYENV_REF: 9ecd803bffaffb949fbdd8c70cb086227f6a3202 # v2.4.10
   PYTHON_VER: 3.12.3
   CACHE_ASAN_VER: 2
   CACHE_DEPS_VER: 1
   IREE_SOURCE_DIR: ${{ github.workspace }}/iree
-  LIBSHORTFIN_DIR: ${{ github.workspace }}/libshortfin/
+  LIBSHORTFIN_DIR: ${{ github.workspace }}/shortfin/
 
 jobs:
   setup-python-asan:
@@ -63,24 +71,24 @@ jobs:
 
     - name: Install pyenv & Python
       if: steps.cache-python-asan.outputs.cache-hit != 'true'
+      working-directory: ${{ env.PYENV_ROOT }}
       run: |
-        cd ${{ env.PYENV_ROOT }}
         src/configure && make -C src
         export PATH=${{ env.PYENV_ROOT }}/bin:$PATH && eval "$(pyenv init -)"
         CC=clang-18 CXX=clang++-18 LDFLAGS="-lstdc++" PYTHON_CONFIGURE_OPTS="--with-address-sanitizer" pyenv install -v ${{ env.PYTHON_VER }}
         pyenv global ${{ env.PYTHON_VER }}
 
 
   build-and-test:
-    name: Build and test libshortfin
+    name: Build and test shortfin
     needs: [setup-python-asan]
     runs-on: ubuntu-24.04
     env:
       # We can't count on being leak free in general (i.e. pip, etc) so disable
       # leak checker by default. Here we suppress any ASAN features needed to
       # pass the build. Test configuration is done specially just for that step.
       ASAN_OPTIONS: detect_leaks=0,detect_odr_violation=0
-      LSAN_OPTIONS: suppressions=${{ github.workspace }}/libshortfin/build_tools/python_lsan_suppressions.txt
+      LSAN_OPTIONS: suppressions=${{ github.workspace }}/shortfin/build_tools/python_lsan_suppressions.txt
     steps:
     - name: Install dependencies
       run: |
@@ -101,8 +109,8 @@ jobs:
         ref: candidate-20240904.1006
 
     - name: Initalize IREE submodules
+      working-directory: ${{ env.IREE_SOURCE_DIR }}
       run : |
-        cd ${{ env.IREE_SOURCE_DIR }}
         git submodule update --init --depth 1 -- third_party/benchmark
         git submodule update --init --depth 1 -- third_party/cpuinfo/
         git submodule update --init --depth 1 -- third_party/flatcc
@@ -114,7 +122,7 @@ jobs:
       uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         path: ${{ env.PYENV_ROOT }}
-        key: ${{ runner.os }}-python-deps-${{ hashFiles('libshortfin/requirements-tests.txt') }}-v${{ env.CACHE_DEPS_VER }}
+        key: ${{ runner.os }}-python-deps-${{ hashFiles('shortfin/requirements-tests.txt') }}-v${{ env.CACHE_DEPS_VER }}
 
     - name: Restore Python ASan cache
       id: cache-python-asan
@@ -130,10 +138,11 @@ jobs:
 
     - name: Install Python dependencies
       if: steps.cache-python-deps-restore.outputs.cache-hit != 'true'
+      working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
         eval "$(pyenv init -)"
-        pip install -r ${{ env.LIBSHORTFIN_DIR }}/requirements-tests.txt
-        pip install -r ${{ env.LIBSHORTFIN_DIR }}/requirements-iree-compiler.txt
+        pip install -r requirements-tests.txt
+        pip install -r requirements-iree-compiler.txt
         pip freeze
 
     - name: Save Python dependencies cache
@@ -144,10 +153,10 @@ jobs:
         path: ${{ env.PYENV_ROOT }}
         key: ${{ steps.cache-python-deps-restore.outputs.cache-primary-key }}
 
-    - name: Build libshortfin
+    - name: Build shortfin
+      working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
         eval "$(pyenv init -)"
-        cd ${{ env.LIBSHORTFIN_DIR }}
         SHORTFIN_IREE_SOURCE_DIR="${{ env.IREE_SOURCE_DIR }}" \
         SHORTFIN_ENABLE_ASAN=ON \
         SHORTFIN_DEV_MODE=ON \
@@ -159,7 +168,7 @@ jobs:
       env:
         # TODO(#151): Don't ignore ODR violations
         ASAN_OPTIONS: detect_odr_violation=0
+      working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
         eval "$(pyenv init -)"
-        cd ${{ env.LIBSHORTFIN_DIR }}
         pytest -s