Skip to content

Commit

Permalink
change: rework GPU features (#810)
Browse files Browse the repository at this point in the history
* change: rework GPU features

* `body`に対してコメント

* `<table>`の組み立てを`build-spec-table`に集約する

* fixup! `<table>`の組み立てを`build-spec-table`に集約する

* `GpuSpec::defaults`の網羅性をテスト

* `DeviceAvailabilities`の表示を日本語にする

* デフォルトを`VOICEVOX/onnxruntime-builder`宛てにする

* `if cfg!(…)`の形にする

* `load-onnxruntime`でCUDAとDirectMLが使えることに言及
  • Loading branch information
qryxip authored Aug 4, 2024
1 parent 21290d0 commit bf8cdb8
Show file tree
Hide file tree
Showing 28 changed files with 1,221 additions and 276 deletions.
104 changes: 34 additions & 70 deletions .github/workflows/build_and_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,123 +61,90 @@ jobs:
includes='[
{
"os": "windows-2019",
"features": "",
"target": "x86_64-pc-windows-msvc",
"artifact_name": "windows-x64-cpu",
"artifact_name": "windows-x64",
"c_release_format": "plain-cdylib",
"whl_local_version": "cpu",
"can_skip_in_simple_test": true
},
{
"os": "windows-2019",
"features": "directml",
"target": "x86_64-pc-windows-msvc",
"artifact_name": "windows-x64-directml",
"c_release_format": "plain-cdylib",
"whl_local_version": "directml",
"python_whl": true,
"can_skip_in_simple_test": false
},
{
"os": "windows-2019",
"features": "cuda",
"target": "x86_64-pc-windows-msvc",
"artifact_name": "windows-x64-cuda",
"c_release_format": "plain-cdylib",
"whl_local_version": "cuda",
"can_skip_in_simple_test": true
},
{
"os": "windows-2019",
"features": "",
"target": "i686-pc-windows-msvc",
"artifact_name": "windows-x86-cpu",
"c_release_format": "plain-cdylib",
"whl_local_version": "cpu",
"can_skip_in_simple_test": true
},
{
"os": "ubuntu-20.04",
"features": "",
"target": "x86_64-unknown-linux-gnu",
"artifact_name": "linux-x64-cpu",
"artifact_name": "windows-x86",
"c_release_format": "plain-cdylib",
"whl_local_version": "cpu",
"python_whl": true,
"can_skip_in_simple_test": true
},
{
"os": "ubuntu-20.04",
"features": "cuda",
"target": "x86_64-unknown-linux-gnu",
"artifact_name": "linux-x64-gpu",
"artifact_name": "linux-x64",
"c_release_format": "plain-cdylib",
"whl_local_version": "cuda",
"python_whl": true,
"can_skip_in_simple_test": false
},
{
"os": "ubuntu-20.04",
"features": "",
"target": "aarch64-unknown-linux-gnu",
"artifact_name": "linux-arm64-cpu",
"artifact_name": "linux-arm64",
"c_release_format": "plain-cdylib",
"whl_local_version": "cpu",
"python_whl": true,
"can_skip_in_simple_test": true
},
{
"os": "ubuntu-20.04",
"features": "",
"target": "aarch64-linux-android",
"artifact_name": "android-arm64-cpu",
"artifact_name": "android-arm64",
"c_release_format": "plain-cdylib",
"python_whl": false,
"can_skip_in_simple_test": true
},
{
"os": "ubuntu-20.04",
"features": "",
"target": "x86_64-linux-android",
"artifact_name": "android-x86_64-cpu",
"artifact_name": "android-x86_64",
"c_release_format": "plain-cdylib",
"python_whl": false,
"can_skip_in_simple_test": true
},
{
"os": "macos-12",
"features": "",
"target": "aarch64-apple-darwin",
"artifact_name": "osx-arm64-cpu",
"artifact_name": "osx-arm64",
"c_release_format": "plain-cdylib",
"whl_local_version": "cpu",
"python_whl": true,
"can_skip_in_simple_test": false
},
{
"os": "macos-12",
"features": "",
"target": "x86_64-apple-darwin",
"artifact_name": "osx-x64-cpu",
"artifact_name": "osx-x64",
"c_release_format": "plain-cdylib",
"whl_local_version": "cpu",
"python_whl": true,
"can_skip_in_simple_test": true
},
{
"os": "macos-12",
"features": "",
"target": "aarch64-apple-ios",
"artifact_name": "ios-arm64-cpu",
"c_release_format": "ios-xcframework",
"python_whl": false,
"can_skip_in_simple_test": true
},
{
"os": "macos-12",
"features": "",
"target": "aarch64-apple-ios-sim",
"artifact_name": "ios-arm64-cpu-sim",
"c_release_format": "ios-xcframework",
"python_whl": false,
"can_skip_in_simple_test": true
},
{
"os": "macos-12",
"features": "",
"target": "x86_64-apple-ios",
"artifact_name": "ios-x64-cpu",
"c_release_format": "ios-xcframework",
"python_whl": false,
"can_skip_in_simple_test": true
}
]'
Expand Down Expand Up @@ -216,7 +183,7 @@ jobs:
git -c user.name=dummy -c [email protected] merge FETCH_HEAD
) > /dev/null 2>&1
- name: Set up Python 3.8
if: matrix.whl_local_version
if: matrix.python_whl
uses: actions/setup-python@v5
with:
python-version: "3.8"
Expand Down Expand Up @@ -258,7 +225,7 @@ jobs:
- name: set cargo version
run: |
cargo set-version "$VERSION" --exclude voicevox_core_python_api --exclude downloader --exclude xtask
if ${{ !!matrix.whl_local_version }}; then cargo set-version "$VERSION+"${{ matrix.whl_local_version }} -p voicevox_core_python_api; fi
if ${{ matrix.python_whl }}; then cargo set-version "$VERSION" -p voicevox_core_python_api; fi
- name: cache target
uses: Swatinem/rust-cache@v2
if: ${{ !inputs.is_production }}
Expand All @@ -270,7 +237,7 @@ jobs:
ios-xcframework) linking=link-onnxruntime ;;
esac
function build() {
cargo build -p voicevox_core_c_api -vv --features "$linking",${{ matrix.features }} --target ${{ matrix.target }} --release
cargo build -p voicevox_core_c_api -vv --features "$linking" --target ${{ matrix.target }} --release
}
if ${{ !inputs.is_production }}; then
build
Expand All @@ -280,15 +247,15 @@ jobs:
env:
RUSTFLAGS: -C panic=abort
- name: build voicevox_core_python_api
if: matrix.whl_local_version
if: matrix.python_whl
id: build-voicevox-core-python-api
run: |
rm -rf ./target/wheels
pip install --upgrade poetry
poetry config virtualenvs.create false
(cd crates/voicevox_core_python_api && poetry install --with dev)
function build() {
maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --features ${{ matrix.features }}, --target ${{ matrix.target }} --release
maturin build --manifest-path ./crates/voicevox_core_python_api/Cargo.toml --target ${{ matrix.target }} --release
}
if ${{ !inputs.is_production }}; then
build
Expand All @@ -300,7 +267,7 @@ jobs:
if: contains(matrix.target, 'android')
run: |
function build() {
cargo build -p voicevox_core_java_api -vv --features ${{ matrix.features }}, --target ${{ matrix.target }} --release
cargo build -p voicevox_core_java_api -vv --target ${{ matrix.target }} --release
}
if ${{ !inputs.is_production }}; then
build
Expand All @@ -318,9 +285,6 @@ jobs:
> "artifact/${{ env.ASSET_NAME }}/voicevox_core.h"
cp -v target/${{ matrix.target }}/release/*voicevox_core.{dll,so,dylib} "artifact/${{ env.ASSET_NAME }}" || true
cp -v target/${{ matrix.target }}/release/voicevox_core.dll.lib "artifact/${{ env.ASSET_NAME }}/voicevox_core.lib" || true
cp -v -n target/${{ matrix.target }}/release/{,lib}onnxruntime*.{dll,so.*,so,dylib} "artifact/${{ env.ASSET_NAME }}" || true
# libonnxruntimeについてはバージョン付のshared libraryを使用するためバージョンがついてないものを削除する
rm -f artifact/${{ env.ASSET_NAME }}/libonnxruntime.{so,dylib}
cp -v README.md "artifact/${{ env.ASSET_NAME }}/README.txt"
echo "${{ env.VERSION }}" > "artifact/${{ env.ASSET_NAME }}/VERSION"
Expand Down Expand Up @@ -354,7 +318,7 @@ jobs:
${{ env.ASSET_NAME }}.zip
target_commitish: ${{ github.sha }}
- name: Upload Python whl to Release
if: fromJson(needs.config.outputs.deploy) && matrix.whl_local_version
if: fromJson(needs.config.outputs.deploy) && matrix.python_whl
uses: softprops/action-gh-release@v2
with:
prerelease: true
Expand Down Expand Up @@ -475,17 +439,17 @@ jobs:
run:
cargo set-version "$VERSION" -p voicevox_core_java_api

- name: "Download artifact (android-arm64-cpu)"
- name: "Download artifact (android-arm64)"
uses: actions/download-artifact@v4
with:
name: voicevox_core_java_api-android-arm64-cpu
path: artifact/android-arm64-cpu
name: voicevox_core_java_api-android-arm64
path: artifact/android-arm64

- name: "Download artifact (android-x86_64-cpu)"
- name: "Download artifact (android-x86_64)"
uses: actions/download-artifact@v4
with:
name: voicevox_core_java_api-android-x86_64-cpu
path: artifact/android-x86_64-cpu
name: voicevox_core_java_api-android-x86_64
path: artifact/android-x86_64

- name: Print tree
run: tree artifact
Expand All @@ -494,8 +458,8 @@ jobs:
run: |
rm -rf crates/voicevox_core_java_api/lib/src/main/resources/dll
cat <<EOF | while read -r line; do
android-arm64-cpu|arm64-v8a
android-x86_64-cpu|x86_64
android-arm64|arm64-v8a
android-x86_64|x86_64
EOF
IFS='|' read -r artifact_name target <<< "$line"
mkdir "crates/voicevox_core_java_api/lib/src/main/resources/jniLibs/${target}/"
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/download_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ jobs:
open_jtalk_dic_utf_8-1.11
- name: DirectML option確認
os: windows-latest
download_command: cargo run -vv -p downloader -- --device directml
download_command: cargo run -vv -p downloader -- --devices directml
download_dir: voicevox_core
check_items: |
voicevox_core.dll
Expand All @@ -121,7 +121,7 @@ jobs:
*curand*
- name: DirectMLかつMin option確認
os: windows-latest
download_command: cargo run -vv -p downloader -- --device directml --min
download_command: cargo run -vv -p downloader -- --devices directml --min
download_dir: voicevox_core
check_items: |
voicevox_core.dll
Expand All @@ -139,7 +139,7 @@ jobs:
open_jtalk_dic_utf_8-1.11
- name: cuda option確認
os: windows-latest
download_command: cargo run -vv -p downloader -- --device cuda
download_command: cargo run -vv -p downloader -- --devices cuda
download_dir: voicevox_core
check_items: |
voicevox_core.dll
Expand All @@ -161,7 +161,7 @@ jobs:
*directml*
- name: cudaかつmin option確認
os: windows-latest
download_command: cargo run -vv -p downloader -- --device cuda --min
download_command: cargo run -vv -p downloader -- --devices cuda --min
download_dir: voicevox_core
check_items: |
voicevox_core.dll
Expand Down
28 changes: 13 additions & 15 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,10 @@ jobs:
with:
python-version: "3.8"
- uses: Swatinem/rust-cache@v2
- run: cargo clippy -vv --features directml,cuda --tests -- -D clippy::all -D warnings --no-deps
- run: cargo clippy -vv --features directml,cuda -- -D clippy::all -D warnings --no-deps
- run: cargo clippy -vv -p voicevox_core -p voicevox_core_c_api --features link-onnxruntime,directml,cuda --tests -- -D clippy::all -D warnings --no-deps
- run: cargo clippy -vv -p voicevox_core -p voicevox_core_c_api --features link-onnxruntime,directml,cuda -- -D clippy::all -D warnings --no-deps
- run: cargo clippy -vv --tests -- -D clippy::all -D warnings --no-deps
- run: cargo clippy -vv -- -D clippy::all -D warnings --no-deps
- run: cargo clippy -vv -p voicevox_core -p voicevox_core_c_api --features link-onnxruntime --tests -- -D clippy::all -D warnings --no-deps
- run: cargo clippy -vv -p voicevox_core -p voicevox_core_c_api --features link-onnxruntime -- -D clippy::all -D warnings --no-deps
- run: cargo fmt -- --check

rust-unit-test:
Expand All @@ -88,7 +88,7 @@ jobs:
with:
key: "cargo-unit-test-cache"
- name: Run cargo unit test
run: RUST_BACKTRACE=full cargo test --lib --bins -vv --features , -- --include-ignored
run: RUST_BACKTRACE=full cargo test --lib --bins -vv -- --include-ignored
- name: Run cargo documentation test
run: RUST_BACKTRACE=full cargo test --doc -vv

Expand All @@ -101,14 +101,12 @@ jobs:
id: strategy-matrix
run: |
includes='[
{ "os": "windows-2019", "features": "", "can_skip_in_simple_test": true },
{ "os": "windows-2022", "features": "", "can_skip_in_simple_test": true },
{ "os": "windows-2019", "features": "directml", "can_skip_in_simple_test": false },
{ "os": "windows-2022", "features": "directml", "can_skip_in_simple_test": true },
{ "os": "macos-12", "features": "", "can_skip_in_simple_test": false },
{ "os": "macos-13", "features": "", "can_skip_in_simple_test": true },
{ "os": "ubuntu-20.04", "features": "", "can_skip_in_simple_test": false },
{ "os": "ubuntu-22.04", "features": "", "can_skip_in_simple_test": true }
{ "os": "windows-2019", "can_skip_in_simple_test": true },
{ "os": "windows-2022", "can_skip_in_simple_test": true },
{ "os": "macos-12", "can_skip_in_simple_test": false },
{ "os": "macos-13", "can_skip_in_simple_test": true },
{ "os": "ubuntu-20.04", "can_skip_in_simple_test": false },
{ "os": "ubuntu-22.04", "can_skip_in_simple_test": true }
]'
# FIXME: composite action に切り出す
Expand All @@ -135,9 +133,9 @@ jobs:
uses: ./.github/actions/rust-toolchain-from-file
- uses: Swatinem/rust-cache@v2
with:
key: "cargo-integration-test-cache-${{ matrix.features }}-${{ matrix.os }}"
key: "cargo-integration-test-cache-${{ matrix.os }}"
- name: Run cargo integration test (load-onnxruntime)
run: RUST_BACKTRACE=full cargo test --test "*" -vv --features ,${{ matrix.features }} -- --include-ignored
run: RUST_BACKTRACE=full cargo test --test "*" -vv -- --include-ignored

c-header:
runs-on: ubuntu-latest
Expand Down
Loading

0 comments on commit bf8cdb8

Please sign in to comment.