From d4c1df404fd9a245b115ed095ebd2c62e70e90b2 Mon Sep 17 00:00:00 2001 From: Ulan Degenbaev Date: Wed, 12 Jun 2024 17:54:47 +0000 Subject: [PATCH 1/3] Use Wasm SIMD in the image classification example The mainnet and dfx version 0.20.2-beta.0 support Wasm SIMD. Sonos Tract also supports Wasm SIMD on `master`. This PR updates the Sonos Tract dependency to the version that supports Wasm SIMD. Now image classification can run in a query. The example is modified to add an option for running as a query or update. --- rust/image-classification/Cargo.lock | 83 +++++++++++++++---- rust/image-classification/README.md | 22 ++--- rust/image-classification/dfx.json | 3 +- .../src/backend/Cargo.toml | 2 +- .../src/backend/backend.did | 1 + .../src/backend/src/lib.rs | 20 ++++- .../src/declarations/backend/backend.did | 1 + .../src/declarations/backend/backend.did.d.ts | 1 + .../src/declarations/backend/backend.did.js | 5 ++ .../src/frontend/assets/main.css | 9 +- .../src/frontend/src/index.html | 9 ++ .../src/frontend/src/index.js | 11 ++- 12 files changed, 131 insertions(+), 36 deletions(-) diff --git a/rust/image-classification/Cargo.lock b/rust/image-classification/Cargo.lock index f88ef3982..326d74b90 100644 --- a/rust/image-classification/Cargo.lock +++ b/rust/image-classification/Cargo.lock @@ -255,6 +255,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crunchy" version = "0.2.2" @@ -334,6 +359,12 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "dyn-hash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a650a461c6a8ff1ef205ed9a2ad56579309853fecefc2423f73dced342f92258" + [[package]] name = "either" version = "1.10.0" @@ -1021,6 +1052,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1331,8 +1382,8 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tract-core" -version = "0.21.2-pre" -source = "git+https://github.com/sonos/tract#43b3a6689816c0797157c75e90435afa6426fd1f" +version = "0.21.6-pre" +source = "git+https://github.com/sonos/tract?rev=2a2914ac29390cc08963301c9f3d437b52dd321a#2a2914ac29390cc08963301c9f3d437b52dd321a" dependencies = [ "anyhow", "bit-set", @@ -1355,10 +1406,12 @@ dependencies = [ [[package]] name = "tract-data" -version = "0.21.2-pre" -source = "git+https://github.com/sonos/tract#43b3a6689816c0797157c75e90435afa6426fd1f" +version = "0.21.6-pre" +source = "git+https://github.com/sonos/tract?rev=2a2914ac29390cc08963301c9f3d437b52dd321a#2a2914ac29390cc08963301c9f3d437b52dd321a" dependencies = [ "anyhow", + "downcast-rs", + "dyn-hash", "half", "itertools 0.12.1", "lazy_static", @@ -1374,8 +1427,8 @@ dependencies = [ [[package]] name = "tract-hir" -version = "0.21.2-pre" -source = "git+https://github.com/sonos/tract#43b3a6689816c0797157c75e90435afa6426fd1f" +version = "0.21.6-pre" +source = "git+https://github.com/sonos/tract?rev=2a2914ac29390cc08963301c9f3d437b52dd321a#2a2914ac29390cc08963301c9f3d437b52dd321a" dependencies = [ "derive-new", "log", @@ -1384,13 +1437,14 @@ dependencies = [ [[package]] name = "tract-linalg" -version = "0.21.2-pre" -source = "git+https://github.com/sonos/tract#43b3a6689816c0797157c75e90435afa6426fd1f" +version = "0.21.6-pre" +source = "git+https://github.com/sonos/tract?rev=2a2914ac29390cc08963301c9f3d437b52dd321a#2a2914ac29390cc08963301c9f3d437b52dd321a" dependencies = [ "cc", "derive-new", "downcast-rs", "dyn-clone", + "dyn-hash", "half", "lazy_static", "liquid", @@ -1398,6 +1452,7 @@ dependencies = [ "log", "num-traits", "paste", + "rayon", "scan_fmt", "smallvec", "time", @@ -1408,8 +1463,8 @@ dependencies = [ [[package]] name = "tract-nnef" -version = "0.21.2-pre" -source = "git+https://github.com/sonos/tract#43b3a6689816c0797157c75e90435afa6426fd1f" +version = "0.21.6-pre" +source = "git+https://github.com/sonos/tract?rev=2a2914ac29390cc08963301c9f3d437b52dd321a#2a2914ac29390cc08963301c9f3d437b52dd321a" dependencies = [ "byteorder", "flate2", @@ -1422,8 +1477,8 @@ dependencies = [ [[package]] name = "tract-onnx" -version = "0.21.2-pre" -source = "git+https://github.com/sonos/tract#43b3a6689816c0797157c75e90435afa6426fd1f" +version = "0.21.6-pre" +source = "git+https://github.com/sonos/tract?rev=2a2914ac29390cc08963301c9f3d437b52dd321a#2a2914ac29390cc08963301c9f3d437b52dd321a" dependencies = [ "bytes", "derive-new", @@ -1439,8 +1494,8 @@ dependencies = [ [[package]] name = "tract-onnx-opl" -version = "0.21.2-pre" -source = "git+https://github.com/sonos/tract#43b3a6689816c0797157c75e90435afa6426fd1f" +version = "0.21.6-pre" +source = "git+https://github.com/sonos/tract?rev=2a2914ac29390cc08963301c9f3d437b52dd321a#2a2914ac29390cc08963301c9f3d437b52dd321a" dependencies = [ "getrandom", "log", diff --git a/rust/image-classification/README.md b/rust/image-classification/README.md index 939e28d61..500a95bc0 100644 --- a/rust/image-classification/README.md +++ b/rust/image-classification/README.md @@ -3,18 +3,12 @@ This is an ICP smart contract that accepts an image from the user and runs image classification inference. The smart contract consists of two canisters: -- the backend canister embeds the [the Tract ONNX inference engine](https://github.com/sonos/tract) with [the MobileNet v2-7 model](https://github.com/onnx/models/tree/main/validated/vision/classification/mobilenet). It provides a `classify()` endpoint for the frontend code to call. +- the backend canister embeds the [the Tract ONNX inference engine](https://github.com/sonos/tract) with [the MobileNet v2-7 model](https://github.com/onnx/models/tree/main/validated/vision/classification/mobilenet). + It provides `classify()` and `classify_query()` endpoints for the frontend code to call. + The former endpoint is used for replicated execution (running on all nodes) whereas the latter runs only on a single node. - the frontend canister contains the Web assets such as HTML, JS, CSS that are served to the browser. -Note that currently Wasm execution is not optimized for this workload. -A single call executes about 24B instructions (~10s). - -This is expected to improve in the future with: - -- faster deterministic floating-point operations. -- Wasm SIMD (Single-Instruction Multiple Data). - -The ICP mainnet subnets and `dfx` running a replica version older than [463296](https://dashboard.internetcomputer.org/release/463296c0bc82ad5999b70245e5f125c14ba7d090) may fail with an instruction-limit-exceeded error. +This example uses Wasm SIMD instructions that are available in `dfx` version `0.20.2-beta.0` or newer. # Dependencies @@ -45,6 +39,12 @@ Install NodeJS dependencies for the frontend: npm install ``` +Install `wasm-opt`: + +``` +cargo install wasm-opt +``` + # Build ``` @@ -52,5 +52,5 @@ dfx start --background dfx deploy ``` -If the deployment is successfull, the it will show the `frontend` URL. +If the deployment is successful, the it will show the `frontend` URL. Open that URL in browser to interact with the smart contract. diff --git a/rust/image-classification/dfx.json b/rust/image-classification/dfx.json index b94cbc412..71440e35c 100644 --- a/rust/image-classification/dfx.json +++ b/rust/image-classification/dfx.json @@ -7,7 +7,8 @@ "wasm": "target/wasm32-wasi/release/backend-ic.wasm", "build": [ "cargo build --release --target=wasm32-wasi", - "wasi2ic ./target/wasm32-wasi/release/backend.wasm ./target/wasm32-wasi/release/backend-ic.wasm" + "wasi2ic ./target/wasm32-wasi/release/backend.wasm ./target/wasm32-wasi/release/backend-ic.wasm", + "wasm-opt -Os -o ./target/wasm32-wasi/release/backend-ic.wasm ./target/wasm32-wasi/release/backend-ic.wasm" ] }, diff --git a/rust/image-classification/src/backend/Cargo.toml b/rust/image-classification/src/backend/Cargo.toml index 048bf833e..03a44b316 100644 --- a/rust/image-classification/src/backend/Cargo.toml +++ b/rust/image-classification/src/backend/Cargo.toml @@ -16,6 +16,6 @@ prost = "0.11.0" prost-types = "0.11.0" image = { version = "0.24", features = ["png"], default-features = false } serde = { version = "1.0", features = ["derive"] } -tract-onnx = { git = "https://github.com/sonos/tract", version = "=0.21.2-pre" } +tract-onnx = { git = "https://github.com/sonos/tract", rev = "2a2914ac29390cc08963301c9f3d437b52dd321a" } ic-stable-structures = "0.6" ic-wasi-polyfill = { git = "https://github.com/wasm-forge/ic-wasi-polyfill", version = "0.3.17" } diff --git a/rust/image-classification/src/backend/backend.did b/rust/image-classification/src/backend/backend.did index 6837d0335..6fa4eda83 100644 --- a/rust/image-classification/src/backend/backend.did +++ b/rust/image-classification/src/backend/backend.did @@ -14,4 +14,5 @@ type ClassificationResult = variant { service : { "classify": (image: blob) -> (ClassificationResult); + "classify_query": (image: blob) -> (ClassificationResult) query; } diff --git a/rust/image-classification/src/backend/src/lib.rs b/rust/image-classification/src/backend/src/lib.rs index 04b227d94..9f5d8e8ea 100644 --- a/rust/image-classification/src/backend/src/lib.rs +++ b/rust/image-classification/src/backend/src/lib.rs @@ -1,6 +1,9 @@ -use std::cell::RefCell; use candid::{CandidType, Deserialize}; -use ic_stable_structures::{memory_manager::{MemoryId, MemoryManager}, DefaultMemoryImpl}; +use ic_stable_structures::{ + memory_manager::{MemoryId, MemoryManager}, + DefaultMemoryImpl, +}; +use std::cell::RefCell; mod onnx; @@ -31,7 +34,7 @@ enum ClassificationResult { Err(ClassificationError), } -#[ic_cdk::update] +#[ic_cdk::query] fn classify(image: Vec) -> ClassificationResult { let result = match onnx::classify(image) { Ok(result) => ClassificationResult::Ok(result), @@ -42,6 +45,17 @@ fn classify(image: Vec) -> ClassificationResult { result } +#[ic_cdk::query] +fn classify_query(image: Vec) -> ClassificationResult { + let result = match onnx::classify(image) { + Ok(result) => ClassificationResult::Ok(result), + Err(err) => ClassificationResult::Err(ClassificationError { + message: err.to_string(), + }), + }; + result +} + #[ic_cdk::init] fn init() { let wasi_memory = MEMORY_MANAGER.with(|m| m.borrow().get(WASI_MEMORY_ID)); diff --git a/rust/image-classification/src/declarations/backend/backend.did b/rust/image-classification/src/declarations/backend/backend.did index 6837d0335..6fa4eda83 100644 --- a/rust/image-classification/src/declarations/backend/backend.did +++ b/rust/image-classification/src/declarations/backend/backend.did @@ -14,4 +14,5 @@ type ClassificationResult = variant { service : { "classify": (image: blob) -> (ClassificationResult); + "classify_query": (image: blob) -> (ClassificationResult) query; } diff --git a/rust/image-classification/src/declarations/backend/backend.did.d.ts b/rust/image-classification/src/declarations/backend/backend.did.d.ts index 7680fb218..418d8e98b 100644 --- a/rust/image-classification/src/declarations/backend/backend.did.d.ts +++ b/rust/image-classification/src/declarations/backend/backend.did.d.ts @@ -8,6 +8,7 @@ export type ClassificationResult = { 'Ok' : Array } | { 'Err' : ClassificationError }; export interface _SERVICE { 'classify' : ActorMethod<[Uint8Array | number[]], ClassificationResult>, + 'classify_query' : ActorMethod<[Uint8Array | number[]], ClassificationResult>, } export declare const idlFactory: IDL.InterfaceFactory; export declare const init: (args: { IDL: typeof IDL }) => IDL.Type[]; diff --git a/rust/image-classification/src/declarations/backend/backend.did.js b/rust/image-classification/src/declarations/backend/backend.did.js index 197a8029d..d6ab35a3a 100644 --- a/rust/image-classification/src/declarations/backend/backend.did.js +++ b/rust/image-classification/src/declarations/backend/backend.did.js @@ -10,6 +10,11 @@ export const idlFactory = ({ IDL }) => { }); return IDL.Service({ 'classify' : IDL.Func([IDL.Vec(IDL.Nat8)], [ClassificationResult], []), + 'classify_query' : IDL.Func( + [IDL.Vec(IDL.Nat8)], + [ClassificationResult], + ['query'], + ), }); }; export const init = ({ IDL }) => { return []; }; diff --git a/rust/image-classification/src/frontend/assets/main.css b/rust/image-classification/src/frontend/assets/main.css index ae094b249..5d63119d2 100644 --- a/rust/image-classification/src/frontend/assets/main.css +++ b/rust/image-classification/src/frontend/assets/main.css @@ -47,6 +47,7 @@ textarea { flex-flow: row; justify-content: left; align-items: center; + margin-bottom: 20px; } .toggle-switch { @@ -71,7 +72,6 @@ textarea { bottom: 0; background-color: #ccc; border-radius: 18px; - transition: 0.4s; } .slider:before { @@ -83,7 +83,6 @@ textarea { bottom: 1px; background-color: white; border-radius: 50%; - transition: 0.4s; } input:checked+.slider { @@ -129,15 +128,15 @@ li { @keyframes astrodance { 0% { - transform: translate(-50%,-50%) rotate(-20deg) + transform: translate(-50%, -50%) rotate(-20deg) } 50% { - transform: translate(-50%,-50%) rotate(10deg) + transform: translate(-50%, -50%) rotate(10deg) } to { - transform: translate(-50%,-50%) rotate(-20deg) + transform: translate(-50%, -50%) rotate(-20deg) } } diff --git a/rust/image-classification/src/frontend/src/index.html b/rust/image-classification/src/frontend/src/index.html index 014236805..c3f25f9ba 100644 --- a/rust/image-classification/src/frontend/src/index.html +++ b/rust/image-classification/src/frontend/src/index.html @@ -20,6 +20,15 @@

ICP image classification

+
diff --git a/rust/image-classification/src/frontend/src/index.js b/rust/image-classification/src/frontend/src/index.js index 2e8bc9368..03470e2bf 100644 --- a/rust/image-classification/src/frontend/src/index.js +++ b/rust/image-classification/src/frontend/src/index.js @@ -11,15 +11,22 @@ async function classify(event) { const message = document.getElementById("message"); const loader = document.getElementById("loader"); const img = document.getElementById("image"); + const repl_option = document.getElementById("replicated_option"); button.disabled = true; button.className = "clean-button invisible"; + repl_option.className = "option invisible"; message.innerText = "Computing..."; loader.className = "loader"; try { const blob = await resize(img); - const result = await backend.classify(new Uint8Array(blob)); + let result;; + if (document.getElementById("replicated").checked) { + result = await backend.classify(new Uint8Array(blob)); + } else { + result = await backend.classify_query(new Uint8Array(blob)); + } if (result.Ok) { render(message, result.Ok); } else { @@ -77,6 +84,7 @@ async function onImageChange(event) { const button = document.getElementById("classify"); const message = document.getElementById("message"); const img = document.getElementById("image"); + const repl_option = document.getElementById("replicated_option"); try { const file = event.target.files[0]; const url = await toDataURL(file); @@ -89,6 +97,7 @@ async function onImageChange(event) { button.disabled = false; button.className = "clean-button"; message.innerText = ""; + repl_option.className = "option" return false; } From 98f595549bc1f15bf578e99b1135ff4261fd1707 Mon Sep 17 00:00:00 2001 From: Ulan Degenbaev Date: Thu, 13 Jun 2024 15:04:19 +0000 Subject: [PATCH 2/3] Add build.sh with RUSTFLAGS --- rust/image-classification/dfx.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/rust/image-classification/dfx.json b/rust/image-classification/dfx.json index 71440e35c..4dcae4122 100644 --- a/rust/image-classification/dfx.json +++ b/rust/image-classification/dfx.json @@ -5,12 +5,7 @@ "package": "backend", "type": "custom", "wasm": "target/wasm32-wasi/release/backend-ic.wasm", - "build": [ - "cargo build --release --target=wasm32-wasi", - "wasi2ic ./target/wasm32-wasi/release/backend.wasm ./target/wasm32-wasi/release/backend-ic.wasm", - "wasm-opt -Os -o ./target/wasm32-wasi/release/backend-ic.wasm ./target/wasm32-wasi/release/backend-ic.wasm" - ] - + "build": [ "bash build.sh" ] }, "frontend": { "dependencies": [ From b6ac19822add7a5f0f97c9d78d7cf9abad76a832 Mon Sep 17 00:00:00 2001 From: Ulan Degenbaev Date: Thu, 13 Jun 2024 15:05:54 +0000 Subject: [PATCH 3/3] Fix comment --- rust/image-classification/src/frontend/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/image-classification/src/frontend/src/index.js b/rust/image-classification/src/frontend/src/index.js index 03470e2bf..1ae61ecf5 100644 --- a/rust/image-classification/src/frontend/src/index.js +++ b/rust/image-classification/src/frontend/src/index.js @@ -21,7 +21,7 @@ async function classify(event) { try { const blob = await resize(img); - let result;; + let result; if (document.getElementById("replicated").checked) { result = await backend.classify(new Uint8Array(blob)); } else {