Add GGUF Qwen 2 (#860)

* Add GGUF Qwen 2 * Update readme
EricLBuehler · Oct 16, 2024 · d82306d · d82306d
1 parent 25174b4
commit d82306d
Show file tree

Hide file tree

Showing 7 changed files with 459 additions and 1 deletion.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -415,6 +415,7 @@ If you do not specify the architecture, an attempt will be made to use the model
 - `phi2`
 - `phi3`
 - `starcoder2`
+- `qwen2`
 
 **With adapters:**
 

diff --git a/mistralrs-core/src/gguf/mod.rs b/mistralrs-core/src/gguf/mod.rs
@@ -26,6 +26,7 @@ pub enum GGUFArchitecture {
     Phi2,
     Phi3,
     Starcoder2,
+    Qwen2,
 }
 
 // Wraps from_str() for some convenience:

diff --git a/mistralrs-core/src/models/mod.rs b/mistralrs-core/src/models/mod.rs
@@ -9,6 +9,7 @@ pub(crate) mod phi3_5_moe;
 pub(crate) mod quantized_llama;
 pub(crate) mod quantized_phi2;
 pub(crate) mod quantized_phi3;
+pub(crate) mod quantized_qwen2;
 pub(crate) mod quantized_starcoder2;
 pub(crate) mod qwen2;
 pub(crate) mod starcoder2;