Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix params.json for llama models #6362

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+8 −3
Open

Fix params.json for llama models #6362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions examples/models/llama/llama_transformer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ class ModelArgs:
bos_count: int = -1 # i.e., a single EOS is used as BOS
eos_count: int = 2

quantization_args: Optional[dict] = None
lora_args: Optional[dict] = None

def __post_init__(self):
if self.n_kv_heads is None:
self.n_kv_heads = self.n_heads
Expand Down
8 changes: 5 additions & 3 deletions examples/models/llama/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ def __init__(self, **kwargs):
)
elif hasattr(self.args, "use_spin_quant") and self.args.use_spin_quant:
print("Using SPIN quantization.")
self._transform_for_pre_quantization(checkpoint)
self._transform_for_pre_quantization(checkpoint, model_args)

from .source_transformation.pre_quantization import (
sanitize_checkpoint_from_pre_quantization,
Expand All @@ -174,8 +174,9 @@ def __init__(self, **kwargs):
sanitize_checkpoint_from_pre_quantization(checkpoint)
elif hasattr(self.args, "use_qat") and self.args.use_qat:
print("Using QAT quantization.")
self._transform_for_pre_quantization(checkpoint)
self._transform_for_pre_quantization(checkpoint, model_args)
if hasattr(self.args, "use_lora") and self.args.use_lora:
assert model_args.lora_args["rank"] == self.args.use_lora
from .source_transformation.lora import (
transform_linear_for_lora_after_quantization,
)
Expand Down Expand Up @@ -251,7 +252,7 @@ def get_example_inputs_kvcache_sdpa(self):
), # start_pos, what token of output are we on.
)

def _transform_for_pre_quantization(self, checkpoint):
def _transform_for_pre_quantization(self, checkpoint, model_args):
assert hasattr(self.args, "preq_mode"), "preq_mode must be specified"
assert self.args.preq_mode in [
"8da4w",
Expand All @@ -264,6 +265,7 @@ def _transform_for_pre_quantization(self, checkpoint):
from .source_transformation.pre_quantization import (
transform_linear_for_pre_quantization,
)
assert self.args.preq_group_size == model_args.quantization_args["group_size"]

mapping = {
"fp32": torch.float32,
Expand Down
Loading