Merge pull request #92 from lean-dojo/peiyang

Fix minor bug: tokenizer typo in models.py
lean-dojo · Jul 1, 2024 · 96e3cbe · 96e3cbe
2 parents 7cd54f6 + 646215e
commit 96e3cbe
Showing 1 changed file with 8 additions and 8 deletions.
diff --git a/python/models.py b/python/models.py
@@ -48,7 +48,7 @@ def __init__(
         length_penalty: float = 0.0,
         device: str = "cpu",
     ) -> None:
-        self.tokenzier = AutoTokenizer.from_pretrained(name)
+        self.tokenizer = AutoTokenizer.from_pretrained(name)
         if device == "auto":
             device = get_cuda_if_available()
         else:
@@ -60,7 +60,7 @@ def __init__(
         self.length_penalty = length_penalty
 
     def generate(self, input: str, target_prefix: str = "") -> List[Tuple[str, float]]:
-        tokenized_input = self.tokenzier(input + target_prefix, return_tensors="pt")
+        tokenized_input = self.tokenizer(input + target_prefix, return_tensors="pt")
         output = self.model.generate(
             tokenized_input.input_ids.to(self.device),
             max_length=self.max_length,
@@ -72,7 +72,7 @@ def generate(self, input: str, target_prefix: str = "") -> List[Tuple[str, float
             return_dict_in_generate=True,
             output_scores=True,
         )
-        raw_outputs = self.tokenzier.batch_decode(
+        raw_outputs = self.tokenizer.batch_decode(
             output.sequences, skip_special_tokens=True
         )
         outputs = []
@@ -113,7 +113,7 @@ def __init__(
         length_penalty: float = 0.0,
         device: str = "cpu",
     ) -> None:
-        self.tokenzier = AutoTokenizer.from_pretrained(name)
+        self.tokenizer = AutoTokenizer.from_pretrained(name)
         if device == "auto":
             device = get_cuda_if_available()
         else:
@@ -128,7 +128,7 @@ def generate(self, input: str, target_prefix: str = "") -> List[Tuple[str, float
         assert (
             target_prefix == ""
         ), "target_prefix is not supported by encoder-decoder Transformer"
-        tokenized_input = self.tokenzier(input, return_tensors="pt")
+        tokenized_input = self.tokenizer(input, return_tensors="pt")
         output = self.model.generate(
             tokenized_input.input_ids.to(self.device),
             max_length=self.max_length,
@@ -140,15 +140,15 @@ def generate(self, input: str, target_prefix: str = "") -> List[Tuple[str, float
             return_dict_in_generate=True,
             output_scores=True,
         )
-        raw_outputs = self.tokenzier.batch_decode(
+        raw_outputs = self.tokenizer.batch_decode(
             output.sequences, skip_special_tokens=True
         )
         return list(zip(raw_outputs, output.sequences_scores.exp().tolist()))
 
 
 class EncoderOnlyTransformer(Encoder, Transformer):
     def __init__(self, name: str, device: str = "cpu") -> None:
-        self.tokenzier = AutoTokenizer.from_pretrained(name)
+        self.tokenizer = AutoTokenizer.from_pretrained(name)
         if device == "auto":
             device = get_cuda_if_available()
         else:
@@ -158,7 +158,7 @@ def __init__(self, name: str, device: str = "cpu") -> None:
 
     @torch.no_grad()
     def encode(self, input: str) -> np.ndarray:
-        tokenized_input = self.tokenzier(input, return_tensors="pt")
+        tokenized_input = self.tokenizer(input, return_tensors="pt")
         hidden_state = self.model(
             tokenized_input.input_ids.to(self.device)
         ).last_hidden_state