From 81149b6059dc89943825bd67ad53f1edbf2f1dea Mon Sep 17 00:00:00 2001 From: kumapo Date: Sat, 28 Oct 2023 17:17:14 +0900 Subject: [PATCH] need additional_special_tokens argument for HFLM initializer --- lm_eval/models/gpt2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lm_eval/models/gpt2.py b/lm_eval/models/gpt2.py index c69b614255..caae4ad611 100644 --- a/lm_eval/models/gpt2.py +++ b/lm_eval/models/gpt2.py @@ -19,6 +19,7 @@ def __init__( load_in_8bit: Optional[bool] = False, trust_remote_code: Optional[bool] = False, use_fast: Optional[bool] = True, + additional_special_tokens: Optional[str] = None, ): super().__init__() @@ -64,6 +65,7 @@ def __init__( revision=revision, trust_remote_code=trust_remote_code, use_fast=use_fast, + additional_special_tokens=additional_special_tokens, ) self.vocab_size = self.tokenizer.vocab_size