various fixes

eval_llm/WIKITEXT2/run_wikitext-2_benchmark.py

@@ -1,24 +1,70 @@
 import copy
 import numpy as np
 import time
+import re
 from onmt.inference_engine import InferenceEnginePY
 import onmt.opts as opts
 from onmt.utils.logging import init_logger
 from onmt.utils.parse import ArgumentParser
 from onmt.utils.misc import use_gpu, set_random_seed
 
 
+def wikitext_detokenizer(line):
+    string = line
+    # contractions
+    string = string.replace("s '", "s'")
+    string = re.sub(r"/' [0-9]/", r"/'[0-9]/", string)
+    # number separators
+    string = string.replace(" @-@ ", "-")
+    string = string.replace(" @,@ ", ",")
+    string = string.replace(" @.@ ", ".")
+    # punctuation
+    string = string.replace(" : ", ": ")
+    string = string.replace(" ; ", "; ")
+    string = string.replace(" . ", ". ")
+    string = string.replace(" ! ", "! ")
+    string = string.replace(" ? ", "? ")
+    string = string.replace(" , ", ", ")
+    # double brackets
+    string = re.sub(r"\(\s*([^\)]*?)\s*\)", r"(\1)", string)
+    string = re.sub(r"\[\s*([^\]]*?)\s*\]", r"[\1]", string)
+    string = re.sub(r"{\s*([^}]*?)\s*}", r"{\1}", string)
+    string = re.sub(r"\"\s*([^\"]*?)\s*\"", r'"\1"', string)
+    string = re.sub(r"'\s*([^']*?)\s*'", r"'\1'", string)
+    # miscellaneous
+    string = string.replace("= = = =", "====")
+    string = string.replace("= = =", "===")
+    string = string.replace("= =", "==")
+    string = string.replace(" " + chr(176) + " ", chr(176))
+    string = string.replace(" \n", "\n")
+    string = string.replace("\n ", "\n")
+    string = string.replace(" N ", " 1 ")
+    string = string.replace(" 's", "'s")
+    return string
+
+
 def tokenize_dataset(opt, context_length):
     print("Tokenization...")
     # Clean and Concat the dataset
-    x = open(opt.src, "r").readlines()
-    xx = [_x for _x in x if _x != " \n"]
-    from onmt.transforms.tokenize import SentencePieceTransform
+    xx = open(opt.src, "r").readlines()
+    if "sentencepiece" in opt.transforms:
+        from onmt.transforms.tokenize import SentencePieceTransform
+
+        tokenizer = SentencePieceTransform(opt)
+    elif "onmt_tokenize" in opt.transforms:
+        from onmt.transforms.tokenize import ONMTTokenizerTransform
 
-    tokenizer = SentencePieceTransform(opt)
+        tokenizer = ONMTTokenizerTransform(opt)
+    else:
+        raise ValueError("No valid tokenizer found")
     tokenizer.warm_up()
-    tokens = tokenizer._tokenize(xx)
-    print("Done !")
+    print("warmup done")
+    # joiner = tokenizer._tokenize("\n")
+    tokens = []
+    for x in xx:
+        tokens += tokenizer._tokenize([wikitext_detokenizer(x)])
+        # tokens += tokenizer._tokenize([x])
+    print("Tokenization Done !")
     return tokens
 
 
@@ -38,7 +84,7 @@ def evaluate(opt):
     set_random_seed(opt.seed, use_gpu(opt))
 
     # Tokenize the dataset.
-    opt.src = "wikitext-2-raw-v1/wikitext-2-raw/wiki.test.raw"
+    opt.src = "eval_llm/WIKITEXT2/wikitext-2-raw-v1/wikitext-2-raw/wiki.test.raw"
     tokens = tokenize_dataset(opt, context_length=512)
 
     # Build the translator (along with the model.
@@ -47,8 +93,8 @@ def evaluate(opt):
     engine = InferenceEnginePY(engine_opt)
 
     # Score the dataset.
-    stride = 512
-    max_seq_length = 4096
+    stride = 256
+    max_seq_length = 512
 
     seq_len = len(tokens)
     src = []
@@ -75,8 +121,7 @@ def evaluate(opt):
     end_time = time.time()
     logger.info("total run time %.2f" % (end_time - start_time))
     logger.info(
-        "wikitext-2 perplexity with rolling likelihood and sliding window size 1000 and stride 512 %.2f"  # noqa: E501
-        % (ppl)
+        "wikitext-2 perplexity with rolling likelihood:  %.2f" % (ppl)  # noqa: E501
     )
 
 

onmt/inputters/text_corpus.py

@@ -38,6 +38,63 @@ def exfile_open(filename, *args, **kwargs):
         _file.close()
 
 
+class BlockwiseCorpus(object):
+    """A corpus class for reading a single file block by block."""
+
+    def __init__(self, name, file_path, block_size=4096):
+        """Initialize file path and block size."""
+        self.id = name
+        self.file_path = file_path
+        self.block_size = block_size
+
+    def load(self, offset=0, stride=1):
+        """
+        Load file and iterate by blocks.
+        `offset` and `stride` allow iterating only on every
+        `stride` block, starting from `offset`.
+        """
+
+        def make_ex(block_content):
+            example = {
+                "src": block_content,
+                "tgt": block_content,
+                "src_original": block_content,
+                "tgt_original": block_content,
+            }
+            return example
+
+        with open(self.file_path, mode="r", encoding="utf-8") as file:
+            block_content = ""
+            block_index = 0
+
+            while True:
+                chunk = file.read(self.block_size)
+                if not chunk:
+                    break
+
+                if (block_index // stride) % stride == offset:
+                    block_content += chunk
+
+                    if len(chunk) < self.block_size:
+                        # Reached end of file
+                        yield make_ex(block_content)
+                        break
+
+                    if len(block_content) >= self.block_size:
+                        yield make_ex(block_content)
+                block_content = ""
+                block_index += 1
+
+    def __str__(self):
+        cls_name = type(self).__name__
+        return (
+            f"{cls_name}({self.id}, {self.file_path}, {self.file_path}"
+            f"align={None}, "
+            f"n_src_feats={0}, "
+            f'src_feats_defaults="{None}")'
+        )
+
+
 class ParallelCorpus(object):
     """A parallel corpus file pair that can be loaded to iterate."""
 
@@ -117,14 +174,21 @@ def get_corpora(opts, task=CorpusTask.TRAIN, src=None, tgt=None, align=None):
     if task == CorpusTask.TRAIN:
         for corpus_id, corpus_dict in opts.data.items():
             if corpus_id != CorpusName.VALID:
-                corpora_dict[corpus_id] = ParallelCorpus(
-                    corpus_id,
-                    corpus_dict["path_src"],
-                    corpus_dict["path_tgt"],
-                    corpus_dict["path_align"],
-                    n_src_feats=opts.n_src_feats,
-                    src_feats_defaults=opts.src_feats_defaults,
-                )
+                if corpus_dict.get("path_txt", None) is None:
+                    corpora_dict[corpus_id] = ParallelCorpus(
+                        corpus_id,
+                        corpus_dict["path_src"],
+                        corpus_dict["path_tgt"],
+                        corpus_dict["path_align"],
+                        n_src_feats=opts.n_src_feats,
+                        src_feats_defaults=opts.src_feats_defaults,
+                    )
+                else:
+                    corpora_dict[corpus_id] = BlockwiseCorpus(
+                        corpus_id,
+                        corpus_dict["path_txt"],
+                        block_size=8192,  # number of characters
+                    )
     elif task == CorpusTask.VALID:
         if CorpusName.VALID in opts.data.keys():
             corpora_dict[CorpusName.VALID] = ParallelCorpus(

onmt/inputters/text_utils.py

@@ -149,7 +149,6 @@ def numericalize(vocabs, example):
         for fv, feat in zip(vocabs["src_feats"], example["src"]["feats"]):
             numeric_feats.append(fv(feat.split(" ")))
         numeric["src"]["feats"] = numeric_feats
-
     return numeric
 
 

onmt/opts.py

@@ -872,8 +872,7 @@ def model_opts(parser):
     group.add(
         "--rotary_interleave",
         "-rotary_interleave",
-        type=bool,
-        default=True,
+        action="store_true",
         help="Interleave the head dimensions when rotary"
         " embeddings are applied."
         "    Otherwise the head dimensions are sliced in half."

onmt/utils/parse.py

@@ -39,9 +39,10 @@ def _validate_data(cls, opt):
             # Check path
             path_src = corpus.get("path_src", None)
             path_tgt = corpus.get("path_tgt", None)
-            if path_src is None:
+            path_txt = corpus.get("path_txt", None)
+            if path_src is None and path_txt is None:
                 raise ValueError(
-                    f"Corpus {cname} src path is required."
+                    f"Corpus {cname} src/txt path is required."
                     "tgt path is also required for non language"
                     " modeling tasks."
                 )
@@ -57,8 +58,12 @@ def _validate_data(cls, opt):
                     corpus["path_tgt"] = path_src
                     corpora[cname] = corpus
                     path_tgt = path_src
-                cls._validate_file(path_src, info=f"{cname}/path_src")
-                cls._validate_file(path_tgt, info=f"{cname}/path_tgt")
+                if path_src is not None:
+                    cls._validate_file(path_src, info=f"{cname}/path_src")
+                if path_txt is not None:
+                    cls._validate_file(path_txt, info=f"{cname}/path_txt")
+                if path_tgt is not None:
+                    cls._validate_file(path_tgt, info=f"{cname}/path_tgt")
             path_align = corpus.get("path_align", None)
             if path_align is None:
                 if hasattr(opt, "lambda_align") and opt.lambda_align > 0.0: