fix valid transform at scoring and tokenize with onmt_tokenize when d…

…ocify

onmt/tests/test_transform.py

@@ -327,12 +327,12 @@ def test_pyonmttok_bpe(self):
                 "struc￭",
                 "tion￭",
                 ":",
-                "\n￭",
+                "｟newline｠￭",
                 "in￭",
                 "struc￭",
                 "tion￭",
-                "\n￭",
-                "\n￭",
+                "｟newline｠￭",
+                "｟newline｠￭",
                 "#￭",
                 "#￭",
                 "#",
@@ -342,7 +342,7 @@ def test_pyonmttok_bpe(self):
                 "on￭",
                 "se",
                 ":",
-                "\n",
+                "｟newline｠",
                 "<blank>",
                 "respon￭",
                 "se",

onmt/trainer.py

@@ -297,7 +297,7 @@ def train(
             logger.info(
                 "Start training loop and validate every %d steps...", valid_steps
             )
-        logger.info("Scoring with: {}".format(self.scoring_preparator.transform))
+        logger.info("Scoring with: {}".format(self.scoring_preparator.transforms))
 
         total_stats = onmt.utils.Statistics()
         report_stats = onmt.utils.Statistics()

onmt/transforms/tokenize.py

@@ -157,7 +157,7 @@ def _tokenize(self, tokens, side="src", is_train=False):
         """Tokenize a list of words."""
         # This method embeds a custom logic to correctly handle certain placeholders
         # in case the tokenizer doesn't preserve them.
-        sentence = " ".join(tokens).replace(DefaultTokens.SEP, "\n")
+        sentence = " ".join(tokens)
         # Locate the end-of-sentence placeholders.
         sent_list = sentence.split(DefaultTokens.EOS)
         # Tokenize each sentence separately.
@@ -257,6 +257,7 @@ def tokenize_string(self, string, side="src", is_train=False):
         """Apply subword sampling or deterministic subwording"""
         sp_model = self.load_models[side]
         nbest_size = self.tgt_subword_nbest if side == "tgt" else self.src_subword_nbest
+        string = string.replace(DefaultTokens.SEP, "\n")
         if is_train is False or nbest_size in [0, 1]:
             # derterministic subwording
             tokens = sp_model.encode(string, out_type=str)
@@ -441,6 +442,9 @@ def _parse_opts(self):
         self.src_other_kwargs = self.opts.src_onmttok_kwargs
         self.tgt_other_kwargs = self.opts.tgt_onmttok_kwargs
         self.gpt2_pretok = self.opts.gpt2_pretok
+        self.preserve_placeholders = self.opts.tgt_onmttok_kwargs.get(
+            "preserve_placeholders", False
+        )
 
     @classmethod
     def get_specials(cls, opts):
@@ -558,6 +562,11 @@ def tokenize_string(self, sentence, side="src", is_train=False):
                     segmented.extend(["Ċ", "Ċ"])
                 else:
                     segmented.append(s)
+        elif (
+            self.src_subword_type == "sentencepiece" and not self.preserve_placeholders
+        ):
+            sentence = sentence.replace(DefaultTokens.SEP, "\n")
+            segmented = tokenizer(sentence)
         else:
             segmented = tokenizer(sentence)
         return segmented

onmt/utils/scoring_utils.py

@@ -19,16 +19,10 @@ def __init__(self, vocabs, opt):
         if self.opt.dump_preds is not None:
             if not os.path.exists(self.opt.dump_preds):
                 os.makedirs(self.opt.dump_preds)
-        self.transforms = opt.transforms
-        transforms_cls = get_transforms_cls(self.transforms)
-        transforms = make_transforms(self.opt, transforms_cls, self.vocabs)
-        self.transform = TransformPipe.build_from(transforms.values())
 
     def warm_up(self, transforms):
         self.transforms = transforms
-        transforms_cls = get_transforms_cls(self.transforms)
-        transforms = make_transforms(self.opt, transforms_cls, self.vocabs)
-        self.transform = TransformPipe.build_from(transforms.values())
+        self.transforms_cls = get_transforms_cls(transforms)
 
     def translate(self, model, gpu_rank, step):
         """Compute and save the sentences predicted by the
@@ -84,7 +78,7 @@ def translate(self, model, gpu_rank, step):
 
         # Reinstantiate the validation iterator
 
-        transforms_cls = get_transforms_cls(model_opt._all_transform)
+        #transforms_cls = get_transforms_cls(model_opt._all_transform)
         model_opt.num_workers = 0
         model_opt.tgt = None
 
@@ -100,7 +94,7 @@ def translate(self, model, gpu_rank, step):
 
         valid_iter = build_dynamic_dataset_iter(
             model_opt,
-            transforms_cls,
+            self.transforms_cls,
             translator.vocabs,
             task=CorpusTask.VALID,
             tgt="",  # This force to clear the target side (needed when using tgt_file_prefix)
@@ -125,12 +119,11 @@ def translate(self, model, gpu_rank, step):
 
         # Flatten predictions
         preds = [x.lstrip() for sublist in preds for x in sublist]
-
         # Save results
         if len(preds) > 0 and self.opt.scoring_debug:
             path = os.path.join(self.opt.dump_preds, f"preds.valid_step_{step}.txt")
             with open(path, "a") as file:
-                for i in range(len(preds)):
+                for i in range(len(raw_srcs)):
                     file.write("SOURCE: {}\n".format(raw_srcs[i]))
                     file.write("REF: {}\n".format(raw_refs[i]))
                     file.write("PRED: {}\n\n".format(preds[i]))