NATURAL MACHINE TRANSLATE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "774c7535",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/serkanarslan/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "37b7494a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.models import Model\n",
    "from tensorflow.python.keras.layers import Input, Dense, GRU, Embedding, CuDNNGRU,LSTM\n",
    "from tensorflow.python.keras.optimizers import RMSprop\n",
    "from tensorflow.python.keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.python.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eeeef5e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "mark_start = 'ssss '\n",
    "mark_end = ' eeee'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "215702aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data(filename: str) -> list():\n",
    "    data_src = []\n",
    "    data_dest = []\n",
    "\n",
    "    for line in open(filename, encoding='UTF-8'):\n",
    "        en_text, tr_text = line.rstrip().split('\\t')\n",
    "\n",
    "        tr_text = mark_start + tr_text + mark_end\n",
    "\n",
    "        data_src.append(en_text)\n",
    "        data_dest.append(tr_text)\n",
    "    \n",
    "    return data_src, data_dest\n",
    "\n",
    "data_src, data_dest = data('tur.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b0d50052",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TokenizerWrap(Tokenizer):\n",
    "    def __init__(self, texts, padding, reverse=False, num_words=None):\n",
    "        Tokenizer.__init__(self, num_words=num_words)\n",
    "        \n",
    "        self.fit_on_texts(texts)\n",
    "        \n",
    "        self.index_to_word = dict(zip(self.word_index.values(), self.word_index.keys()))\n",
    "        \n",
    "        self.tokens = self.texts_to_sequences(texts)\n",
    "        \n",
    "        if reverse:\n",
    "            self.tokens = [list(reversed(x)) for x in self.tokens]\n",
    "            truncating = 'pre'\n",
    "        else:\n",
    "            truncating = 'post'\n",
    "            \n",
    "        self.num_tokens = [len(x) for x in self.tokens]\n",
    "        self.max_tokens = np.mean(self.num_tokens) + 2 * np.std(self.num_tokens)\n",
    "        self.max_tokens = int(self.max_tokens)\n",
    "        \n",
    "        self.tokens_padded = pad_sequences(self.tokens,\n",
    "                                           maxlen=self.max_tokens,\n",
    "                                           padding=padding,\n",
    "                                           truncating=truncating)\n",
    "        \n",
    "    def token_to_word(self, token):\n",
    "        word = ' ' if token == 0 else self.index_to_word[token]\n",
    "        return word\n",
    "    \n",
    "    def tokens_to_string(self, tokens):\n",
    "        words = [self.index_to_word[token] for token in tokens if token != 0]\n",
    "        text = ' '.join(words)\n",
    "        return text\n",
    "    \n",
    "    def text_to_tokens(self, text, padding, reverse=False):\n",
    "        tokens = self.texts_to_sequences([text])\n",
    "        tokens = np.array(tokens)\n",
    "        \n",
    "        if reverse:\n",
    "            tokens = np.flip(tokens, axis=1)\n",
    "            truncating = 'pre'\n",
    "        else:\n",
    "            truncating = 'post'\n",
    "            \n",
    "        tokens = pad_sequences(tokens,\n",
    "                               maxlen=self.max_tokens,\n",
    "                               padding=padding,\n",
    "                               truncating=truncating)\n",
    "        \n",
    "        return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "47748e12",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_src = TokenizerWrap(texts=data_src,\n",
    "                              padding='pre',\n",
    "                              reverse=True,\n",
    "                              num_words=None)\n",
    "\n",
    "tokenizer_dest = TokenizerWrap(texts=data_dest,\n",
    "                              padding='post',\n",
    "                              reverse=False,\n",
    "                              num_words=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "82069847",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(473035, 11)\n",
      "(473035, 10)\n"
     ]
    }
   ],
   "source": [
    "tokens_src = tokenizer_src.tokens_padded\n",
    "tokens_dest = tokenizer_dest.tokens_padded\n",
    "print(tokens_src.shape)\n",
    "print(tokens_dest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9b9fd093",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_start = tokenizer_dest.word_index[mark_start.strip()]\n",
    "token_end = tokenizer_dest.word_index[mark_end.strip()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "72ec0466",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_input_data = tokens_src\n",
    "\n",
    "decoder_input_data = tokens_dest[:, :-1]\n",
    "decoder_output_data = tokens_dest[:, 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3683d9f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_encoder_words = len(tokenizer_src.word_index)+1\n",
    "num_decoder_words = len(tokenizer_dest.word_index)+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c849ce5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94059"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_decoder_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "70375763",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 100\n",
    "\n",
    "word2vec = {}\n",
    "with open('glove.6B.100d.txt', encoding='UTF-8') as f:\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        vec = np.asarray(values[1:], dtype='float32')\n",
    "        word2vec[word] = vec\n",
    "        \n",
    "        \n",
    "embedding_matrix = np.random.uniform(-1, 1, (num_encoder_words, embedding_size))\n",
    "for word, i in tokenizer_src.word_index.items():\n",
    "    if i < num_encoder_words:\n",
    "        embedding_vector = word2vec.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "067801ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "FastText FaceBook tarafından geliştirilen açık kaynak bir vektörleştirme aracıdır.\n",
    "Kendim test ettiğimde modelin daha iyi çıktılar verdiğini gördüm dileyen benim github heseabımdan inceleyebilir.\n",
    "Github hesabım https://github.com/fawern/\n",
    "'''\n",
    "\n",
    "fasttext_tr_vec = {}\n",
    "\n",
    "fin = io.open('./cc.tr.300.vec/cc.tr.300.vec', 'r', encoding='utf-8').readlines()\n",
    "\n",
    "for line in fin:\n",
    "  values = line.split()\n",
    "  word = values[0]\n",
    "  vec = np.asarray(values[1:], dtype='float32')\n",
    "  fasttext_tr_vec[word] = vec\n",
    "\n",
    "fasttext_embedding_matrix = np.random.uniform(-1, 1, (num_decoder_words, 300))\n",
    "\n",
    "for word, index in tokenizer_tr.word_index.items():\n",
    "  if index < num_decoder_words:\n",
    "    ft_embedding_vector = fasttext_tr_vec.get(word)\n",
    "    if ft_embedding_vector is not None:\n",
    "      fasttext_embedding_matrix[index] = ft_embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "646659fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder:\n",
    "    def __init__(self,**kwargs):\n",
    "        \n",
    "        self.encoder_input = Input(shape=(None,), name='encoder_input')\n",
    "        \n",
    "        self.encoder_embedding = Embedding(input_dim=num_encoder_words,\n",
    "                              output_dim=embedding_size,\n",
    "                              weights=[embedding_matrix],\n",
    "                              trainable=True,\n",
    "                              name='encoder_embedding')\n",
    "        \n",
    "        self.encoder_lstm1 = LSTM(state_size, name='encoder_lstm1', return_sequences=True)\n",
    "        self.encoder_lstm2 = LSTM(state_size, name='encoder_lstm2', return_sequences=True)\n",
    "        self.encoder_lstm3 = LSTM(state_size, name='encoder_lstm3', return_sequences=False)\n",
    "        \n",
    "        \n",
    "    def connect_encoder(self):\n",
    "        net = self.encoder_input\n",
    "\n",
    "        net = self.encoder_embedding(net)\n",
    "\n",
    "        net = self.encoder_lstm1(net)\n",
    "        net = self.encoder_lstm2(net)\n",
    "        net = self.encoder_lstm3(net)\n",
    "\n",
    "        self.encoder_output = net\n",
    "\n",
    "        return self.encoder_output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d9b5ccc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_size = 256\n",
    "\n",
    "encoder = Encoder(num_encoder_words = num_encoder_words, embedding_size = embedding_size, \n",
    "                  embedding_matrix = embedding_matrix, state_size = state_size)\n",
    "\n",
    "\n",
    "encoder_output = encoder.connect_encoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e2512453",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder:\n",
    "    def __init__(self, **kwargs):\n",
    "        \n",
    "        self.decoder_initial_state = Input(shape=(state_size,), name='decoder_initial_state')\n",
    "        \n",
    "        self.decoder_input = Input(shape=(None,), name='decoder_input')\n",
    "        \n",
    "        self.decoder_embedding = Embedding(input_dim=num_decoder_words,\n",
    "                            #   output_dim=embedding_size,\n",
    "                              output_dim=300,\n",
    "                              weights=[fasttext_embedding_matrix], \n",
    "                              trainable=True,\n",
    "                              name='decoder_embedding')\n",
    "        \n",
    "        self.decoder_lstm1 = LSTM(state_size, name='decoder_lstm1', return_sequences=True)\n",
    "        self.decoder_lstm2 = LSTM(state_size, name='decoder_lstm2', return_sequences=True)\n",
    "        self.decoder_lstm3 = LSTM(state_size, name='decoder_lstm3', return_sequences=True)\n",
    "        \n",
    "        self.decoder_dense = Dense(num_decoder_words,\n",
    "                      activation='linear',\n",
    "                      name='decoder_output')\n",
    "        \n",
    "    def connect_decoder(self,initial_state):\n",
    "        net = self.decoder_input\n",
    "\n",
    "        net = self.decoder_embedding(net)\n",
    "\n",
    "        net = self.decoder_lstm1(net, initial_state=[initial_state,initial_state])\n",
    "        net = self.decoder_lstm2(net, initial_state=[initial_state,initial_state])\n",
    "        net = self.decoder_lstm3(net, initial_state=[initial_state,initial_state])\n",
    "\n",
    "        self.decoder_output = self.decoder_dense(net)\n",
    "\n",
    "        return self.decoder_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e570d9d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = Decoder(state_size=state_size, num_decoder_words=num_decoder_words, embedding_size=embedding_size)\n",
    "\n",
    "decoder_output = decoder.connect_decoder(initial_state = encoder_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "eb150d4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_train = Model(inputs=[encoder.encoder_input, decoder.decoder_input], outputs=[decoder_output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "6a6b9353",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sparse_cross_entropy(y_true, y_pred):\n",
    "    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)\n",
    "    loss_mean = tf.reduce_mean(loss)\n",
    "    return loss_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "888b2b2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = RMSprop(lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "60afe419",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "ValueError: `target_tensors` argument is not supported when executing eagerly. Received: \n",
    "[<KerasTensor: shape=(None, None) dtype=int32 (created by layer 'input_1')>].\n",
    "'''\n",
    "'''\n",
    "Elde ettiğiniz hata, TensorFlow'da Eager Execution (Eager İcra) modunda çalışırken, target_tensors argümanının bu \n",
    "modda desteklenmediğini gösteriyor. Eager Execution, TensorFlow 2.x'de varsayılan olarak etkin olduğunda, eğitim sırasında \n",
    "target_tensors'ı açıkça belirtmeniz gerekmez. decoder_targeti çıkartığımız zaman proje aynı şekilde çalışıyor.\n",
    "'''\n",
    "# decoder_target = tf.placeholder(dtype='int32', shape=(None,None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "879d16bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_train.compile(optimizer=optimizer,\n",
    "                    loss=sparse_cross_entropy,\n",
    "                    # target_tensors=[decoder_target]\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e8eda9cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = {'encoder_input': encoder_input_data, 'decoder_input': decoder_input_data}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "5d89f024",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_data = {'decoder_output': decoder_output_data}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "1f5e6f30",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_checkpoint = 'checkpoint.lstm'\n",
    "checkpoint = ModelCheckpoint(filepath=path_checkpoint, save_weights_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1937c216",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/serkanarslan/.local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Epoch 1/5\n",
      "472576/473035 [============================>.] - ETA: 7s - loss: 3.8343 WARNING:tensorflow:This model was compiled with a Keras optimizer (<tensorflow.python.keras.optimizers.RMSprop object at 0x7f0410f2a080>) but is being saved in TensorFlow format with `save_weights`. The model's weights will be saved, but unlike with TensorFlow optimizers in the TensorFlow format the optimizer's state will not be saved.\n",
      "\n",
      "Consider using a TensorFlow optimizer from `tf.train`.\n",
      "473035/473035 [==============================] - 7458s 16ms/sample - loss: 3.8336\n",
      "Epoch 2/5\n",
      "472576/473035 [============================>.] - ETA: 7s - loss: 2.6871 WARNING:tensorflow:This model was compiled with a Keras optimizer (<tensorflow.python.keras.optimizers.RMSprop object at 0x7f0410f2a080>) but is being saved in TensorFlow format with `save_weights`. The model's weights will be saved, but unlike with TensorFlow optimizers in the TensorFlow format the optimizer's state will not be saved.\n",
      "\n",
      "Consider using a TensorFlow optimizer from `tf.train`.\n",
      "473035/473035 [==============================] - 7683s 16ms/sample - loss: 2.6869\n",
      "Epoch 3/5\n",
      "223744/473035 [=============>................] - ETA: 1:03:35 - loss: 2.2170"
     ]
    }
   ],
   "source": [
    "model_train.fit(x=x_data,\n",
    "                y=y_data,\n",
    "                batch_size=512,\n",
    "                epochs=5,\n",
    "                callbacks=[checkpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6ba27de",
   "metadata": {},
   "source": [
    "## Çeviri için Encoder ve Decoder modellerinin oluşturulması"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "997979ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_encoder = Model(inputs=[encoder.encoder_input], outputs=[encoder_output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "64a19141",
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder_output = decoder.connect_decoder(initial_state = decoder.decoder_initial_state)\n",
    "\n",
    "model_decoder = Model(inputs=[decoder.decoder_input, decoder.decoder_initial_state], outputs=[decoder_output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "983402b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(input_text, true_output_text=None):\n",
    "    input_tokens = tokenizer_src.text_to_tokens(text=input_text,\n",
    "                                                reverse=True,\n",
    "                                                padding='pre')\n",
    "    \n",
    "    initial_state = model_encoder.predict(input_tokens)\n",
    "    \n",
    "    max_tokens = tokenizer_dest.max_tokens\n",
    "    \n",
    "    decoder_input_data = np.zeros(shape=(1, max_tokens), dtype=np.int)\n",
    "    \n",
    "    token_int = token_start\n",
    "    output_text = ''\n",
    "    count_tokens = 0\n",
    "    \n",
    "    while token_int != token_end and count_tokens < max_tokens:\n",
    "        decoder_input_data[0, count_tokens] = token_int\n",
    "        x_data = {'decoder_initial_state': initial_state, 'decoder_input': decoder_input_data}\n",
    "        \n",
    "        decoder_output = model_decoder.predict(x_data)\n",
    "        \n",
    "        token_onehot = decoder_output[0, count_tokens, :]\n",
    "        token_int = np.argmax(token_onehot)\n",
    "        \n",
    "        sampled_word = tokenizer_dest.token_to_word(token_int)\n",
    "        output_text += ' ' + sampled_word\n",
    "        count_tokens += 1\n",
    "        \n",
    "    print('Input text:')\n",
    "    print(input_text)\n",
    "    print()\n",
    "        \n",
    "    print('Translated text:')\n",
    "    print(output_text)\n",
    "    print()\n",
    "    \n",
    "    if true_output_text is not None:\n",
    "        print('True output text:')\n",
    "        print(true_output_text)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd0d4651",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate(input_text=data_src[99999], true_output_text=data_dest[99999])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f813877",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}