pt_to_en_transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "961cc8e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "src_dir = './data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "74ea8e62",
   "metadata": {},
   "outputs": [],
   "source": [
    "from data.dataloader import DataLoader\n",
    "from data.tokenizer import Tokenizer\n",
    "\n",
    "tokenizer = Tokenizer(data_dir=src_dir)\n",
    "src_tokenizer, target_tokenizer = tokenizer.get_ted_tokenizer()\n",
    "loader = DataLoader(src_tokenizer,target_tokenizer,batch_size=8)\n",
    "\n",
    "train_encorp_ds, val_encorp_ds = loader.create_ted_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d6a199cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_en_in(train_batches, src_tokenizer, target_tokenizer):\n",
    "    \n",
    "    for (en_line, hi_line), label in train_batches.take(1):\n",
    "        break\n",
    "    #print('en_line\\n', en_line, type(en_line))\n",
    "    #print('hi_line\\n', hi_line, type(hi_line))\n",
    "    #print('label\\n', label, type(label))\n",
    "\n",
    "    en_tokens = src_tokenizer.detokenize(en_line.numpy())\n",
    "    hi_tokens = target_tokenizer.detokenize(hi_line.numpy())\n",
    "\n",
    "    print('pt line\\n', en_tokens.numpy()[0].decode('utf-8'))\n",
    "    #print('pt tokens\\n', src_tokenizer.lookup(en_line.numpy()))\n",
    "    print('en line\\n', hi_tokens.numpy()[0].decode('utf-8'))\n",
    "    #print('en tokens\\n', target_tokenizer.lookup(hi_line.numpy()))\n",
    "    label_tokens = target_tokenizer.detokenize(label.numpy())\n",
    "    print('en label\\n', label_tokens.numpy()[0].decode('utf-8'))\n",
    "    #print('label tokens\\n', target_tokenizer.lookup(label.numpy()))\n",
    "\n",
    "    print(en_line.shape)\n",
    "    print(hi_line.shape)\n",
    "    print(label.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d3361af1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-04 11:59:45.048644: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_33' with dtype int64\n",
      "\t [[{{node Placeholder/_33}}]]\n",
      "2023-07-04 11:59:45.049130: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_31' with dtype resource\n",
      "\t [[{{node Placeholder/_31}}]]\n",
      "2023-07-04 11:59:45.907746: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset  will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pt line\n",
      " aqui esta ele .\n",
      "en line\n",
      " so there he is there .\n",
      "en label\n",
      " so there he is there .\n",
      "(8, 44)\n",
      "(8, 50)\n",
      "(8, 50)\n"
     ]
    }
   ],
   "source": [
    "test_en_in(train_encorp_ds,src_tokenizer, target_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cd1e6395",
   "metadata": {},
   "outputs": [],
   "source": [
    "from train import TransformerTraining\n",
    "\n",
    "training = TransformerTraining(src_tokenizer, \n",
    "                               target_tokenizer, \n",
    "                               num_layers=4, \n",
    "                               d_mode=512, \n",
    "                               dff=2048,\n",
    "                               num_heads=8, \n",
    "                               dropout_rate=0.1, \n",
    "                               num_epochs=5,\n",
    "                               steps_per_epochs=0.1,\n",
    "                               save_freq=5)\n",
    "training.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "87fc40d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-04 11:59:55.179807: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_34' with dtype int64\n",
      "\t [[{{node Placeholder/_34}}]]\n",
      "2023-07-04 11:59:55.180306: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_3' with dtype int64 and shape [1]\n",
      "\t [[{{node Placeholder/_3}}]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-04 12:00:13.185339: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:637] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2023-07-04 12:00:13.961736: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:424] Loaded cuDNN version 8600\n",
      "2023-07-04 12:00:14.606028: I tensorflow/compiler/xla/service/service.cc:169] XLA service 0x7fb9a6de7270 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2023-07-04 12:00:14.606097: I tensorflow/compiler/xla/service/service.cc:177]   StreamExecutor device (0): NVIDIA GeForce RTX 3060 Laptop GPU, Compute Capability 8.6\n",
      "2023-07-04 12:00:14.624424: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2023-07-04 12:00:14.904297: I ./tensorflow/compiler/jit/device_compiler.h:180] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "319/647 [=============>................] - ETA: 55s - loss: 7.2470 - masked_accuracy: 0.0988\n",
      "Epoch 1: saving model to training/cp-0001.ckpt\n",
      "639/647 [============================>.] - ETA: 1s - loss: 6.4269 - masked_accuracy: 0.1506\n",
      "Epoch 1: saving model to training/cp-0001.ckpt\n",
      "647/647 [==============================] - ETA: 0s - loss: 6.4142 - masked_accuracy: 0.1516"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-04 12:02:12.163646: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_35' with dtype resource\n",
      "\t [[{{node Placeholder/_35}}]]\n",
      "2023-07-04 12:02:12.168065: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_36' with dtype int64\n",
      "\t [[{{node Placeholder/_36}}]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "647/647 [==============================] - 139s 166ms/step - loss: 6.4142 - masked_accuracy: 0.1516 - val_loss: 5.4055 - val_masked_accuracy: 0.2294\n",
      "Epoch 2/5\n",
      "312/647 [=============>................] - ETA: 35s - loss: 5.2510 - masked_accuracy: 0.2398\n",
      "Epoch 2: saving model to training/cp-0002.ckpt\n",
      "632/647 [============================>.] - ETA: 1s - loss: 5.1370 - masked_accuracy: 0.2496\n",
      "Epoch 2: saving model to training/cp-0002.ckpt\n",
      "647/647 [==============================] - 74s 115ms/step - loss: 5.1325 - masked_accuracy: 0.2498 - val_loss: 5.0478 - val_masked_accuracy: 0.2476\n",
      "Epoch 3/5\n",
      "305/647 [=============>................] - ETA: 36s - loss: 4.9746 - masked_accuracy: 0.2621\n",
      "Epoch 3: saving model to training/cp-0003.ckpt\n",
      "625/647 [===========================>..] - ETA: 2s - loss: 4.9601 - masked_accuracy: 0.2618\n",
      "Epoch 3: saving model to training/cp-0003.ckpt\n",
      "647/647 [==============================] - 76s 117ms/step - loss: 4.9602 - masked_accuracy: 0.2617 - val_loss: 4.8664 - val_masked_accuracy: 0.2804\n",
      "Epoch 4/5\n",
      "298/647 [============>.................] - ETA: 35s - loss: 4.9826 - masked_accuracy: 0.2522\n",
      "Epoch 4: saving model to training/cp-0004.ckpt\n",
      "618/647 [===========================>..] - ETA: 3s - loss: 5.0397 - masked_accuracy: 0.2440\n",
      "Epoch 4: saving model to training/cp-0004.ckpt\n",
      "647/647 [==============================] - 73s 113ms/step - loss: 5.0437 - masked_accuracy: 0.2435 - val_loss: 5.2333 - val_masked_accuracy: 0.2075\n",
      "Epoch 5/5\n",
      "291/647 [============>.................] - ETA: 37s - loss: 5.2436 - masked_accuracy: 0.1997\n",
      "Epoch 5: saving model to training/cp-0005.ckpt\n",
      "611/647 [===========================>..] - ETA: 3s - loss: 5.3066 - masked_accuracy: 0.1908\n",
      "Epoch 5: saving model to training/cp-0005.ckpt\n",
      "647/647 [==============================] - 74s 115ms/step - loss: 5.3097 - masked_accuracy: 0.1906 - val_loss: 5.2479 - val_masked_accuracy: 0.1830\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-04 12:07:12.084889: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset  will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fbbb858e160>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training.fit(train_encorp_ds, val_encorp_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ce80b13",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}