train_and_save_mode.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "701f06bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-01 13:37:36.861361: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "from data_loader import create_ted_datasets, get_ted_tokenizer, create_ted_datasets_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6e56a0ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-01 13:37:38.852468: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:38.859919: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:38.860194: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:38.862736: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:38.862989: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:38.863237: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:39.419406: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:39.419676: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:39.419689: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1722] Could not identify NUMA node of platform GPU id 0, defaulting to 0.  Your kernel may not have been built with NUMA support.\n",
      "2023-07-01 13:37:39.419952: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-01 13:37:39.419990: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1635] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 3383 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3060 Laptop GPU, pci bus id: 0000:01:00.0, compute capability: 8.6\n"
     ]
    }
   ],
   "source": [
    "train_ted_ds, val_ted_ds = create_ted_datasets()\n",
    "en_tokenizer, pt_tokenizer = get_ted_tokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "82ce28e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from train import TransformerTraining\n",
    "\n",
    "training = TransformerTraining()\n",
    "training.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b1187e0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-01 13:37:42.251643: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_32' with dtype resource\n",
      "\t [[{{node Placeholder/_32}}]]\n",
      "2023-07-01 13:37:42.252056: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_4' with dtype int64 and shape [1]\n",
      "\t [[{{node Placeholder/_4}}]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-01 13:37:54.439830: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:637] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2023-07-01 13:37:54.653855: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:424] Loaded cuDNN version 8600\n",
      "2023-07-01 13:37:55.017026: I tensorflow/compiler/xla/service/service.cc:169] XLA service 0x46e6f060 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2023-07-01 13:37:55.017068: I tensorflow/compiler/xla/service/service.cc:177]   StreamExecutor device (0): NVIDIA GeForce RTX 3060 Laptop GPU, Compute Capability 8.6\n",
      "2023-07-01 13:37:55.020766: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2023-07-01 13:37:55.114129: I ./tensorflow/compiler/jit/device_compiler.h:180] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 25/323 [=>............................] - ETA: 1:52 - loss: 8.8007 - masked_accuracy: 0.0044"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-01 13:38:25.236738: W tensorflow/tsl/framework/bfc_allocator.cc:485] Allocator (GPU_0_bfc) ran out of memory trying to allocate 27.25MiB (rounded to 28573696)requested by op transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum\n",
      "If the cause is memory fragmentation maybe the environment variable 'TF_GPU_ALLOCATOR=cuda_malloc_async' will improve the situation. \n",
      "Current allocation summary follows.\n",
      "Current allocation summary follows.\n",
      "2023-07-01 13:38:25.237017: I tensorflow/tsl/framework/bfc_allocator.cc:1039] BFCAllocator dump for GPU_0_bfc\n",
      "2023-07-01 13:38:25.237042: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (256): \tTotal Chunks: 66, Chunks in use: 65. 16.5KiB allocated for chunks. 16.2KiB in use in bin. 366B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237055: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (512): \tTotal Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237062: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (1024): \tTotal Chunks: 4, Chunks in use: 4. 6.5KiB allocated for chunks. 6.5KiB in use in bin. 6.2KiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237087: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (2048): \tTotal Chunks: 180, Chunks in use: 180. 360.0KiB allocated for chunks. 360.0KiB in use in bin. 360.0KiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237097: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (4096): \tTotal Chunks: 58, Chunks in use: 55. 399.2KiB allocated for chunks. 379.5KiB in use in bin. 373.2KiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237103: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (8192): \tTotal Chunks: 29, Chunks in use: 28. 253.5KiB allocated for chunks. 239.5KiB in use in bin. 232.8KiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237109: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (16384): \tTotal Chunks: 111, Chunks in use: 111. 1.77MiB allocated for chunks. 1.77MiB in use in bin. 1.77MiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237114: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (32768): \tTotal Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237119: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (65536): \tTotal Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237124: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (131072): \tTotal Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237130: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (262144): \tTotal Chunks: 1, Chunks in use: 0. 424.2KiB allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237136: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (524288): \tTotal Chunks: 14, Chunks in use: 10. 11.11MiB allocated for chunks. 8.42MiB in use in bin. 8.34MiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237142: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (1048576): \tTotal Chunks: 10, Chunks in use: 10. 14.75MiB allocated for chunks. 14.75MiB in use in bin. 14.38MiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237148: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (2097152): \tTotal Chunks: 46, Chunks in use: 46. 155.76MiB allocated for chunks. 155.76MiB in use in bin. 154.02MiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237154: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (4194304): \tTotal Chunks: 82, Chunks in use: 82. 388.39MiB allocated for chunks. 388.39MiB in use in bin. 358.03MiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237159: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (8388608): \tTotal Chunks: 159, Chunks in use: 159. 1.33GiB allocated for chunks. 1.33GiB in use in bin. 1.32GiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237165: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (16777216): \tTotal Chunks: 46, Chunks in use: 45. 1.21GiB allocated for chunks. 1.19GiB in use in bin. 1.17GiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237171: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (33554432): \tTotal Chunks: 5, Chunks in use: 5. 207.36MiB allocated for chunks. 207.36MiB in use in bin. 136.00MiB client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237176: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (67108864): \tTotal Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237188: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (134217728): \tTotal Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237193: I tensorflow/tsl/framework/bfc_allocator.cc:1046] Bin (268435456): \tTotal Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.\n",
      "2023-07-01 13:38:25.237200: I tensorflow/tsl/framework/bfc_allocator.cc:1062] Bin for 27.25MiB was 16.00MiB, Chunk State: \n",
      "2023-07-01 13:38:25.237215: I tensorflow/tsl/framework/bfc_allocator.cc:1068]   Size: 20.44MiB | Requested Size: 6.8KiB | in_use: 0 | bin_num: 16, prev:   Size: 3.41MiB | Requested Size: 3.41MiB | in_use: 1 | bin_num: -1, next:   Size: 27.25MiB | Requested Size: 27.25MiB | in_use: 1 | bin_num: -1\n",
      "2023-07-01 13:38:25.237221: I tensorflow/tsl/framework/bfc_allocator.cc:1075] Next region of size 3547332608\n",
      "2023-07-01 13:38:25.237230: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00000 of size 1280 next 1\n",
      "2023-07-01 13:38:25.237234: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00500 of size 256 next 2\n",
      "2023-07-01 13:38:25.237239: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00600 of size 256 next 3\n",
      "2023-07-01 13:38:25.237243: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00700 of size 256 next 4\n",
      "2023-07-01 13:38:25.237247: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00800 of size 256 next 5\n",
      "2023-07-01 13:38:25.237251: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00900 of size 256 next 6\n",
      "2023-07-01 13:38:25.237255: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00a00 of size 256 next 7\n",
      "2023-07-01 13:38:25.237259: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00b00 of size 256 next 8\n",
      "2023-07-01 13:38:25.237263: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00c00 of size 256 next 9\n",
      "2023-07-01 13:38:25.237266: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00d00 of size 256 next 10\n",
      "2023-07-01 13:38:25.237270: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00e00 of size 256 next 11\n",
      "2023-07-01 13:38:25.237274: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e00f00 of size 256 next 12\n",
      "2023-07-01 13:38:25.237278: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01000 of size 256 next 13\n",
      "2023-07-01 13:38:25.237282: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01100 of size 256 next 14\n",
      "2023-07-01 13:38:25.237286: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01200 of size 256 next 15\n",
      "2023-07-01 13:38:25.237290: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01300 of size 256 next 16\n",
      "2023-07-01 13:38:25.237294: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01400 of size 256 next 17\n",
      "2023-07-01 13:38:25.237298: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01500 of size 256 next 18\n",
      "2023-07-01 13:38:25.237302: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01600 of size 256 next 19\n",
      "2023-07-01 13:38:25.237306: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01700 of size 256 next 20\n",
      "2023-07-01 13:38:25.237309: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01800 of size 256 next 21\n",
      "2023-07-01 13:38:25.237313: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01900 of size 256 next 22\n",
      "2023-07-01 13:38:25.237317: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01a00 of size 256 next 23\n",
      "2023-07-01 13:38:25.237321: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01b00 of size 256 next 24\n",
      "2023-07-01 13:38:25.237325: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01c00 of size 256 next 25\n",
      "2023-07-01 13:38:25.237329: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01d00 of size 256 next 26\n",
      "2023-07-01 13:38:25.237333: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01e00 of size 256 next 32\n",
      "2023-07-01 13:38:25.237337: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e01f00 of size 256 next 33\n",
      "2023-07-01 13:38:25.237341: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e02000 of size 256 next 29\n",
      "2023-07-01 13:38:25.237346: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e02100 of size 256 next 30\n",
      "2023-07-01 13:38:25.237350: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e02200 of size 256 next 36\n",
      "2023-07-01 13:38:25.237353: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e02300 of size 256 next 34\n",
      "2023-07-01 13:38:25.237358: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e02400 of size 16384 next 40\n",
      "2023-07-01 13:38:25.237365: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e06400 of size 16384 next 41\n",
      "2023-07-01 13:38:25.237369: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0a400 of size 16384 next 44\n",
      "2023-07-01 13:38:25.237373: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0e400 of size 256 next 35\n",
      "2023-07-01 13:38:25.237378: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0e500 of size 256 next 39\n",
      "2023-07-01 13:38:25.237382: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0e600 of size 2048 next 48\n",
      "2023-07-01 13:38:25.237386: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0ee00 of size 256 next 46\n",
      "2023-07-01 13:38:25.237390: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0ef00 of size 2048 next 47\n",
      "2023-07-01 13:38:25.237394: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0f700 of size 2048 next 50\n",
      "2023-07-01 13:38:25.237398: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e0ff00 of size 256 next 51\n",
      "2023-07-01 13:38:25.237402: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e10000 of size 256 next 52\n",
      "2023-07-01 13:38:25.237407: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 713e10100 of size 8331520 next 27\n",
      "2023-07-01 13:38:25.237411: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714602200 of size 4194304 next 28\n",
      "2023-07-01 13:38:25.237416: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714a02200 of size 4194304 next 31\n",
      "2023-07-01 13:38:25.237420: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e02200 of size 8192 next 54\n",
      "2023-07-01 13:38:25.237425: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e04200 of size 2048 next 53\n",
      "2023-07-01 13:38:25.237429: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e04a00 of size 2048 next 55\n",
      "2023-07-01 13:38:25.237433: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e05200 of size 2048 next 59\n",
      "2023-07-01 13:38:25.237437: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e05a00 of size 16384 next 62\n",
      "2023-07-01 13:38:25.237441: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e09a00 of size 16384 next 65\n",
      "2023-07-01 13:38:25.237445: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e0da00 of size 16384 next 67\n",
      "2023-07-01 13:38:25.237449: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e11a00 of size 2048 next 69\n",
      "2023-07-01 13:38:25.237453: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e12200 of size 2048 next 60\n",
      "2023-07-01 13:38:25.237457: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e12a00 of size 2048 next 61\n",
      "2023-07-01 13:38:25.237462: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e13200 of size 8192 next 71\n",
      "2023-07-01 13:38:25.237467: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e15200 of size 2048 next 72\n",
      "2023-07-01 13:38:25.237472: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e15a00 of size 2048 next 74\n",
      "2023-07-01 13:38:25.237477: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e16200 of size 2048 next 76\n",
      "2023-07-01 13:38:25.237481: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e16a00 of size 16384 next 77\n",
      "2023-07-01 13:38:25.237485: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e1aa00 of size 16384 next 78\n",
      "2023-07-01 13:38:25.237489: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e1ea00 of size 16384 next 81\n",
      "2023-07-01 13:38:25.237492: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e22a00 of size 2048 next 83\n",
      "2023-07-01 13:38:25.237496: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e23200 of size 2048 next 85\n",
      "2023-07-01 13:38:25.237499: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e23a00 of size 2048 next 87\n",
      "2023-07-01 13:38:25.237503: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e24200 of size 8192 next 88\n",
      "2023-07-01 13:38:25.237507: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e26200 of size 2048 next 89\n",
      "2023-07-01 13:38:25.237510: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e26a00 of size 2048 next 91\n",
      "2023-07-01 13:38:25.237514: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e27200 of size 2048 next 92\n",
      "2023-07-01 13:38:25.237519: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e27a00 of size 16384 next 93\n",
      "2023-07-01 13:38:25.237522: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e2ba00 of size 16384 next 94\n",
      "2023-07-01 13:38:25.237526: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e2fa00 of size 16384 next 97\n",
      "2023-07-01 13:38:25.237529: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e33a00 of size 2048 next 99\n",
      "2023-07-01 13:38:25.237533: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e34200 of size 2048 next 101\n",
      "2023-07-01 13:38:25.237537: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e34a00 of size 2048 next 103\n",
      "2023-07-01 13:38:25.237540: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e35200 of size 8192 next 104\n",
      "2023-07-01 13:38:25.237544: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e37200 of size 2048 next 105\n",
      "2023-07-01 13:38:25.237548: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e37a00 of size 2048 next 107\n",
      "2023-07-01 13:38:25.237551: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e38200 of size 2048 next 108\n",
      "2023-07-01 13:38:25.237555: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e38a00 of size 16384 next 111\n",
      "2023-07-01 13:38:25.237558: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e3ca00 of size 16384 next 109\n",
      "2023-07-01 13:38:25.237562: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e40a00 of size 16384 next 115\n",
      "2023-07-01 13:38:25.237565: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e44a00 of size 2048 next 116\n",
      "2023-07-01 13:38:25.237569: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e45200 of size 2048 next 118\n",
      "2023-07-01 13:38:25.237572: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e45a00 of size 2048 next 120\n",
      "2023-07-01 13:38:25.237576: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e46200 of size 16384 next 121\n",
      "2023-07-01 13:38:25.237579: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e4a200 of size 16384 next 122\n",
      "2023-07-01 13:38:25.237583: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e4e200 of size 16384 next 124\n",
      "2023-07-01 13:38:25.237586: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e52200 of size 2048 next 126\n",
      "2023-07-01 13:38:25.237590: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e52a00 of size 2048 next 128\n",
      "2023-07-01 13:38:25.237594: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e53200 of size 2048 next 130\n",
      "2023-07-01 13:38:25.237597: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e53a00 of size 8192 next 131\n",
      "2023-07-01 13:38:25.237601: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e55a00 of size 2048 next 132\n",
      "2023-07-01 13:38:25.237605: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e56200 of size 2048 next 134\n",
      "2023-07-01 13:38:25.237608: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e56a00 of size 2048 next 135\n",
      "2023-07-01 13:38:25.237612: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e57200 of size 16384 next 136\n",
      "2023-07-01 13:38:25.237615: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e5b200 of size 16384 next 137\n",
      "2023-07-01 13:38:25.237619: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e5f200 of size 16384 next 140\n",
      "2023-07-01 13:38:25.237622: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e63200 of size 2048 next 142\n",
      "2023-07-01 13:38:25.237626: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e63a00 of size 2048 next 144\n",
      "2023-07-01 13:38:25.237629: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e64200 of size 2048 next 146\n",
      "2023-07-01 13:38:25.237633: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e64a00 of size 16384 next 147\n",
      "2023-07-01 13:38:25.237636: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e68a00 of size 16384 next 148\n",
      "2023-07-01 13:38:25.237640: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e6ca00 of size 16384 next 150\n",
      "2023-07-01 13:38:25.237643: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e70a00 of size 2048 next 152\n",
      "2023-07-01 13:38:25.237647: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e71200 of size 2048 next 154\n",
      "2023-07-01 13:38:25.237651: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e71a00 of size 2048 next 156\n",
      "2023-07-01 13:38:25.237654: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e72200 of size 8192 next 157\n",
      "2023-07-01 13:38:25.237658: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e74200 of size 2048 next 158\n",
      "2023-07-01 13:38:25.237662: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e74a00 of size 2048 next 160\n",
      "2023-07-01 13:38:25.237665: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e75200 of size 2048 next 161\n",
      "2023-07-01 13:38:25.237669: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e75a00 of size 16384 next 162\n",
      "2023-07-01 13:38:25.237673: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e79a00 of size 16384 next 163\n",
      "2023-07-01 13:38:25.237677: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e7da00 of size 16384 next 166\n",
      "2023-07-01 13:38:25.237681: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e81a00 of size 2048 next 168\n",
      "2023-07-01 13:38:25.237685: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e82200 of size 2048 next 170\n",
      "2023-07-01 13:38:25.237688: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e82a00 of size 2048 next 172\n",
      "2023-07-01 13:38:25.237692: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e83200 of size 16384 next 173\n",
      "2023-07-01 13:38:25.237696: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e87200 of size 16384 next 174\n",
      "2023-07-01 13:38:25.237700: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e8b200 of size 16384 next 176\n",
      "2023-07-01 13:38:25.237703: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e8f200 of size 2048 next 178\n",
      "2023-07-01 13:38:25.237707: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e8fa00 of size 2048 next 180\n",
      "2023-07-01 13:38:25.237710: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e90200 of size 2048 next 182\n",
      "2023-07-01 13:38:25.237714: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e90a00 of size 8192 next 183\n",
      "2023-07-01 13:38:25.237717: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e92a00 of size 2048 next 184\n",
      "2023-07-01 13:38:25.237721: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e93200 of size 2048 next 186\n",
      "2023-07-01 13:38:25.237724: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e93a00 of size 2048 next 187\n",
      "2023-07-01 13:38:25.237728: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e94200 of size 16384 next 188\n",
      "2023-07-01 13:38:25.237731: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e98200 of size 16384 next 189\n",
      "2023-07-01 13:38:25.237735: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714e9c200 of size 16384 next 192\n",
      "2023-07-01 13:38:25.237739: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ea0200 of size 2048 next 194\n",
      "2023-07-01 13:38:25.237743: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ea0a00 of size 2048 next 196\n",
      "2023-07-01 13:38:25.237746: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ea1200 of size 2048 next 198\n",
      "2023-07-01 13:38:25.237750: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ea1a00 of size 16384 next 199\n",
      "2023-07-01 13:38:25.237753: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ea5a00 of size 16384 next 200\n",
      "2023-07-01 13:38:25.237757: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ea9a00 of size 16384 next 202\n",
      "2023-07-01 13:38:25.237761: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eada00 of size 2048 next 204\n",
      "2023-07-01 13:38:25.237764: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eae200 of size 2048 next 206\n",
      "2023-07-01 13:38:25.237768: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eaea00 of size 2048 next 208\n",
      "2023-07-01 13:38:25.237771: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eaf200 of size 8192 next 209\n",
      "2023-07-01 13:38:25.237775: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb1200 of size 2048 next 210\n",
      "2023-07-01 13:38:25.237778: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb1a00 of size 2048 next 212\n",
      "2023-07-01 13:38:25.237782: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb2200 of size 2048 next 213\n",
      "2023-07-01 13:38:25.237786: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb2a00 of size 256 next 214\n",
      "2023-07-01 13:38:25.237789: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb2b00 of size 256 next 215\n",
      "2023-07-01 13:38:25.237793: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb2c00 of size 28160 next 218\n",
      "2023-07-01 13:38:25.237796: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb9a00 of size 256 next 216\n",
      "2023-07-01 13:38:25.237800: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eb9b00 of size 16384 next 222\n",
      "2023-07-01 13:38:25.237803: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ebdb00 of size 16384 next 223\n",
      "2023-07-01 13:38:25.237807: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ec1b00 of size 16384 next 226\n",
      "2023-07-01 13:38:25.237811: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ec5b00 of size 16384 next 227\n",
      "2023-07-01 13:38:25.237815: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ec9b00 of size 16384 next 230\n",
      "2023-07-01 13:38:25.237819: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ecdb00 of size 16384 next 231\n",
      "2023-07-01 13:38:25.237822: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed1b00 of size 2048 next 234\n",
      "2023-07-01 13:38:25.237826: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed2300 of size 2048 next 235\n",
      "2023-07-01 13:38:25.237829: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed2b00 of size 2048 next 236\n",
      "2023-07-01 13:38:25.237833: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed3300 of size 2048 next 237\n",
      "2023-07-01 13:38:25.237836: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed3b00 of size 2048 next 238\n",
      "2023-07-01 13:38:25.237840: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed4300 of size 2048 next 239\n",
      "2023-07-01 13:38:25.237843: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed4b00 of size 8192 next 240\n",
      "2023-07-01 13:38:25.237847: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed6b00 of size 8192 next 241\n",
      "2023-07-01 13:38:25.237851: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed8b00 of size 2048 next 244\n",
      "2023-07-01 13:38:25.237854: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed9300 of size 2048 next 245\n",
      "2023-07-01 13:38:25.237858: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ed9b00 of size 2048 next 246\n",
      "2023-07-01 13:38:25.237861: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eda300 of size 2048 next 247\n",
      "2023-07-01 13:38:25.237865: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714edab00 of size 2048 next 248\n",
      "2023-07-01 13:38:25.237868: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714edb300 of size 2048 next 249\n",
      "2023-07-01 13:38:25.237872: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714edbb00 of size 16384 next 252\n",
      "2023-07-01 13:38:25.237875: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714edfb00 of size 16384 next 253\n",
      "2023-07-01 13:38:25.237879: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ee3b00 of size 16384 next 256\n",
      "2023-07-01 13:38:25.237882: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ee7b00 of size 16384 next 257\n",
      "2023-07-01 13:38:25.237886: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eebb00 of size 16384 next 260\n",
      "2023-07-01 13:38:25.237889: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714eefb00 of size 16384 next 261\n",
      "2023-07-01 13:38:25.237893: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef3b00 of size 2048 next 264\n",
      "2023-07-01 13:38:25.237896: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef4300 of size 2048 next 265\n",
      "2023-07-01 13:38:25.237900: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef4b00 of size 2048 next 266\n",
      "2023-07-01 13:38:25.237903: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef5300 of size 2048 next 267\n",
      "2023-07-01 13:38:25.237907: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef5b00 of size 2048 next 268\n",
      "2023-07-01 13:38:25.237910: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef6300 of size 2048 next 269\n",
      "2023-07-01 13:38:25.237914: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef6b00 of size 8192 next 272\n",
      "2023-07-01 13:38:25.237921: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ef8b00 of size 8192 next 273\n",
      "2023-07-01 13:38:25.237924: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714efab00 of size 2048 next 276\n",
      "2023-07-01 13:38:25.237928: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714efb300 of size 2048 next 277\n",
      "2023-07-01 13:38:25.237931: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714efbb00 of size 2048 next 278\n",
      "2023-07-01 13:38:25.237935: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714efc300 of size 2048 next 279\n",
      "2023-07-01 13:38:25.237938: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714efcb00 of size 2048 next 280\n",
      "2023-07-01 13:38:25.237942: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714efd300 of size 2048 next 281\n",
      "2023-07-01 13:38:25.237945: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714efdb00 of size 16384 next 284\n",
      "2023-07-01 13:38:25.237949: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f01b00 of size 16384 next 285\n",
      "2023-07-01 13:38:25.237952: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f05b00 of size 16384 next 288\n",
      "2023-07-01 13:38:25.237956: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f09b00 of size 16384 next 289\n",
      "2023-07-01 13:38:25.237959: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f0db00 of size 16384 next 292\n",
      "2023-07-01 13:38:25.237963: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f11b00 of size 16384 next 293\n",
      "2023-07-01 13:38:25.237966: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f15b00 of size 2048 next 296\n",
      "2023-07-01 13:38:25.237969: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f16300 of size 2048 next 297\n",
      "2023-07-01 13:38:25.237973: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f16b00 of size 2048 next 298\n",
      "2023-07-01 13:38:25.237977: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f17300 of size 2048 next 299\n",
      "2023-07-01 13:38:25.237980: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f17b00 of size 2048 next 300\n",
      "2023-07-01 13:38:25.237984: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f18300 of size 2048 next 301\n",
      "2023-07-01 13:38:25.237987: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f18b00 of size 8192 next 304\n",
      "2023-07-01 13:38:25.237991: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1ab00 of size 8192 next 305\n",
      "2023-07-01 13:38:25.237995: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1cb00 of size 2048 next 308\n",
      "2023-07-01 13:38:25.237998: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1d300 of size 2048 next 309\n",
      "2023-07-01 13:38:25.238004: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1db00 of size 2048 next 310\n",
      "2023-07-01 13:38:25.238008: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1e300 of size 2048 next 311\n",
      "2023-07-01 13:38:25.238011: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1eb00 of size 2048 next 312\n",
      "2023-07-01 13:38:25.238015: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1f300 of size 2048 next 313\n",
      "2023-07-01 13:38:25.238018: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f1fb00 of size 16384 next 316\n",
      "2023-07-01 13:38:25.238022: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f23b00 of size 16384 next 317\n",
      "2023-07-01 13:38:25.238025: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f27b00 of size 16384 next 320\n",
      "2023-07-01 13:38:25.238029: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f2bb00 of size 16384 next 321\n",
      "2023-07-01 13:38:25.238032: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f2fb00 of size 16384 next 324\n",
      "2023-07-01 13:38:25.238036: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f33b00 of size 16384 next 325\n",
      "2023-07-01 13:38:25.238040: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f37b00 of size 2048 next 328\n",
      "2023-07-01 13:38:25.238043: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f38300 of size 2048 next 329\n",
      "2023-07-01 13:38:25.238047: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f38b00 of size 2048 next 330\n",
      "2023-07-01 13:38:25.238050: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f39300 of size 2048 next 331\n",
      "2023-07-01 13:38:25.238054: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f39b00 of size 2048 next 332\n",
      "2023-07-01 13:38:25.238058: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f3a300 of size 2048 next 333\n",
      "2023-07-01 13:38:25.238062: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f3ab00 of size 8192 next 336\n",
      "2023-07-01 13:38:25.238065: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f3cb00 of size 8192 next 337\n",
      "2023-07-01 13:38:25.238069: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f3eb00 of size 2048 next 340\n",
      "2023-07-01 13:38:25.238072: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f3f300 of size 2048 next 341\n",
      "2023-07-01 13:38:25.238076: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f3fb00 of size 2048 next 342\n",
      "2023-07-01 13:38:25.238079: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f40300 of size 2048 next 343\n",
      "2023-07-01 13:38:25.238083: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f40b00 of size 2048 next 344\n",
      "2023-07-01 13:38:25.238086: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f41300 of size 2048 next 345\n",
      "2023-07-01 13:38:25.238090: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f41b00 of size 16384 next 350\n",
      "2023-07-01 13:38:25.238093: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f45b00 of size 16384 next 351\n",
      "2023-07-01 13:38:25.238097: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f49b00 of size 16384 next 354\n",
      "2023-07-01 13:38:25.238100: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f4db00 of size 16384 next 355\n",
      "2023-07-01 13:38:25.238104: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f51b00 of size 16384 next 358\n",
      "2023-07-01 13:38:25.238107: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f55b00 of size 16384 next 359\n",
      "2023-07-01 13:38:25.238111: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f59b00 of size 2048 next 362\n",
      "2023-07-01 13:38:25.238114: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f5a300 of size 2048 next 363\n",
      "2023-07-01 13:38:25.238118: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f5ab00 of size 2048 next 364\n",
      "2023-07-01 13:38:25.238122: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f5b300 of size 2048 next 365\n",
      "2023-07-01 13:38:25.238125: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f5bb00 of size 2048 next 366\n",
      "2023-07-01 13:38:25.238128: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f5c300 of size 2048 next 367\n",
      "2023-07-01 13:38:25.238132: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f5cb00 of size 16384 next 370\n",
      "2023-07-01 13:38:25.238135: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f60b00 of size 16384 next 371\n",
      "2023-07-01 13:38:25.238140: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f64b00 of size 16384 next 374\n",
      "2023-07-01 13:38:25.238146: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f68b00 of size 16384 next 375\n",
      "2023-07-01 13:38:25.238152: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f6cb00 of size 16384 next 378\n",
      "2023-07-01 13:38:25.238159: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f70b00 of size 16384 next 379\n",
      "2023-07-01 13:38:25.238166: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f74b00 of size 2048 next 382\n",
      "2023-07-01 13:38:25.238172: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f75300 of size 2048 next 383\n",
      "2023-07-01 13:38:25.238183: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f75b00 of size 2048 next 384\n",
      "2023-07-01 13:38:25.238191: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f76300 of size 2048 next 385\n",
      "2023-07-01 13:38:25.238198: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f76b00 of size 2048 next 386\n",
      "2023-07-01 13:38:25.238205: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f77300 of size 2048 next 387\n",
      "2023-07-01 13:38:25.238215: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f77b00 of size 8192 next 390\n",
      "2023-07-01 13:38:25.238223: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f79b00 of size 8192 next 391\n",
      "2023-07-01 13:38:25.238229: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f7bb00 of size 2048 next 394\n",
      "2023-07-01 13:38:25.238233: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f7c300 of size 2048 next 395\n",
      "2023-07-01 13:38:25.238236: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f7cb00 of size 2048 next 396\n",
      "2023-07-01 13:38:25.238240: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f7d300 of size 2048 next 397\n",
      "2023-07-01 13:38:25.238243: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f7db00 of size 2048 next 398\n",
      "2023-07-01 13:38:25.238247: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f7e300 of size 2048 next 399\n",
      "2023-07-01 13:38:25.238251: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f7eb00 of size 16384 next 402\n",
      "2023-07-01 13:38:25.238254: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f82b00 of size 16384 next 403\n",
      "2023-07-01 13:38:25.238257: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f86b00 of size 16384 next 406\n",
      "2023-07-01 13:38:25.238261: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f8ab00 of size 16384 next 407\n",
      "2023-07-01 13:38:25.238265: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f8eb00 of size 16384 next 410\n",
      "2023-07-01 13:38:25.238268: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f92b00 of size 16384 next 411\n",
      "2023-07-01 13:38:25.238274: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f96b00 of size 2048 next 414\n",
      "2023-07-01 13:38:25.238281: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f97300 of size 2048 next 415\n",
      "2023-07-01 13:38:25.238288: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f97b00 of size 2048 next 416\n",
      "2023-07-01 13:38:25.238295: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f98300 of size 2048 next 417\n",
      "2023-07-01 13:38:25.238302: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f98b00 of size 2048 next 418\n",
      "2023-07-01 13:38:25.238309: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f99300 of size 2048 next 419\n",
      "2023-07-01 13:38:25.238313: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f99b00 of size 16384 next 422\n",
      "2023-07-01 13:38:25.238317: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714f9db00 of size 16384 next 423\n",
      "2023-07-01 13:38:25.238321: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fa1b00 of size 16384 next 426\n",
      "2023-07-01 13:38:25.238324: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fa5b00 of size 16384 next 427\n",
      "2023-07-01 13:38:25.238328: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fa9b00 of size 16384 next 430\n",
      "2023-07-01 13:38:25.238331: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fadb00 of size 16384 next 431\n",
      "2023-07-01 13:38:25.238335: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb1b00 of size 2048 next 434\n",
      "2023-07-01 13:38:25.238338: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb2300 of size 2048 next 435\n",
      "2023-07-01 13:38:25.238342: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb2b00 of size 2048 next 436\n",
      "2023-07-01 13:38:25.238345: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb3300 of size 2048 next 437\n",
      "2023-07-01 13:38:25.238349: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb3b00 of size 2048 next 438\n",
      "2023-07-01 13:38:25.238353: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb4300 of size 2048 next 439\n",
      "2023-07-01 13:38:25.238356: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb4b00 of size 8192 next 442\n",
      "2023-07-01 13:38:25.238361: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb6b00 of size 8192 next 443\n",
      "2023-07-01 13:38:25.238364: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb8b00 of size 2048 next 446\n",
      "2023-07-01 13:38:25.238368: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb9300 of size 2048 next 447\n",
      "2023-07-01 13:38:25.238372: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fb9b00 of size 2048 next 448\n",
      "2023-07-01 13:38:25.238375: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fba300 of size 2048 next 449\n",
      "2023-07-01 13:38:25.238379: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fbab00 of size 2048 next 450\n",
      "2023-07-01 13:38:25.238381: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fbb300 of size 2048 next 451\n",
      "2023-07-01 13:38:25.238384: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fbbb00 of size 16384 next 454\n",
      "2023-07-01 13:38:25.238387: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fbfb00 of size 16384 next 455\n",
      "2023-07-01 13:38:25.238390: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fc3b00 of size 16384 next 458\n",
      "2023-07-01 13:38:25.238393: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fc7b00 of size 16384 next 459\n",
      "2023-07-01 13:38:25.238395: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fcbb00 of size 16384 next 462\n",
      "2023-07-01 13:38:25.238398: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fcfb00 of size 16384 next 463\n",
      "2023-07-01 13:38:25.238401: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fd3b00 of size 2048 next 466\n",
      "2023-07-01 13:38:25.238404: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fd4300 of size 2048 next 467\n",
      "2023-07-01 13:38:25.238407: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fd4b00 of size 2048 next 468\n",
      "2023-07-01 13:38:25.238410: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fd5300 of size 2048 next 469\n",
      "2023-07-01 13:38:25.238413: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fd5b00 of size 2048 next 470\n",
      "2023-07-01 13:38:25.238415: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fd6300 of size 2048 next 471\n",
      "2023-07-01 13:38:25.238418: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fd6b00 of size 16384 next 474\n",
      "2023-07-01 13:38:25.238421: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fdab00 of size 16384 next 475\n",
      "2023-07-01 13:38:25.238424: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fdeb00 of size 16384 next 478\n",
      "2023-07-01 13:38:25.238426: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fe2b00 of size 16384 next 479\n",
      "2023-07-01 13:38:25.238429: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fe6b00 of size 16384 next 482\n",
      "2023-07-01 13:38:25.238432: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714feab00 of size 16384 next 483\n",
      "2023-07-01 13:38:25.238435: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714feeb00 of size 2048 next 486\n",
      "2023-07-01 13:38:25.238437: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fef300 of size 2048 next 487\n",
      "2023-07-01 13:38:25.238440: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714fefb00 of size 2048 next 488\n",
      "2023-07-01 13:38:25.238443: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff0300 of size 2048 next 489\n",
      "2023-07-01 13:38:25.238446: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff0b00 of size 2048 next 490\n",
      "2023-07-01 13:38:25.238449: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff1300 of size 2048 next 491\n",
      "2023-07-01 13:38:25.238452: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff1b00 of size 8192 next 494\n",
      "2023-07-01 13:38:25.238454: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff3b00 of size 8192 next 495\n",
      "2023-07-01 13:38:25.238457: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff5b00 of size 2048 next 498\n",
      "2023-07-01 13:38:25.238461: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff6300 of size 2048 next 499\n",
      "2023-07-01 13:38:25.238464: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff6b00 of size 2048 next 500\n",
      "2023-07-01 13:38:25.238467: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff7300 of size 2048 next 501\n",
      "2023-07-01 13:38:25.238470: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff7b00 of size 2048 next 502\n",
      "2023-07-01 13:38:25.238473: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff8300 of size 2048 next 503\n",
      "2023-07-01 13:38:25.238476: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ff8b00 of size 16384 next 506\n",
      "2023-07-01 13:38:25.238479: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 714ffcb00 of size 16384 next 507\n",
      "2023-07-01 13:38:25.238482: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715000b00 of size 16384 next 510\n",
      "2023-07-01 13:38:25.238485: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715004b00 of size 16384 next 511\n",
      "2023-07-01 13:38:25.238488: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715008b00 of size 16384 next 514\n",
      "2023-07-01 13:38:25.238490: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71500cb00 of size 16384 next 515\n",
      "2023-07-01 13:38:25.238493: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715010b00 of size 2048 next 518\n",
      "2023-07-01 13:38:25.238496: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715011300 of size 2048 next 519\n",
      "2023-07-01 13:38:25.238499: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715011b00 of size 2048 next 520\n",
      "2023-07-01 13:38:25.238502: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715012300 of size 2048 next 521\n",
      "2023-07-01 13:38:25.238505: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715012b00 of size 2048 next 522\n",
      "2023-07-01 13:38:25.238508: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715013300 of size 2048 next 523\n",
      "2023-07-01 13:38:25.238511: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715013b00 of size 16384 next 526\n",
      "2023-07-01 13:38:25.238513: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715017b00 of size 16384 next 527\n",
      "2023-07-01 13:38:25.238516: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71501bb00 of size 16384 next 530\n",
      "2023-07-01 13:38:25.238519: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71501fb00 of size 16384 next 531\n",
      "2023-07-01 13:38:25.238522: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715023b00 of size 16384 next 534\n",
      "2023-07-01 13:38:25.238525: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715027b00 of size 16384 next 535\n",
      "2023-07-01 13:38:25.238528: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71502bb00 of size 2048 next 538\n",
      "2023-07-01 13:38:25.238531: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71502c300 of size 2048 next 539\n",
      "2023-07-01 13:38:25.238534: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71502cb00 of size 2048 next 540\n",
      "2023-07-01 13:38:25.238536: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71502d300 of size 2048 next 541\n",
      "2023-07-01 13:38:25.238539: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71502db00 of size 2048 next 542\n",
      "2023-07-01 13:38:25.238542: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71502e300 of size 2048 next 543\n",
      "2023-07-01 13:38:25.238545: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71502eb00 of size 8192 next 546\n",
      "2023-07-01 13:38:25.238548: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715030b00 of size 8192 next 547\n",
      "2023-07-01 13:38:25.238551: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715032b00 of size 2048 next 550\n",
      "2023-07-01 13:38:25.238554: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715033300 of size 2048 next 551\n",
      "2023-07-01 13:38:25.238557: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715033b00 of size 2048 next 552\n",
      "2023-07-01 13:38:25.238560: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715034300 of size 2048 next 553\n",
      "2023-07-01 13:38:25.238562: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715034b00 of size 2048 next 554\n",
      "2023-07-01 13:38:25.238565: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715035300 of size 2048 next 555\n",
      "2023-07-01 13:38:25.238568: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715035b00 of size 28160 next 558\n",
      "2023-07-01 13:38:25.238571: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71503c900 of size 28160 next 559\n",
      "2023-07-01 13:38:25.238574: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043700 of size 256 next 560\n",
      "2023-07-01 13:38:25.238577: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043800 of size 256 next 561\n",
      "2023-07-01 13:38:25.238580: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043900 of size 256 next 562\n",
      "2023-07-01 13:38:25.238582: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043a00 of size 256 next 563\n",
      "2023-07-01 13:38:25.238586: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043b00 of size 256 next 564\n",
      "2023-07-01 13:38:25.238588: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043c00 of size 256 next 565\n",
      "2023-07-01 13:38:25.238591: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043d00 of size 256 next 566\n",
      "2023-07-01 13:38:25.238594: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043e00 of size 256 next 567\n",
      "2023-07-01 13:38:25.238597: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715043f00 of size 256 next 568\n",
      "2023-07-01 13:38:25.238600: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044000 of size 256 next 569\n",
      "2023-07-01 13:38:25.238603: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044100 of size 256 next 570\n",
      "2023-07-01 13:38:25.238605: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044200 of size 256 next 571\n",
      "2023-07-01 13:38:25.238609: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044300 of size 256 next 572\n",
      "2023-07-01 13:38:25.238611: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044400 of size 256 next 573\n",
      "2023-07-01 13:38:25.238614: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044500 of size 256 next 574\n",
      "2023-07-01 13:38:25.238617: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044600 of size 256 next 704\n",
      "2023-07-01 13:38:25.238620: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044700 of size 256 next 769\n",
      "2023-07-01 13:38:25.238623: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044800 of size 256 next 577\n",
      "2023-07-01 13:38:25.238626: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044900 of size 256 next 578\n",
      "2023-07-01 13:38:25.238629: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715044a00 of size 6912 next 647\n",
      "2023-07-01 13:38:25.238633: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715046500 of size 7168 next 705\n",
      "2023-07-01 13:38:25.238636: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715048100 of size 7168 next 604\n",
      "2023-07-01 13:38:25.238639: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715049d00 of size 7168 next 787\n",
      "2023-07-01 13:38:25.238642: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504b900 of size 256 next 731\n",
      "2023-07-01 13:38:25.238645: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504ba00 of size 256 next 710\n",
      "2023-07-01 13:38:25.238648: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504bb00 of size 256 next 668\n",
      "2023-07-01 13:38:25.238651: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504bc00 of size 256 next 722\n",
      "2023-07-01 13:38:25.238654: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 71504bd00 of size 6144 next 738\n",
      "2023-07-01 13:38:25.238656: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504d500 of size 1792 next 820\n",
      "2023-07-01 13:38:25.238659: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504dc00 of size 256 next 812\n",
      "2023-07-01 13:38:25.238662: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504dd00 of size 256 next 623\n",
      "2023-07-01 13:38:25.238665: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 71504de00 of size 256 next 818\n",
      "2023-07-01 13:38:25.238668: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504df00 of size 256 next 804\n",
      "2023-07-01 13:38:25.238671: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71504e000 of size 8960 next 643\n",
      "2023-07-01 13:38:25.238675: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715050300 of size 7168 next 625\n",
      "2023-07-01 13:38:25.238678: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715051f00 of size 6912 next 624\n",
      "2023-07-01 13:38:25.238681: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715053a00 of size 14080 next 749\n",
      "2023-07-01 13:38:25.238684: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715057100 of size 1792 next 653\n",
      "2023-07-01 13:38:25.238687: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715057800 of size 6912 next 741\n",
      "2023-07-01 13:38:25.238690: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715059300 of size 6912 next 829\n",
      "2023-07-01 13:38:25.238693: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71505ae00 of size 7424 next 603\n",
      "2023-07-01 13:38:25.238696: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71505cb00 of size 1792 next 815\n",
      "2023-07-01 13:38:25.238699: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71505d200 of size 14080 next 746\n",
      "2023-07-01 13:38:25.238702: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715060900 of size 7168 next 744\n",
      "2023-07-01 13:38:25.238705: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715062500 of size 6912 next 794\n",
      "2023-07-01 13:38:25.238707: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715064000 of size 6912 next 628\n",
      "2023-07-01 13:38:25.238710: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715065b00 of size 6912 next 734\n",
      "2023-07-01 13:38:25.238713: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715067600 of size 6912 next 709\n",
      "2023-07-01 13:38:25.238716: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715069100 of size 6912 next 707\n",
      "2023-07-01 13:38:25.238719: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71506ac00 of size 6912 next 605\n",
      "2023-07-01 13:38:25.238722: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71506c700 of size 6912 next 659\n",
      "2023-07-01 13:38:25.238725: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71506e200 of size 6912 next 825\n",
      "2023-07-01 13:38:25.238727: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71506fd00 of size 6912 next 684\n",
      "2023-07-01 13:38:25.238730: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715071800 of size 6912 next 803\n",
      "2023-07-01 13:38:25.238733: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715073300 of size 6912 next 816\n",
      "2023-07-01 13:38:25.238736: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715074e00 of size 6912 next 613\n",
      "2023-07-01 13:38:25.238739: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715076900 of size 6912 next 790\n",
      "2023-07-01 13:38:25.238742: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715078400 of size 6912 next 692\n",
      "2023-07-01 13:38:25.238745: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715079f00 of size 6912 next 640\n",
      "2023-07-01 13:38:25.238747: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 71507ba00 of size 6912 next 717\n",
      "2023-07-01 13:38:25.238750: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71507d500 of size 6912 next 681\n",
      "2023-07-01 13:38:25.238753: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71507f000 of size 6912 next 618\n",
      "2023-07-01 13:38:25.238756: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 715080b00 of size 14336 next 743\n",
      "2023-07-01 13:38:25.238759: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715084300 of size 7168 next 806\n",
      "2023-07-01 13:38:25.238761: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715085f00 of size 7168 next 579\n",
      "2023-07-01 13:38:25.238764: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715087b00 of size 7168 next 646\n",
      "2023-07-01 13:38:25.238767: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715089700 of size 7168 next 791\n",
      "2023-07-01 13:38:25.238770: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71508b300 of size 7168 next 608\n",
      "2023-07-01 13:38:25.238773: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71508cf00 of size 7168 next 675\n",
      "2023-07-01 13:38:25.238776: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71508eb00 of size 11520 next 737\n",
      "2023-07-01 13:38:25.238780: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715091800 of size 7168 next 768\n",
      "2023-07-01 13:38:25.238783: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715093400 of size 7168 next 677\n",
      "2023-07-01 13:38:25.238786: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715095000 of size 7168 next 752\n",
      "2023-07-01 13:38:25.238789: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715096c00 of size 7168 next 697\n",
      "2023-07-01 13:38:25.238791: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715098800 of size 7168 next 696\n",
      "2023-07-01 13:38:25.238794: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71509a400 of size 7168 next 590\n",
      "2023-07-01 13:38:25.238797: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71509c000 of size 7168 next 661\n",
      "2023-07-01 13:38:25.238800: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71509dc00 of size 7168 next 757\n",
      "2023-07-01 13:38:25.238803: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71509f800 of size 7168 next 634\n",
      "2023-07-01 13:38:25.238806: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150a1400 of size 7168 next 756\n",
      "2023-07-01 13:38:25.238809: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150a3000 of size 7168 next 666\n",
      "2023-07-01 13:38:25.238811: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150a4c00 of size 7168 next 771\n",
      "2023-07-01 13:38:25.238814: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150a6800 of size 7168 next 789\n",
      "2023-07-01 13:38:25.238817: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150a8400 of size 7168 next 730\n",
      "2023-07-01 13:38:25.238820: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150aa000 of size 7168 next 802\n",
      "2023-07-01 13:38:25.238823: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150abc00 of size 7168 next 584\n",
      "2023-07-01 13:38:25.238825: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150ad800 of size 7168 next 713\n",
      "2023-07-01 13:38:25.238828: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 7150af400 of size 7168 next 786\n",
      "2023-07-01 13:38:25.238831: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150b1000 of size 7168 next 599\n",
      "2023-07-01 13:38:25.238834: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7150b2c00 of size 7168 next 811\n",
      "2023-07-01 13:38:25.238837: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 7150b4800 of size 434432 next 814\n",
      "2023-07-01 13:38:25.238840: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71511e900 of size 932352 next 56\n",
      "2023-07-01 13:38:25.238843: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715202300 of size 4194304 next 57\n",
      "2023-07-01 13:38:25.238846: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715602300 of size 8388352 next 42\n",
      "2023-07-01 13:38:25.238848: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 715e02200 of size 15028224 next 38\n",
      "2023-07-01 13:38:25.238853: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 716c57200 of size 15902720 next 37\n",
      "2023-07-01 13:38:25.238856: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 717b81a00 of size 8388608 next 43\n",
      "2023-07-01 13:38:25.238859: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 718381a00 of size 8388608 next 45\n",
      "2023-07-01 13:38:25.238861: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 718b81a00 of size 8388608 next 49\n",
      "2023-07-01 13:38:25.238864: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 719381a00 of size 4194304 next 58\n",
      "2023-07-01 13:38:25.238867: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 719781a00 of size 4194304 next 73\n",
      "2023-07-01 13:38:25.238870: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 719b81a00 of size 4194304 next 75\n",
      "2023-07-01 13:38:25.238873: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 719f81a00 of size 4194304 next 90\n",
      "2023-07-01 13:38:25.238876: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71a381a00 of size 4194304 next 64\n",
      "2023-07-01 13:38:25.238879: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71a781a00 of size 8388608 next 63\n",
      "2023-07-01 13:38:25.238881: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71af81a00 of size 8388608 next 66\n",
      "2023-07-01 13:38:25.238884: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71b781a00 of size 8388608 next 68\n",
      "2023-07-01 13:38:25.238887: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71bf81a00 of size 8388608 next 70\n",
      "2023-07-01 13:38:25.238890: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71c781a00 of size 4194304 next 106\n",
      "2023-07-01 13:38:25.238893: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71cb81a00 of size 4194304 next 80\n",
      "2023-07-01 13:38:25.238896: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71cf81a00 of size 8388608 next 79\n",
      "2023-07-01 13:38:25.238899: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71d781a00 of size 8388608 next 82\n",
      "2023-07-01 13:38:25.238902: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71df81a00 of size 8388608 next 84\n",
      "2023-07-01 13:38:25.238905: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71e781a00 of size 8388608 next 86\n",
      "2023-07-01 13:38:25.238908: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71ef81a00 of size 4194304 next 133\n",
      "2023-07-01 13:38:25.238910: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71f381a00 of size 4194304 next 96\n",
      "2023-07-01 13:38:25.238913: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71f781a00 of size 8388608 next 95\n",
      "2023-07-01 13:38:25.238916: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 71ff81a00 of size 8388608 next 98\n",
      "2023-07-01 13:38:25.238919: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 720781a00 of size 8388608 next 100\n",
      "2023-07-01 13:38:25.238922: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 720f81a00 of size 8388608 next 102\n",
      "2023-07-01 13:38:25.238925: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 721781a00 of size 4194304 next 159\n",
      "2023-07-01 13:38:25.238928: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 721b81a00 of size 4194304 next 114\n",
      "2023-07-01 13:38:25.238930: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 721f81a00 of size 8388608 next 110\n",
      "2023-07-01 13:38:25.238933: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 722781a00 of size 11935744 next 113\n",
      "2023-07-01 13:38:25.238938: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7232e3a00 of size 14356480 next 112\n",
      "2023-07-01 13:38:25.238941: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 724094a00 of size 8388608 next 117\n",
      "2023-07-01 13:38:25.238943: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 724894a00 of size 8388608 next 119\n",
      "2023-07-01 13:38:25.238946: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 725094a00 of size 8388608 next 123\n",
      "2023-07-01 13:38:25.238949: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 725894a00 of size 8388608 next 125\n",
      "2023-07-01 13:38:25.238952: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 726094a00 of size 8388608 next 127\n",
      "2023-07-01 13:38:25.238955: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 726894a00 of size 8388608 next 129\n",
      "2023-07-01 13:38:25.238957: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 727094a00 of size 4194304 next 185\n",
      "2023-07-01 13:38:25.238960: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 727494a00 of size 4194304 next 139\n",
      "2023-07-01 13:38:25.238963: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 727894a00 of size 8388608 next 138\n",
      "2023-07-01 13:38:25.238966: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 728094a00 of size 8388608 next 141\n",
      "2023-07-01 13:38:25.238969: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 728894a00 of size 8388608 next 143\n",
      "2023-07-01 13:38:25.238972: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 729094a00 of size 8388608 next 145\n",
      "2023-07-01 13:38:25.238975: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 729894a00 of size 8388608 next 149\n",
      "2023-07-01 13:38:25.238978: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72a094a00 of size 8388608 next 151\n",
      "2023-07-01 13:38:25.238980: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72a894a00 of size 8388608 next 153\n",
      "2023-07-01 13:38:25.238983: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72b094a00 of size 8388608 next 155\n",
      "2023-07-01 13:38:25.238986: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72b894a00 of size 4194304 next 211\n",
      "2023-07-01 13:38:25.238989: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72bc94a00 of size 4194304 next 165\n",
      "2023-07-01 13:38:25.238992: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72c094a00 of size 8388608 next 164\n",
      "2023-07-01 13:38:25.238995: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72c894a00 of size 8388608 next 167\n",
      "2023-07-01 13:38:25.238997: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72d094a00 of size 8388608 next 169\n",
      "2023-07-01 13:38:25.239000: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72d894a00 of size 8388608 next 171\n",
      "2023-07-01 13:38:25.239003: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72e094a00 of size 8388608 next 175\n",
      "2023-07-01 13:38:25.239006: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72e894a00 of size 8388608 next 177\n",
      "2023-07-01 13:38:25.239009: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72f094a00 of size 8388608 next 179\n",
      "2023-07-01 13:38:25.239012: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 72f894a00 of size 8388608 next 181\n",
      "2023-07-01 13:38:25.239014: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 730094a00 of size 8388608 next 191\n",
      "2023-07-01 13:38:25.239017: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 730894a00 of size 8388608 next 190\n",
      "2023-07-01 13:38:25.239020: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 731094a00 of size 8388608 next 193\n",
      "2023-07-01 13:38:25.239023: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 731894a00 of size 8388608 next 195\n",
      "2023-07-01 13:38:25.239025: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 732094a00 of size 8388608 next 197\n",
      "2023-07-01 13:38:25.239028: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 732894a00 of size 8388608 next 201\n",
      "2023-07-01 13:38:25.239031: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 733094a00 of size 8388608 next 203\n",
      "2023-07-01 13:38:25.239033: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 733894a00 of size 8388608 next 205\n",
      "2023-07-01 13:38:25.239036: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 734094a00 of size 8388608 next 207\n",
      "2023-07-01 13:38:25.239040: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 734894a00 of size 28712960 next 220\n",
      "2023-07-01 13:38:25.239043: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7363f6a00 of size 14356480 next 219\n",
      "2023-07-01 13:38:25.239045: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7371a7a00 of size 15902720 next 217\n",
      "2023-07-01 13:38:25.239048: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7380d2200 of size 8388608 next 221\n",
      "2023-07-01 13:38:25.239051: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7388d2200 of size 8388608 next 224\n",
      "2023-07-01 13:38:25.239054: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7390d2200 of size 8388608 next 225\n",
      "2023-07-01 13:38:25.239057: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7398d2200 of size 8388608 next 228\n",
      "2023-07-01 13:38:25.239060: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73a0d2200 of size 8388608 next 229\n",
      "2023-07-01 13:38:25.239062: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73a8d2200 of size 8388608 next 232\n",
      "2023-07-01 13:38:25.239065: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73b0d2200 of size 8388608 next 233\n",
      "2023-07-01 13:38:25.239068: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73b8d2200 of size 4194304 next 242\n",
      "2023-07-01 13:38:25.239071: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73bcd2200 of size 4194304 next 243\n",
      "2023-07-01 13:38:25.239074: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73c0d2200 of size 8388608 next 250\n",
      "2023-07-01 13:38:25.239076: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73c8d2200 of size 8388608 next 251\n",
      "2023-07-01 13:38:25.239079: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73d0d2200 of size 8388608 next 254\n",
      "2023-07-01 13:38:25.239082: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73d8d2200 of size 8388608 next 255\n",
      "2023-07-01 13:38:25.239085: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73e0d2200 of size 8388608 next 258\n",
      "2023-07-01 13:38:25.239087: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73e8d2200 of size 8388608 next 259\n",
      "2023-07-01 13:38:25.239090: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73f0d2200 of size 8388608 next 262\n",
      "2023-07-01 13:38:25.239093: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 73f8d2200 of size 8388608 next 263\n",
      "2023-07-01 13:38:25.239096: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7400d2200 of size 4194304 next 270\n",
      "2023-07-01 13:38:25.239098: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7404d2200 of size 4194304 next 271\n",
      "2023-07-01 13:38:25.239101: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7408d2200 of size 4194304 next 274\n",
      "2023-07-01 13:38:25.239104: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 740cd2200 of size 4194304 next 275\n",
      "2023-07-01 13:38:25.239107: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7410d2200 of size 8388608 next 282\n",
      "2023-07-01 13:38:25.239110: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7418d2200 of size 8388608 next 283\n",
      "2023-07-01 13:38:25.239112: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7420d2200 of size 8388608 next 286\n",
      "2023-07-01 13:38:25.239115: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7428d2200 of size 8388608 next 287\n",
      "2023-07-01 13:38:25.239118: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7430d2200 of size 8388608 next 290\n",
      "2023-07-01 13:38:25.239121: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7438d2200 of size 8388608 next 291\n",
      "2023-07-01 13:38:25.239124: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7440d2200 of size 8388608 next 294\n",
      "2023-07-01 13:38:25.239126: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7448d2200 of size 8388608 next 295\n",
      "2023-07-01 13:38:25.239129: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7450d2200 of size 4194304 next 302\n",
      "2023-07-01 13:38:25.239132: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7454d2200 of size 4194304 next 303\n",
      "2023-07-01 13:38:25.239135: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7458d2200 of size 4194304 next 306\n",
      "2023-07-01 13:38:25.239138: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 745cd2200 of size 4194304 next 307\n",
      "2023-07-01 13:38:25.239141: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7460d2200 of size 8388608 next 314\n",
      "2023-07-01 13:38:25.239143: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7468d2200 of size 8388608 next 315\n",
      "2023-07-01 13:38:25.239146: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7470d2200 of size 8388608 next 318\n",
      "2023-07-01 13:38:25.239149: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7478d2200 of size 8388608 next 319\n",
      "2023-07-01 13:38:25.239152: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7480d2200 of size 8388608 next 322\n",
      "2023-07-01 13:38:25.239154: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7488d2200 of size 8388608 next 323\n",
      "2023-07-01 13:38:25.239157: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7490d2200 of size 8388608 next 326\n",
      "2023-07-01 13:38:25.239160: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7498d2200 of size 8388608 next 327\n",
      "2023-07-01 13:38:25.239163: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74a0d2200 of size 4194304 next 334\n",
      "2023-07-01 13:38:25.239165: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74a4d2200 of size 4194304 next 335\n",
      "2023-07-01 13:38:25.239168: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74a8d2200 of size 4194304 next 338\n",
      "2023-07-01 13:38:25.239171: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74acd2200 of size 4194304 next 339\n",
      "2023-07-01 13:38:25.239174: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74b0d2200 of size 14356480 next 346\n",
      "2023-07-01 13:38:25.239177: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74be83200 of size 14356480 next 347\n",
      "2023-07-01 13:38:25.239180: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74cc34200 of size 8388608 next 348\n",
      "2023-07-01 13:38:25.239182: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74d434200 of size 8388608 next 349\n",
      "2023-07-01 13:38:25.239185: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74dc34200 of size 8388608 next 352\n",
      "2023-07-01 13:38:25.239188: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74e434200 of size 8388608 next 353\n",
      "2023-07-01 13:38:25.239191: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74ec34200 of size 8388608 next 356\n",
      "2023-07-01 13:38:25.239194: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74f434200 of size 8388608 next 357\n",
      "2023-07-01 13:38:25.239196: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 74fc34200 of size 8388608 next 360\n",
      "2023-07-01 13:38:25.239199: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 750434200 of size 8388608 next 361\n",
      "2023-07-01 13:38:25.239202: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 750c34200 of size 8388608 next 368\n",
      "2023-07-01 13:38:25.239205: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 751434200 of size 8388608 next 369\n",
      "2023-07-01 13:38:25.239207: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 751c34200 of size 8388608 next 372\n",
      "2023-07-01 13:38:25.239210: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 752434200 of size 8388608 next 373\n",
      "2023-07-01 13:38:25.239213: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 752c34200 of size 8388608 next 376\n",
      "2023-07-01 13:38:25.239216: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 753434200 of size 8388608 next 377\n",
      "2023-07-01 13:38:25.239219: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 753c34200 of size 8388608 next 380\n",
      "2023-07-01 13:38:25.239221: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 754434200 of size 8388608 next 381\n",
      "2023-07-01 13:38:25.239224: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 754c34200 of size 4194304 next 388\n",
      "2023-07-01 13:38:25.239227: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 755034200 of size 4194304 next 389\n",
      "2023-07-01 13:38:25.239230: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 755434200 of size 4194304 next 392\n",
      "2023-07-01 13:38:25.239233: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 755834200 of size 4194304 next 393\n",
      "2023-07-01 13:38:25.239235: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 755c34200 of size 8388608 next 400\n",
      "2023-07-01 13:38:25.239238: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 756434200 of size 8388608 next 401\n",
      "2023-07-01 13:38:25.239241: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 756c34200 of size 8388608 next 404\n",
      "2023-07-01 13:38:25.239244: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 757434200 of size 8388608 next 405\n",
      "2023-07-01 13:38:25.239246: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 757c34200 of size 8388608 next 408\n",
      "2023-07-01 13:38:25.239249: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 758434200 of size 8388608 next 409\n",
      "2023-07-01 13:38:25.239252: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 758c34200 of size 8388608 next 412\n",
      "2023-07-01 13:38:25.239255: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 759434200 of size 8388608 next 413\n",
      "2023-07-01 13:38:25.239257: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 759c34200 of size 8388608 next 420\n",
      "2023-07-01 13:38:25.239260: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75a434200 of size 8388608 next 421\n",
      "2023-07-01 13:38:25.239263: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75ac34200 of size 8388608 next 424\n",
      "2023-07-01 13:38:25.239266: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75b434200 of size 8388608 next 425\n",
      "2023-07-01 13:38:25.239269: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75bc34200 of size 8388608 next 428\n",
      "2023-07-01 13:38:25.239271: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75c434200 of size 8388608 next 429\n",
      "2023-07-01 13:38:25.239274: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75cc34200 of size 8388608 next 432\n",
      "2023-07-01 13:38:25.239277: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75d434200 of size 8388608 next 433\n",
      "2023-07-01 13:38:25.239280: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75dc34200 of size 4194304 next 440\n",
      "2023-07-01 13:38:25.239282: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75e034200 of size 4194304 next 441\n",
      "2023-07-01 13:38:25.239285: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75e434200 of size 4194304 next 444\n",
      "2023-07-01 13:38:25.239288: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75e834200 of size 4194304 next 445\n",
      "2023-07-01 13:38:25.239291: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75ec34200 of size 8388608 next 452\n",
      "2023-07-01 13:38:25.239293: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75f434200 of size 8388608 next 453\n",
      "2023-07-01 13:38:25.239296: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 75fc34200 of size 8388608 next 456\n",
      "2023-07-01 13:38:25.239299: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 760434200 of size 8388608 next 457\n",
      "2023-07-01 13:38:25.239301: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 760c34200 of size 8388608 next 460\n",
      "2023-07-01 13:38:25.239304: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 761434200 of size 8388608 next 461\n",
      "2023-07-01 13:38:25.239307: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 761c34200 of size 8388608 next 464\n",
      "2023-07-01 13:38:25.239310: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 762434200 of size 8388608 next 465\n",
      "2023-07-01 13:38:25.239313: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 762c34200 of size 8388608 next 472\n",
      "2023-07-01 13:38:25.239315: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 763434200 of size 8388608 next 473\n",
      "2023-07-01 13:38:25.239318: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 763c34200 of size 8388608 next 476\n",
      "2023-07-01 13:38:25.239321: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 764434200 of size 8388608 next 477\n",
      "2023-07-01 13:38:25.239324: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 764c34200 of size 8388608 next 480\n",
      "2023-07-01 13:38:25.239326: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 765434200 of size 8388608 next 481\n",
      "2023-07-01 13:38:25.239329: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 765c34200 of size 8388608 next 484\n",
      "2023-07-01 13:38:25.239332: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 766434200 of size 8388608 next 485\n",
      "2023-07-01 13:38:25.239335: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 766c34200 of size 4194304 next 492\n",
      "2023-07-01 13:38:25.239338: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 767034200 of size 4194304 next 493\n",
      "2023-07-01 13:38:25.239340: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 767434200 of size 4194304 next 496\n",
      "2023-07-01 13:38:25.239343: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 767834200 of size 4194304 next 497\n",
      "2023-07-01 13:38:25.239346: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 767c34200 of size 8388608 next 504\n",
      "2023-07-01 13:38:25.239349: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 768434200 of size 8388608 next 505\n",
      "2023-07-01 13:38:25.239352: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 768c34200 of size 8388608 next 508\n",
      "2023-07-01 13:38:25.239355: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 769434200 of size 8388608 next 509\n",
      "2023-07-01 13:38:25.239358: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 769c34200 of size 8388608 next 512\n",
      "2023-07-01 13:38:25.239361: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76a434200 of size 8388608 next 513\n",
      "2023-07-01 13:38:25.239364: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76ac34200 of size 8388608 next 516\n",
      "2023-07-01 13:38:25.239367: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76b434200 of size 8388608 next 517\n",
      "2023-07-01 13:38:25.239370: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76bc34200 of size 8388608 next 524\n",
      "2023-07-01 13:38:25.239373: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76c434200 of size 8388608 next 525\n",
      "2023-07-01 13:38:25.239375: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76cc34200 of size 8388608 next 528\n",
      "2023-07-01 13:38:25.239378: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76d434200 of size 8388608 next 529\n",
      "2023-07-01 13:38:25.239381: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76dc34200 of size 8388608 next 532\n",
      "2023-07-01 13:38:25.239384: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76e434200 of size 8388608 next 533\n",
      "2023-07-01 13:38:25.239387: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76ec34200 of size 8388608 next 536\n",
      "2023-07-01 13:38:25.239390: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76f434200 of size 8388608 next 537\n",
      "2023-07-01 13:38:25.239392: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 76fc34200 of size 4194304 next 544\n",
      "2023-07-01 13:38:25.239395: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 770034200 of size 4194304 next 545\n",
      "2023-07-01 13:38:25.239398: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 770434200 of size 4194304 next 548\n",
      "2023-07-01 13:38:25.239401: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 770834200 of size 4194304 next 549\n",
      "2023-07-01 13:38:25.239404: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 770c34200 of size 14356480 next 556\n",
      "2023-07-01 13:38:25.239408: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7719e5200 of size 14356480 next 557\n",
      "2023-07-01 13:38:25.239411: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 772796200 of size 3538944 next 690\n",
      "2023-07-01 13:38:25.239416: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 772af6200 of size 3571712 next 611\n",
      "2023-07-01 13:38:25.239420: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 772e5e200 of size 7168 next 581\n",
      "2023-07-01 13:38:25.239423: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 772e5fe00 of size 6912 next 658\n",
      "2023-07-01 13:38:25.239427: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 772e61900 of size 6912 next 750\n",
      "2023-07-01 13:38:25.239431: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 772e63400 of size 912128 next 765\n",
      "2023-07-01 13:38:25.239434: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 772f41f00 of size 760576 next 635\n",
      "2023-07-01 13:38:25.239438: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 772ffba00 of size 1878016 next 801\n",
      "2023-07-01 13:38:25.239442: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7731c6200 of size 892928 next 651\n",
      "2023-07-01 13:38:25.239446: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 7732a0200 of size 746496 next 725\n",
      "2023-07-01 13:38:25.239450: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 773356600 of size 28573696 next 728\n",
      "2023-07-01 13:38:25.239453: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 774e96600 of size 28573696 next 767\n",
      "2023-07-01 13:38:25.239457: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7769d6600 of size 28573696 next 828\n",
      "2023-07-01 13:38:25.239461: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 778516600 of size 28311552 next 726\n",
      "2023-07-01 13:38:25.239466: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 77a016600 of size 28311552 next 667\n",
      "2023-07-01 13:38:25.239469: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 77bb16600 of size 28311552 next 694\n",
      "2023-07-01 13:38:25.239473: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 77d616600 of size 6083072 next 614\n",
      "2023-07-01 13:38:25.239476: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 77dbe3800 of size 3538944 next 729\n",
      "2023-07-01 13:38:25.239480: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 77df43800 of size 6083072 next 776\n",
      "2023-07-01 13:38:25.239484: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 77e510a00 of size 1520896 next 701\n",
      "2023-07-01 13:38:25.239487: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 77e683f00 of size 46004480 next 700\n",
      "2023-07-01 13:38:25.239492: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 781263800 of size 6083072 next 824\n",
      "2023-07-01 13:38:25.239496: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 781830a00 of size 5971968 next 594\n",
      "2023-07-01 13:38:25.239499: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 781de2a00 of size 5971968 next 827\n",
      "2023-07-01 13:38:25.239503: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 782394a00 of size 28311552 next 693\n",
      "2023-07-01 13:38:25.239513: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 783e94a00 of size 565248 next 685\n",
      "2023-07-01 13:38:25.239517: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 783f1ea00 of size 3538944 next 670\n",
      "2023-07-01 13:38:25.239520: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78427ea00 of size 1506816 next 680\n",
      "2023-07-01 13:38:25.239524: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 7843ee800 of size 753408 next 638\n",
      "2023-07-01 13:38:25.239527: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7844a6700 of size 753408 next 664\n",
      "2023-07-01 13:38:25.239531: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78455e600 of size 3538944 next 600\n",
      "2023-07-01 13:38:25.239535: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7848be600 of size 3571712 next 716\n",
      "2023-07-01 13:38:25.239538: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 784c26600 of size 3571712 next 720\n",
      "2023-07-01 13:38:25.239542: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 784f8e600 of size 3538944 next 576\n",
      "2023-07-01 13:38:25.239545: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7852ee600 of size 3538944 next 595\n",
      "2023-07-01 13:38:25.239549: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78564e600 of size 884736 next 591\n",
      "2023-07-01 13:38:25.239552: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 785726600 of size 2810880 next 809\n",
      "2023-07-01 13:38:25.239556: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7859d4a00 of size 28573696 next 650\n",
      "2023-07-01 13:38:25.239560: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 787514a00 of size 14155776 next 687\n",
      "2023-07-01 13:38:25.239563: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 788294a00 of size 3538944 next 648\n",
      "2023-07-01 13:38:25.239567: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7885f4a00 of size 28311552 next 592\n",
      "2023-07-01 13:38:25.239570: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78a0f4a00 of size 28311552 next 617\n",
      "2023-07-01 13:38:25.239574: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78bbf4a00 of size 28311552 next 817\n",
      "2023-07-01 13:38:25.239577: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78d6f4a00 of size 5971968 next 718\n",
      "2023-07-01 13:38:25.239581: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78dca6a00 of size 3538944 next 761\n",
      "2023-07-01 13:38:25.239584: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78e006a00 of size 3538944 next 819\n",
      "2023-07-01 13:38:25.239588: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78e366a00 of size 3538944 next 633\n",
      "2023-07-01 13:38:25.239591: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78e6c6a00 of size 14155776 next 637\n",
      "2023-07-01 13:38:25.239595: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78f446a00 of size 3538944 next 798\n",
      "2023-07-01 13:38:25.239598: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78f7a6a00 of size 3538944 next 596\n",
      "2023-07-01 13:38:25.239602: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78fb06a00 of size 3538944 next 747\n",
      "2023-07-01 13:38:25.239605: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 78fe66a00 of size 5971968 next 808\n",
      "2023-07-01 13:38:25.239609: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 790418a00 of size 5971968 next 813\n",
      "2023-07-01 13:38:25.239612: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7909caa00 of size 3538944 next 773\n",
      "2023-07-01 13:38:25.239616: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 790d2aa00 of size 5750784 next 800\n",
      "2023-07-01 13:38:25.239620: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7912a6a00 of size 5971968 next 796\n",
      "2023-07-01 13:38:25.239623: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 791858a00 of size 28311552 next 715\n",
      "2023-07-01 13:38:25.239627: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 793358a00 of size 28311552 next 826\n",
      "2023-07-01 13:38:25.239630: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 794e58a00 of size 28311552 next 723\n",
      "2023-07-01 13:38:25.239634: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 796958a00 of size 28311552 next 759\n",
      "2023-07-01 13:38:25.239637: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 798458a00 of size 1492992 next 627\n",
      "2023-07-01 13:38:25.239641: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7985c5200 of size 3538944 next 688\n",
      "2023-07-01 13:38:25.239644: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 798925200 of size 14155776 next 764\n",
      "2023-07-01 13:38:25.239648: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7996a5200 of size 3538944 next 695\n",
      "2023-07-01 13:38:25.239651: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 799a05200 of size 3538944 next 830\n",
      "2023-07-01 13:38:25.239655: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 799d65200 of size 884736 next 766\n",
      "2023-07-01 13:38:25.239658: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 799e3d200 of size 3538944 next 642\n",
      "2023-07-01 13:38:25.239662: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79a19d200 of size 5971968 next 636\n",
      "2023-07-01 13:38:25.239665: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79a74f200 of size 5971968 next 665\n",
      "2023-07-01 13:38:25.239669: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79ad01200 of size 5971968 next 632\n",
      "2023-07-01 13:38:25.239672: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79b2b3200 of size 1492992 next 580\n",
      "2023-07-01 13:38:25.239676: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79b41fa00 of size 3538944 next 598\n",
      "2023-07-01 13:38:25.239679: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79b77fa00 of size 4478976 next 607\n",
      "2023-07-01 13:38:25.239683: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79bbc5200 of size 28311552 next 793\n",
      "2023-07-01 13:38:25.239686: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79d6c5200 of size 28311552 next 587\n",
      "2023-07-01 13:38:25.239690: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 79f1c5200 of size 28311552 next 739\n",
      "2023-07-01 13:38:25.239693: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a0cc5200 of size 28311552 next 807\n",
      "2023-07-01 13:38:25.239697: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a27c5200 of size 14155776 next 702\n",
      "2023-07-01 13:38:25.239700: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a3545200 of size 3538944 next 621\n",
      "2023-07-01 13:38:25.239704: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a38a5200 of size 3538944 next 671\n",
      "2023-07-01 13:38:25.239707: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a3c05200 of size 884736 next 831\n",
      "2023-07-01 13:38:25.239711: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a3cdd200 of size 3538944 next 712\n",
      "2023-07-01 13:38:25.239714: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a403d200 of size 30965760 next 662\n",
      "2023-07-01 13:38:25.239718: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a5dc5200 of size 28311552 next 644\n",
      "2023-07-01 13:38:25.239721: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a78c5200 of size 28311552 next 727\n",
      "2023-07-01 13:38:25.239725: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7a93c5200 of size 28311552 next 760\n",
      "2023-07-01 13:38:25.239728: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7aaec5200 of size 28311552 next 797\n",
      "2023-07-01 13:38:25.239732: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7ac9c5200 of size 28311552 next 703\n",
      "2023-07-01 13:38:25.239735: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7ae4c5200 of size 28311552 next 762\n",
      "2023-07-01 13:38:25.239739: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7affc5200 of size 28311552 next 755\n",
      "2023-07-01 13:38:25.239742: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b1ac5200 of size 28311552 next 626\n",
      "2023-07-01 13:38:25.239746: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b35c5200 of size 6027264 next 639\n",
      "2023-07-01 13:38:25.239750: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b3b84a00 of size 6027264 next 606\n",
      "2023-07-01 13:38:25.239754: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b4144200 of size 1506816 next 779\n",
      "2023-07-01 13:38:25.239758: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b42b4000 of size 3571712 next 582\n",
      "2023-07-01 13:38:25.239762: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b461c000 of size 3571712 next 610\n",
      "2023-07-01 13:38:25.239765: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b4984000 of size 892928 next 645\n",
      "2023-07-01 13:38:25.239769: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b4a5e000 of size 3571712 next 669\n",
      "2023-07-01 13:38:25.239773: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b4dc6000 of size 37743104 next 672\n",
      "2023-07-01 13:38:25.239777: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b71c4a00 of size 6027264 next 583\n",
      "2023-07-01 13:38:25.239781: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b7784200 of size 3571712 next 656\n",
      "2023-07-01 13:38:25.239785: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b7aec200 of size 3571712 next 655\n",
      "2023-07-01 13:38:25.239789: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b7e54200 of size 21430272 next 735\n",
      "2023-07-01 13:38:25.239793: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7b92c4200 of size 28573696 next 810\n",
      "2023-07-01 13:38:25.239797: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7bae04200 of size 28573696 next 630\n",
      "2023-07-01 13:38:25.239801: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7bc944200 of size 28573696 next 711\n",
      "2023-07-01 13:38:25.239805: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7be484200 of size 6083072 next 615\n",
      "2023-07-01 13:38:25.239809: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7bea51400 of size 6083072 next 673\n",
      "2023-07-01 13:38:25.239813: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7bf01e600 of size 1520896 next 770\n",
      "2023-07-01 13:38:25.239817: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7bf191b00 of size 49543424 next 660\n",
      "2023-07-01 13:38:25.239821: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c20d1400 of size 6083072 next 609\n",
      "2023-07-01 13:38:25.239825: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c269e600 of size 3571712 next 821\n",
      "2023-07-01 13:38:25.239828: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c2a06600 of size 3571712 next 593\n",
      "2023-07-01 13:38:25.239832: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c2d6e600 of size 6027264 next 763\n",
      "2023-07-01 13:38:25.239835: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c332de00 of size 6027264 next 753\n",
      "2023-07-01 13:38:25.239839: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c38ed600 of size 3571712 next 784\n",
      "2023-07-01 13:38:25.239842: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c3c55600 of size 5804032 next 663\n",
      "2023-07-01 13:38:25.239846: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c41de600 of size 28573696 next 754\n",
      "2023-07-01 13:38:25.239849: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c5d1e600 of size 3571712 next 778\n",
      "2023-07-01 13:38:25.239853: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c6086600 of size 14286848 next 588\n",
      "2023-07-01 13:38:25.239857: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c6e26600 of size 3571712 next 732\n",
      "2023-07-01 13:38:25.239860: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c718e600 of size 3571712 next 678\n",
      "2023-07-01 13:38:25.239864: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c74f6600 of size 892928 next 775\n",
      "2023-07-01 13:38:25.239867: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c75d0600 of size 3571712 next 733\n",
      "2023-07-01 13:38:25.239871: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c7938600 of size 6083072 next 777\n",
      "2023-07-01 13:38:25.239874: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c7f05800 of size 6083072 next 721\n",
      "2023-07-01 13:38:25.239878: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c84d2a00 of size 6083072 next 698\n",
      "2023-07-01 13:38:25.239881: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c8a9fc00 of size 1520896 next 805\n",
      "2023-07-01 13:38:25.239884: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c8c13100 of size 3571712 next 788\n",
      "2023-07-01 13:38:25.239888: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c8f7b100 of size 4076800 next 641\n",
      "2023-07-01 13:38:25.239892: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7c935e600 of size 28573696 next 785\n",
      "2023-07-01 13:38:25.239895: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7cae9e600 of size 28573696 next 649\n",
      "2023-07-01 13:38:25.239899: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7cc9de600 of size 28573696 next 823\n",
      "2023-07-01 13:38:25.239902: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7ce51e600 of size 28573696 next 742\n",
      "2023-07-01 13:38:25.239906: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d005e600 of size 28573696 next 714\n",
      "2023-07-01 13:38:25.239909: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d1b9e600 of size 6027264 next 654\n",
      "2023-07-01 13:38:25.239913: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d215de00 of size 6027264 next 748\n",
      "2023-07-01 13:38:25.239916: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d271d600 of size 1506816 next 708\n",
      "2023-07-01 13:38:25.239920: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d288d400 of size 3571712 next 612\n",
      "2023-07-01 13:38:25.239923: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d2bf5400 of size 3571712 next 683\n",
      "2023-07-01 13:38:25.239927: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d2f5d400 of size 3571712 next 676\n",
      "2023-07-01 13:38:25.239930: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d32c5400 of size 4297216 next 795\n",
      "2023-07-01 13:38:25.239934: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d36de600 of size 28573696 next 586\n",
      "2023-07-01 13:38:25.239938: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d521e600 of size 34600960 next 674\n",
      "2023-07-01 13:38:25.239941: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d731de00 of size 6027264 next 589\n",
      "2023-07-01 13:38:25.239945: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d78dd600 of size 14286848 next 689\n",
      "2023-07-01 13:38:25.239948: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d867d600 of size 892928 next 679\n",
      "2023-07-01 13:38:25.239952: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d8757600 of size 3571712 next 657\n",
      "2023-07-01 13:38:25.239955: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d8abf600 of size 9822208 next 652\n",
      "2023-07-01 13:38:25.239959: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7d941d600 of size 28573696 next 772\n",
      "2023-07-01 13:38:25.239963: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7daf5d600 of size 28573696 next 740\n",
      "2023-07-01 13:38:25.239966: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7dca9d600 of size 28573696 next 631\n",
      "2023-07-01 13:38:25.239969: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7de5dd600 of size 6083072 next 719\n",
      "2023-07-01 13:38:25.239973: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7debaa800 of size 1520896 next 585\n",
      "2023-07-01 13:38:25.239977: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7ded1dd00 of size 49543424 next 620\n",
      "2023-07-01 13:38:25.239980: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7e1c5d600 of size 6083072 next 745\n",
      "2023-07-01 13:38:25.239983: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7e222a800 of size 3571712 next 751\n",
      "2023-07-01 13:38:25.239987: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7e2592800 of size 3571712 next 781\n",
      "2023-07-01 13:38:25.239990: I tensorflow/tsl/framework/bfc_allocator.cc:1095] Free  at 7e28fa800 of size 21430272 next 597\n",
      "2023-07-01 13:38:25.239994: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7e3d6a800 of size 28573696 next 686\n",
      "2023-07-01 13:38:25.239998: I tensorflow/tsl/framework/bfc_allocator.cc:1095] InUse at 7e58aa800 of size 29710336 next 18446744073709551615\n",
      "2023-07-01 13:38:25.240001: I tensorflow/tsl/framework/bfc_allocator.cc:1100]      Summary of in-use Chunks by size: \n",
      "2023-07-01 13:38:25.240007: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 65 Chunks of size 256 totalling 16.2KiB\n",
      "2023-07-01 13:38:25.240012: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 1280 totalling 1.2KiB\n",
      "2023-07-01 13:38:25.240016: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 3 Chunks of size 1792 totalling 5.2KiB\n",
      "2023-07-01 13:38:25.240021: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 180 Chunks of size 2048 totalling 360.0KiB\n",
      "2023-07-01 13:38:25.240025: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 23 Chunks of size 6912 totalling 155.2KiB\n",
      "2023-07-01 13:38:25.240029: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 31 Chunks of size 7168 totalling 217.0KiB\n",
      "2023-07-01 13:38:25.240033: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 7424 totalling 7.2KiB\n",
      "2023-07-01 13:38:25.240037: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 24 Chunks of size 8192 totalling 192.0KiB\n",
      "2023-07-01 13:38:25.240041: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 8960 totalling 8.8KiB\n",
      "2023-07-01 13:38:25.240045: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 11520 totalling 11.2KiB\n",
      "2023-07-01 13:38:25.240049: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 2 Chunks of size 14080 totalling 27.5KiB\n",
      "2023-07-01 13:38:25.240053: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 108 Chunks of size 16384 totalling 1.69MiB\n",
      "2023-07-01 13:38:25.240057: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 3 Chunks of size 28160 totalling 82.5KiB\n",
      "2023-07-01 13:38:25.240061: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 753408 totalling 735.8KiB\n",
      "2023-07-01 13:38:25.240065: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 3 Chunks of size 884736 totalling 2.53MiB\n",
      "2023-07-01 13:38:25.240069: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 4 Chunks of size 892928 totalling 3.41MiB\n",
      "2023-07-01 13:38:25.240073: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 912128 totalling 890.8KiB\n",
      "2023-07-01 13:38:25.240077: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 932352 totalling 910.5KiB\n",
      "2023-07-01 13:38:25.240081: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 2 Chunks of size 1492992 totalling 2.85MiB\n",
      "2023-07-01 13:38:25.240085: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 3 Chunks of size 1506816 totalling 4.31MiB\n",
      "2023-07-01 13:38:25.240089: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 4 Chunks of size 1520896 totalling 5.80MiB\n",
      "2023-07-01 13:38:25.240093: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 1878016 totalling 1.79MiB\n",
      "2023-07-01 13:38:25.240097: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 2810880 totalling 2.68MiB\n",
      "2023-07-01 13:38:25.240101: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 22 Chunks of size 3538944 totalling 74.25MiB\n",
      "2023-07-01 13:38:25.240105: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 22 Chunks of size 3571712 totalling 74.94MiB\n",
      "2023-07-01 13:38:25.240108: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 4076800 totalling 3.89MiB\n",
      "2023-07-01 13:38:25.240113: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 48 Chunks of size 4194304 totalling 192.00MiB\n",
      "2023-07-01 13:38:25.240117: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 4297216 totalling 4.10MiB\n",
      "2023-07-01 13:38:25.240120: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 4478976 totalling 4.27MiB\n",
      "2023-07-01 13:38:25.240124: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 5750784 totalling 5.48MiB\n",
      "2023-07-01 13:38:25.240128: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 5804032 totalling 5.54MiB\n",
      "2023-07-01 13:38:25.240132: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 9 Chunks of size 5971968 totalling 51.26MiB\n",
      "2023-07-01 13:38:25.240136: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 8 Chunks of size 6027264 totalling 45.98MiB\n",
      "2023-07-01 13:38:25.240141: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 11 Chunks of size 6083072 totalling 63.81MiB\n",
      "2023-07-01 13:38:25.240144: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 8331520 totalling 7.95MiB\n",
      "2023-07-01 13:38:25.240148: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 8388352 totalling 8.00MiB\n",
      "2023-07-01 13:38:25.240152: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 142 Chunks of size 8388608 totalling 1.11GiB\n",
      "2023-07-01 13:38:25.240156: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 9822208 totalling 9.37MiB\n",
      "2023-07-01 13:38:25.240160: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 11935744 totalling 11.38MiB\n",
      "2023-07-01 13:38:25.240164: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 4 Chunks of size 14155776 totalling 54.00MiB\n",
      "2023-07-01 13:38:25.240168: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 2 Chunks of size 14286848 totalling 27.25MiB\n",
      "2023-07-01 13:38:25.240172: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 6 Chunks of size 14356480 totalling 82.15MiB\n",
      "2023-07-01 13:38:25.240176: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 15028224 totalling 14.33MiB\n",
      "2023-07-01 13:38:25.240180: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 2 Chunks of size 15902720 totalling 30.33MiB\n",
      "2023-07-01 13:38:25.240184: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 21430272 totalling 20.44MiB\n",
      "2023-07-01 13:38:25.240189: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 23 Chunks of size 28311552 totalling 621.00MiB\n",
      "2023-07-01 13:38:25.240193: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 18 Chunks of size 28573696 totalling 490.50MiB\n",
      "2023-07-01 13:38:25.240197: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 28712960 totalling 27.38MiB\n",
      "2023-07-01 13:38:25.240201: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 29710336 totalling 28.33MiB\n",
      "2023-07-01 13:38:25.240205: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 30965760 totalling 29.53MiB\n",
      "2023-07-01 13:38:25.240209: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 34600960 totalling 33.00MiB\n",
      "2023-07-01 13:38:25.240213: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 37743104 totalling 35.99MiB\n",
      "2023-07-01 13:38:25.240217: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 1 Chunks of size 46004480 totalling 43.87MiB\n",
      "2023-07-01 13:38:25.240221: I tensorflow/tsl/framework/bfc_allocator.cc:1103] 2 Chunks of size 49543424 totalling 94.50MiB\n",
      "2023-07-01 13:38:25.240225: I tensorflow/tsl/framework/bfc_allocator.cc:1107] Sum Total of in-use chunks: 3.28GiB\n",
      "2023-07-01 13:38:25.240229: I tensorflow/tsl/framework/bfc_allocator.cc:1109] Total bytes in pool: 3547332608 memory_limit_: 3547332608 available bytes: 0 curr_region_allocation_bytes_: 7094665216\n",
      "2023-07-01 13:38:25.240238: I tensorflow/tsl/framework/bfc_allocator.cc:1114] Stats: \n",
      "Limit:                      3547332608\n",
      "InUse:                      3522607360\n",
      "MaxInUse:                   3541378560\n",
      "NumAllocs:                       33507\n",
      "MaxAllocSize:                 53807104\n",
      "Reserved:                            0\n",
      "PeakReserved:                        0\n",
      "LargestFreeBlock:                    0\n",
      "\n",
      "2023-07-01 13:38:25.240261: W tensorflow/tsl/framework/bfc_allocator.cc:497] ****************************************************************************************************\n",
      "2023-07-01 13:38:25.240284: W tensorflow/core/framework/op_kernel.cc:1830] OP_REQUIRES failed at einsum_op_impl.h:518 : RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[16,8,109,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n",
      "2023-07-01 13:38:25.240306: I tensorflow/core/common_runtime/executor.cc:1197] [/job:localhost/replica:0/task:0/device:GPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[16,8,109,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n",
      "\t [[{{node transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum}}]]\n",
      "Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n",
      "\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "Graph execution error:\n\nDetected at node 'transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum' defined at (most recent call last):\n    File \"/usr/lib/python3.8/runpy.py\", line 194, in _run_module_as_main\n      return _run_code(code, main_globals, None,\n    File \"/usr/lib/python3.8/runpy.py\", line 87, in _run_code\n      exec(code, run_globals)\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/usr/local/lib/python3.8/dist-packages/traitlets/config/application.py\", line 1043, in launch_instance\n      app.start()\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelapp.py\", line 725, in start\n      self.io_loop.start()\n    File \"/usr/local/lib/python3.8/dist-packages/tornado/platform/asyncio.py\", line 215, in start\n      self.asyncio_loop.run_forever()\n    File \"/usr/lib/python3.8/asyncio/base_events.py\", line 570, in run_forever\n      self._run_once()\n    File \"/usr/lib/python3.8/asyncio/base_events.py\", line 1859, in _run_once\n      handle._run()\n    File \"/usr/lib/python3.8/asyncio/events.py\", line 81, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 513, in dispatch_queue\n      await self.process_one()\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 502, in process_one\n      await dispatch(*args)\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 409, in dispatch_shell\n      await result\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 729, in execute_request\n      reply_content = await reply_content\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py\", line 422, in do_execute\n      res = shell.run_cell(\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/zmqshell.py\", line 540, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 2961, in run_cell\n      result = self._run_cell(\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3016, in _run_cell\n      result = runner(coro)\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3221, in run_cell_async\n      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3400, in run_ast_nodes\n      if await self.run_code(code, result, async_=asy):\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3460, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_12376/1555153759.py\", line 1, in <module>\n      training.fit(train_ted_ds, val_ted_ds)\n    File \"/tf/2023/attention-is-all-you-need/train.py\", line 76, in fit\n      self.training_history = self.transformer.fit(train_batches,\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1685, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1284, in train_function\n      return step_function(self, iterator)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1268, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1249, in run_step\n      outputs = model.train_step(data)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1050, in train_step\n      y_pred = self(x, training=True)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 558, in __call__\n      return super().__call__(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/transformer.py\", line 27, in call\n      x = self.decoder(x, context)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/decoder.py\", line 52, in call\n      for i in range(self.num_layers):\n    File \"/tf/2023/attention-is-all-you-need/model/decoder.py\", line 53, in call\n      x = self.decoder_layer[i](x, context)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/decoder.py\", line 22, in call\n      x = self.cross_attn(x=x, context=context)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/attention.py\", line 16, in call\n      attn_output, attn_scores = self.multi_head_attention(query=x,\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/layers/attention/multi_head_attention.py\", line 598, in call\n      attention_output, attention_scores = self._compute_attention(\n    File \"/usr/local/lib/python3.8/dist-packages/keras/layers/attention/multi_head_attention.py\", line 527, in _compute_attention\n      attention_scores = tf.einsum(self._dot_product_equation, key, query)\nNode: 'transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum'\nOOM when allocating tensor with shape[16,8,109,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n [Op:__inference_train_function_26118]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtraining\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_ted_ds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_ted_ds\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/tf/2023/attention-is-all-you-need/train.py:76\u001b[0m, in \u001b[0;36mTransformerTraining.fit\u001b[0;34m(self, train_batches, val_batches)\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m, train_batches, val_batches):\n\u001b[0;32m---> 76\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_history \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransformer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_batches\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     77\u001b[0m \u001b[43m                                                 \u001b[49m\u001b[43msteps_per_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msteps_per_epochs\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtrain_batches\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     78\u001b[0m \u001b[43m                                                 \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnum_epochs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     79\u001b[0m \u001b[43m                                                 \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mval_batches\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     80\u001b[0m \u001b[43m                                                 \u001b[49m\u001b[43mvalidation_steps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msteps_per_epochs\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mval_batches\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     81\u001b[0m \u001b[43m                                                \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcp_callback\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     82\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_history\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py:52\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     51\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 52\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m pywrap_tfe\u001b[38;5;241m.\u001b[39mTFE_Py_Execute(ctx\u001b[38;5;241m.\u001b[39m_handle, device_name, op_name,\n\u001b[1;32m     53\u001b[0m                                       inputs, attrs, num_outputs)\n\u001b[1;32m     54\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     55\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: Graph execution error:\n\nDetected at node 'transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum' defined at (most recent call last):\n    File \"/usr/lib/python3.8/runpy.py\", line 194, in _run_module_as_main\n      return _run_code(code, main_globals, None,\n    File \"/usr/lib/python3.8/runpy.py\", line 87, in _run_code\n      exec(code, run_globals)\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/usr/local/lib/python3.8/dist-packages/traitlets/config/application.py\", line 1043, in launch_instance\n      app.start()\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelapp.py\", line 725, in start\n      self.io_loop.start()\n    File \"/usr/local/lib/python3.8/dist-packages/tornado/platform/asyncio.py\", line 215, in start\n      self.asyncio_loop.run_forever()\n    File \"/usr/lib/python3.8/asyncio/base_events.py\", line 570, in run_forever\n      self._run_once()\n    File \"/usr/lib/python3.8/asyncio/base_events.py\", line 1859, in _run_once\n      handle._run()\n    File \"/usr/lib/python3.8/asyncio/events.py\", line 81, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 513, in dispatch_queue\n      await self.process_one()\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 502, in process_one\n      await dispatch(*args)\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 409, in dispatch_shell\n      await result\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py\", line 729, in execute_request\n      reply_content = await reply_content\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py\", line 422, in do_execute\n      res = shell.run_cell(\n    File \"/usr/local/lib/python3.8/dist-packages/ipykernel/zmqshell.py\", line 540, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 2961, in run_cell\n      result = self._run_cell(\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3016, in _run_cell\n      result = runner(coro)\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3221, in run_cell_async\n      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3400, in run_ast_nodes\n      if await self.run_code(code, result, async_=asy):\n    File \"/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py\", line 3460, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_12376/1555153759.py\", line 1, in <module>\n      training.fit(train_ted_ds, val_ted_ds)\n    File \"/tf/2023/attention-is-all-you-need/train.py\", line 76, in fit\n      self.training_history = self.transformer.fit(train_batches,\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1685, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1284, in train_function\n      return step_function(self, iterator)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1268, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1249, in run_step\n      outputs = model.train_step(data)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 1050, in train_step\n      y_pred = self(x, training=True)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/training.py\", line 558, in __call__\n      return super().__call__(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/transformer.py\", line 27, in call\n      x = self.decoder(x, context)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/decoder.py\", line 52, in call\n      for i in range(self.num_layers):\n    File \"/tf/2023/attention-is-all-you-need/model/decoder.py\", line 53, in call\n      x = self.decoder_layer[i](x, context)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/decoder.py\", line 22, in call\n      x = self.cross_attn(x=x, context=context)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/tf/2023/attention-is-all-you-need/model/attention.py\", line 16, in call\n      attn_output, attn_scores = self.multi_head_attention(query=x,\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py\", line 1145, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.8/dist-packages/keras/layers/attention/multi_head_attention.py\", line 598, in call\n      attention_output, attention_scores = self._compute_attention(\n    File \"/usr/local/lib/python3.8/dist-packages/keras/layers/attention/multi_head_attention.py\", line 527, in _compute_attention\n      attention_scores = tf.einsum(self._dot_product_equation, key, query)\nNode: 'transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum'\nOOM when allocating tensor with shape[16,8,109,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node transformer/decoder_block/decoder_layer_3/cross_attention_3/multi_head_attention_11/einsum/Einsum}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n [Op:__inference_train_function_26118]"
     ]
    }
   ],
   "source": [
    "training.fit(train_ted_ds, val_ted_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a64fa25c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#training.get_model().save_weights('transformer/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93853828",
   "metadata": {},
   "outputs": [],
   "source": [
    "#training.get_model().evaluate(val_ted_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef1cace6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}