tfrecord_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from tqdm import tqdm\n",
    "import glob\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "name_dict = dict(\n",
    "    inputs=[\n",
    "        # Your Inputs\n",
    "    ],\n",
    "    targets=[\n",
    "        # Your Targets For Inputs Respectively.\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_files = glob.glob('../val_2005/summary/*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../val_2005/summary/US20050225040A1_summary',\n",
       " '../val_2005/summary/US20050149233A1_summary',\n",
       " '../val_2005/summary/US20050034453A1_summary',\n",
       " '../val_2005/summary/US20050073128A1_summary',\n",
       " '../val_2005/summary/US20050243340A1_summary',\n",
       " '../val_2005/summary/US20050083025A1_summary',\n",
       " '../val_2005/summary/US20050158169A1_summary',\n",
       " '../val_2005/summary/US20050007372A1_summary',\n",
       " '../val_2005/summary/US20050118919A1_summary',\n",
       " '../val_2005/summary/US20050103926A1_summary']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_files[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "for summary_file in summary_files:\n",
    "    \n",
    "    filepath_elements = summary_file.split('/')\n",
    "    pa_id = filepath_elements[3].split('_')[0]\n",
    "    \n",
    "    with open(summary_file, \"r\") as file:\n",
    "        # Read each line in the file, readlines() returns a list of lines\n",
    "        summary = file.readlines()\n",
    "        # Combine the lines in the list into a string\n",
    "        summary = \"\".join(summary)\n",
    "        if (len(summary.split())) > 800:\n",
    "            continue\n",
    "    \n",
    "    firstclaim_file = '../val_2005/firstclaim/' + pa_id + '_firstclaim'\n",
    "    \n",
    "    with open(firstclaim_file, \"r\") as file2:\n",
    "        # Read each line in the file, readlines() returns a list of lines\n",
    "        firstclaim = file2.readlines()\n",
    "        # Combine the lines in the list into a string\n",
    "        firstclaim = \"\".join(firstclaim)\n",
    "    \n",
    "    name_dict[\"inputs\"].append(summary)\n",
    "    name_dict[\"targets\"].append(firstclaim)\n",
    "    \n",
    "    count += 1\n",
    "    if count >= 100:\n",
    "        break\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(name_dict)\n",
    "\n",
    "#print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = [\"inputs\", \"targets\"]\n",
    "\n",
    "df.to_csv('val_10k_output.csv', columns=header, index=False)\n",
    "\n",
    "csv = pd.read_csv(\"val_10k_output.csv\").values\n",
    "with tf.io.TFRecordWriter(\"pegasus/data/testdata/val_10k.tfrecords\") as writer:\n",
    "    for row in csv:\n",
    "        inputs, targets = row[:-1], row[-1]\n",
    "        example = tf.train.Example(\n",
    "            features=tf.train.Features(\n",
    "                feature={\n",
    "                    \"inputs\": tf.train.Feature(bytes_list=tf.train.BytesList(value=[inputs[0].encode('utf-8')])),\n",
    "                    \"targets\": tf.train.Feature(bytes_list=tf.train.BytesList(value=[targets.encode('utf-8')])),\n",
    "                }\n",
    "            )\n",
    "        )\n",
    "        writer.write(example.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = [\"inputs\", \"targets\"]\n",
    "df = pd.read_csv('val_10k_output.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inputs</th>\n",
       "      <th>targets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Generally, the present invention pertains to ...</td>\n",
       "      <td>1. A controller comprising: \\na programmable s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>An object of the present invention in an inve...</td>\n",
       "      <td>1. An inverted type front fork usable in a two...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The foregoing objects are realized by the pre...</td>\n",
       "      <td>1. An apparatus for processing image data repr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>If the transfer function of a power supply re...</td>\n",
       "      <td>1. A power supply comprising: \\na switching co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Therefore, the present invention aims at miti...</td>\n",
       "      <td>1. A clearance control device for controlling ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>The present invention resides in an EMI feedt...</td>\n",
       "      <td>1. A feedthrough terminal assembly for an acti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>The exemplary embodiments of the present inve...</td>\n",
       "      <td>1. A rinse solution for removing at least one ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>The present invention provides such a method ...</td>\n",
       "      <td>1. A method of integrating the operations of a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>The present invention provides an industrial ...</td>\n",
       "      <td>1. An equipment area network (EAN) for a plura...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>One object of the invention is to provide a b...</td>\n",
       "      <td>1. A bottle container provided with a protecti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               inputs  \\\n",
       "0    Generally, the present invention pertains to ...   \n",
       "1    An object of the present invention in an inve...   \n",
       "2    The foregoing objects are realized by the pre...   \n",
       "3    If the transfer function of a power supply re...   \n",
       "4    Therefore, the present invention aims at miti...   \n",
       "..                                                ...   \n",
       "95   The present invention resides in an EMI feedt...   \n",
       "96   The exemplary embodiments of the present inve...   \n",
       "97   The present invention provides such a method ...   \n",
       "98   The present invention provides an industrial ...   \n",
       "99   One object of the invention is to provide a b...   \n",
       "\n",
       "                                              targets  \n",
       "0   1. A controller comprising: \\na programmable s...  \n",
       "1   1. An inverted type front fork usable in a two...  \n",
       "2   1. An apparatus for processing image data repr...  \n",
       "3   1. A power supply comprising: \\na switching co...  \n",
       "4   1. A clearance control device for controlling ...  \n",
       "..                                                ...  \n",
       "95  1. A feedthrough terminal assembly for an acti...  \n",
       "96  1. A rinse solution for removing at least one ...  \n",
       "97  1. A method of integrating the operations of a...  \n",
       "98  1. An equipment area network (EAN) for a plura...  \n",
       "99  1. A bottle container provided with a protecti...  \n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_tensorflow2_p36)",
   "language": "python",
   "name": "conda_tensorflow2_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}