text_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c706c5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# download and install tesseract from here: https://github.com/UB-Mannheim/tesseract/wiki\n",
    "# install poppler : https://anaconda.org/conda-forge/poppler \n",
    "    # in anaconda prompt: conda install -c conda-forge poppler\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "85c00d8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pdf2image import convert_from_path\n",
    "\n",
    "from PIL import Image\n",
    "from pytesseract import pytesseract\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d793101d",
   "metadata": {},
   "source": [
    "# code for a single pdf below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bd1e2ac7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # pdf to an image\n",
    "\n",
    "# pdfs = r\"invoices/pdf_trial2.pdf\"\n",
    "# pages = convert_from_path(pdfs, 350)\n",
    "\n",
    "# i = 1\n",
    "# for page in pages:\n",
    "#     image_name = \"Page_\" + str(i) + \".jpg\"  \n",
    "#     page.save(image_name, \"JPEG\")\n",
    "#     i = i+1       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a4cccafd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e Brexit Party Organisaticn\n",
      "\n",
      "Date Item\n",
      "7-Nov-19 Manor House Single Bed & Breakfast\n",
      "7-Nov-19 Manor House Double Bed & Breakfast\n",
      "8-Nov-19 Manor House Single Bed & Breakfast\n",
      "8-Nov-19 Manor House Double Bed & Breakfast\n",
      "Deposit Due at 100%\n",
      "\n",
      "Proforma Number [i\n",
      "\n",
      "Page 1\n",
      "\n",
      "Date +Nov-i9\n",
      "\n",
      "Gross Rate Numbers\n",
      "\n",
      "£97.00 3\n",
      "£127.00 9\n",
      "£97.00 4\n",
      "£127.00 9\n",
      "\n",
      "Total\n",
      "£291.00\n",
      "£1,143.00\n",
      "£388.00\n",
      "£1,143.00\n",
      "\n",
      "£2,965.00\n",
      "\n",
      "To enable us to process your deposit payment efficiently, please quote the above\n",
      "pro forma reference number on all payments made to The Celtic Manor Resort\n",
      "\n",
      "surcharge bein Be n rd\n",
      "Please note This is Not a VAT Invoice\n",
      "\n",
      "To enabie you to pay us BACS we nave liste\n",
      "\n",
      "Cee\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# # Defining paths to tesseract.exe\n",
    "# # and the image we would be using\n",
    "# path_to_tesseract = r\"C:\\Program Files\\Tesseract-OCR\\tesseract.exe\"\n",
    "# image_path = r\"Page_1.jpg\"\n",
    "\n",
    "# # Opening the image & storing it in an image object\n",
    "# img = Image.open(image_path)\n",
    "\n",
    "# # Providing the tesseract executable\n",
    "# # location to pytesseract library\n",
    "# pytesseract.tesseract_cmd = path_to_tesseract\n",
    "\n",
    "# # Passing the image object to image_to_string() function\n",
    "# # This function will extract the text from the image\n",
    "# text = pytesseract.image_to_string(img)\n",
    "\n",
    "# # Displaying the extracted text\n",
    "# print(text[:-1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "48cf3dd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_full = pd.read_csv('spending_2019.csv')\n",
    "data = data_full.iloc[:,[0,1,3,4,5,7, 15]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "35c76056",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-4-24d4e5e181fc>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['invoice_id'] = data['RedactedSupportingInvoiceId']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        66836.0\n",
       "1        65844.0\n",
       "2        66755.0\n",
       "3        68767.0\n",
       "4        65153.0\n",
       "          ...   \n",
       "22715        NaN\n",
       "22716        NaN\n",
       "22717        NaN\n",
       "22718        NaN\n",
       "22719        NaN\n",
       "Name: RedactedSupportingInvoiceId, Length: 22720, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['invoice_id'] = data['RedactedSupportingInvoiceId'] \n",
    "data.pop('RedactedSupportingInvoiceId') \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82b18d0d",
   "metadata": {},
   "source": [
    "# code for all pdfs and images in a given folder\n",
    "\n",
    "- see paths below\n",
    "- check paths and directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c7d962b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished\n"
     ]
    }
   ],
   "source": [
    " # change invoices_trial to invoices\n",
    "invoice_directory = 'C:/Users/Helena/DISCUS_polspending/DISCUS_Hack_PoliticalSpending/invoices' \n",
    "\n",
    "Image.MAX_IMAGE_PIXELS = None # otherwise will get a DecompressionBombWarning due to a large size\n",
    "\n",
    "# print(\"Files and directories in a specified path:\")\n",
    "\n",
    "for filename in os.listdir(invoice_directory):\n",
    "    f = os.path.join(invoice_directory,filename)\n",
    "    if os.path.isfile(f):\n",
    "        # print(f)\n",
    "        \n",
    "        pdfs = f\n",
    "        pages = convert_from_path(pdfs, 350)\n",
    "        \n",
    "        i = 1\n",
    "        invoice_id = f[-9:-4]\n",
    "        # print('invoice no.' + invoice_id + 'converted and saved')\n",
    "        \n",
    "        for page in pages:\n",
    "            image_name = invoice_id +\"_Page_\" + str(i) + \".jpg\"  \n",
    "            page.save('images/'+ image_name, \"JPEG\")\n",
    "            i = i+1 \n",
    "\n",
    "print('finished')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0bc597f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished\n"
     ]
    }
   ],
   "source": [
    " # extracting text from images\n",
    "    \n",
    "images_directory = 'C:/Users/Helena/DISCUS_polspending/DISCUS_Hack_PoliticalSpending/images' \n",
    "path_to_tesseract = r\"C:\\Program Files\\Tesseract-OCR\\tesseract.exe\"\n",
    "\n",
    "dict_invoices = {'invoice_id':[] , 'text':[] }\n",
    "\n",
    "# print(\"Files and directories in a specified path:\")\n",
    "\n",
    "for filename in os.listdir(images_directory):\n",
    "    \n",
    "    f = os.path.join(images_directory,filename)\n",
    "    if os.path.isfile(f):\n",
    "       # print(f)\n",
    "        \n",
    "        image_path = f\n",
    "        \n",
    "        img = Image.open(image_path)\n",
    "        pytesseract.tesseract_cmd = path_to_tesseract\n",
    "        \n",
    "        invoice_no = image_path[-16:-11] \n",
    "        \n",
    "        text = pytesseract.image_to_string(img)\n",
    "        \n",
    "        # text needs to be appended to a dictionary: key = invoice_id, all pages are text\n",
    "        \n",
    "        dict_invoices[\"invoice_id\"].append(invoice_no)\n",
    "        dict_invoices[\"text\"].append(text)\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "#         print('invoice_no.:'+ invoice_no + '\\n********************\\n'+text +'\\n\\n')\n",
    "\n",
    "\n",
    "print('finished')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6b78d220",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>invoice_id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62311</td>\n",
       "      <td>SRIPT\\n\\nSth February 2020\\n\\nFantage, postpro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62312</td>\n",
       "      <td>PBD\\n\\nINVOICE\\n\\nAlliance HQ, lewelce Date Pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62313</td>\n",
       "      <td>Alexander Boyd Displays Ltd\\nLambeg Mis\\n\\nBal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>62314</td>\n",
       "      <td>fa £ayes-co 7128 Ueq Wo BS\\n\\npave 2B Jia aire...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>62315</td>\n",
       "      <td>Patrick Brown Sole Trader\\nConsultancy service...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   invoice_id                                               text\n",
       "0       62311  SRIPT\\n\\nSth February 2020\\n\\nFantage, postpro...\n",
       "1       62312  PBD\\n\\nINVOICE\\n\\nAlliance HQ, lewelce Date Pa...\n",
       "2       62313  Alexander Boyd Displays Ltd\\nLambeg Mis\\n\\nBal...\n",
       "3       62314  fa £ayes-co 7128 Ueq Wo BS\\n\\npave 2B Jia aire...\n",
       "4       62315  Patrick Brown Sole Trader\\nConsultancy service..."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating a dataframe  \n",
    "\n",
    "invoice_info_df = pd.DataFrame(data = dict_invoices)\n",
    "\n",
    "# clean strings -remove underscore\n",
    "\n",
    "invoice_info_df['invoice_id'] = invoice_info_df['invoice_id'].str.replace('_','')\n",
    "\n",
    "# change invoice_id type from string to numeric - necessary for merging later\n",
    "\n",
    "invoice_info_df['invoice_id'] = invoice_info_df['invoice_id'].apply(pd.to_numeric)\n",
    "\n",
    "invoice_info_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8b586519",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'—_—\\n\\nSALTIRE\\nGRAPHICS\\nPAYMENT INVOICE\\n\\nDESIGN & ARTWORK #& LITHO AND SCREEN PRINTERS\\nSNTHG03\\n\\nATTENTION\\nSCOTTISH NATIONAL PARTY INVOICE No. ia\\n5 JACKSONS’S ENTRY\\nEDINBURGH VAT REGISTRATION\\nEH8 8PdJ\\nDATE& TAXPOINT 19.77.19\\n:\\nZ Fold Concertina, leaflets. |\\n\\n| To cost of deliveries as per addresses on\\n, Supplied spreadsheet. Deliveries to 17\\n| addresses, plus 1 extra address for\\nRenfrewshire South.\\nTotal of 210 boxes x 500 copies = 105,000\\n\\nleaflets. 1,035 25 20%\\n\\nPlease remit to:\\n\\n-SALTIRE GRAPHICS SUB TOTAL 1,035 25 = R07 O05 |\\nook Street Studios, 60 Brook Street,\\n\\nGlasgow G40 2AB | INVOICE\\n-&E.O. Terms 30 days\\n\\n'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invoice_info_df.text[456]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f0dce94d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Design Place Creative Services Limited\\nPlaza 9 KD Tower\\n\\nCotterells\\n\\nHemel Hempstead\\n\\nHerts\\nHP1 1FW\\nDesign Place\\nCreative Services\\nCCHQ Invoice # ||\\n4 Matthew Parker Street Invoice Date 30/11/2019\\nLondon Invoice Total £21 ,600.00\\nSW1H 9HQ\\nQuantity Rate Description Line Total\\n1.00 £18,000.00 Consultancy costs November 2019 £18,000.00\\nSubtotal £18,000.00\\nVAT 20.00% £3,600.00\\n\\nTotal £21,600.00\\n\\nPayment should be made within 30 days of the invoice.\\n\\nThank you for your business!\\nPlaza 9 KD Tower, Cotterells, Hemel Hempstead, Herts, HP1 1FW\\n'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge data and invoice_data on invoice_id variable\n",
    "\n",
    "data_final = data.merge(invoice_info_df, left_on='invoice_id', right_on ='invoice_id')\n",
    "\n",
    "data_final.text[456] # this is ok\n",
    "\n",
    "# data_final.to_csv('data_incl_ExtractedText.csv')  # this doesn't work properly\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}