Unbiased_Testing_Cleaned.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa9ea407",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports:\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9a67ab6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>ID</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>26-Jul-17</td>\n",
       "      <td>1.010040e+11</td>\n",
       "      <td>please call daughter Dianne Thomas 07920 07565...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28-Jul-17</td>\n",
       "      <td>1.010040e+11</td>\n",
       "      <td>please speak to sister - Alyson Powell on abpv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28-Jul-17</td>\n",
       "      <td>1.010040e+11</td>\n",
       "      <td>KELLY HAS RUNG - HER PARENTS ARE DUE TO HAVE W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31-Jul-17</td>\n",
       "      <td>1.010040e+11</td>\n",
       "      <td>Mark Hitchings, Scheme Manager Swn Yr Afon con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28-Jul-17</td>\n",
       "      <td>1.010040e+11</td>\n",
       "      <td>SON BRIAN HASFORD RUN G- HE IS RESIDENT IN AUS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11353</th>\n",
       "      <td>13-Jun-23</td>\n",
       "      <td>1.010000e+11</td>\n",
       "      <td>Enquiring about a downstairs toilet, states he...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11354</th>\n",
       "      <td>13-Jun-23</td>\n",
       "      <td>1.010000e+11</td>\n",
       "      <td>Phoning on behalf on her mother she has no fee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11355</th>\n",
       "      <td>13-Jun-23</td>\n",
       "      <td>1.010000e+11</td>\n",
       "      <td>States on saturday they had to call an ambulan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11356</th>\n",
       "      <td>13-Jun-23</td>\n",
       "      <td>1.010000e+11</td>\n",
       "      <td>Nadia Obaji daughter of Abdel Obaji contacted ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11357</th>\n",
       "      <td>13-Jun-23</td>\n",
       "      <td>1.010000e+11</td>\n",
       "      <td>Louise Hancock contacted in. \\n\\nShe has recen...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11358 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Date            ID  \\\n",
       "0      26-Jul-17  1.010040e+11   \n",
       "1      28-Jul-17  1.010040e+11   \n",
       "2      28-Jul-17  1.010040e+11   \n",
       "3      31-Jul-17  1.010040e+11   \n",
       "4      28-Jul-17  1.010040e+11   \n",
       "...          ...           ...   \n",
       "11353  13-Jun-23  1.010000e+11   \n",
       "11354  13-Jun-23  1.010000e+11   \n",
       "11355  13-Jun-23  1.010000e+11   \n",
       "11356  13-Jun-23  1.010000e+11   \n",
       "11357  13-Jun-23  1.010000e+11   \n",
       "\n",
       "                                                    Text  \n",
       "0      please call daughter Dianne Thomas 07920 07565...  \n",
       "1      please speak to sister - Alyson Powell on abpv...  \n",
       "2      KELLY HAS RUNG - HER PARENTS ARE DUE TO HAVE W...  \n",
       "3      Mark Hitchings, Scheme Manager Swn Yr Afon con...  \n",
       "4      SON BRIAN HASFORD RUN G- HE IS RESIDENT IN AUS...  \n",
       "...                                                  ...  \n",
       "11353  Enquiring about a downstairs toilet, states he...  \n",
       "11354  Phoning on behalf on her mother she has no fee...  \n",
       "11355  States on saturday they had to call an ambulan...  \n",
       "11356  Nadia Obaji daughter of Abdel Obaji contacted ...  \n",
       "11357  Louise Hancock contacted in. \\n\\nShe has recen...  \n",
       "\n",
       "[11358 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the original data:\n",
    "df = pd.read_csv('Combined_SPA_Callback.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8b6df7ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the result data:\n",
    "df_dict = {}\n",
    "\n",
    "for i in range(1,5):\n",
    "    df_name = \"df_urgent{}\".format(i)\n",
    "    df_value = pd.read_csv('Huggingface_Results_Cleaned\\Model_Results_Cleaned_Urgent{}.csv'.format(i))\n",
    "    # Create df of entailments:\n",
    "    df_value = df_value[df_value[\"Result\"]==\"Entailment\"]\n",
    "    df_dict[df_name] = df_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b0d77e99",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_urgent1 402\n",
      "df_urgent2 89\n",
      "df_urgent3 785\n",
      "df_urgent4 2307\n"
     ]
    }
   ],
   "source": [
    "# Find number of entailments in each result to choose top 3:\n",
    "for df_name, data in df_dict.items():\n",
    "    print(df_name,len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "397bdc2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Urgent in 3: 266\n",
      "Urgent in 2: 537\n",
      "Urgent in 1: 1622\n",
      "Urgent in None: 8933\n"
     ]
    }
   ],
   "source": [
    "# Selecting the indexes of the original df:\n",
    "idx0 = set(df.index)\n",
    "\n",
    "# Selecting the indexes of only the Top 3:\n",
    "idx1 = set(df_dict[\"df_urgent1\"].index)\n",
    "idx2 = set(df_dict[\"df_urgent3\"].index)\n",
    "idx3 = set(df_dict[\"df_urgent4\"].index)\n",
    "\n",
    "# Find elements common in all sets:\n",
    "common_3 = list(idx1 & idx2 & idx3)\n",
    "\n",
    "# Find elements common in only any 2 sets:\n",
    "common_2 = list(((idx1 & idx2)|(idx2 & idx3)|(idx1 & idx3)) - set(common_3))\n",
    "\n",
    "# Find elements deemed Urgent in only 1 set:\n",
    "common_1 = list((idx1 | idx2 | idx3) - set(common_2) - set(common_3))\n",
    "\n",
    "# Find elements not in any set:\n",
    "uncommon = list(idx0 - set(common_1) - set(common_2) - set(common_3))\n",
    "\n",
    "print(\"Urgent in 3: {}\\nUrgent in 2: {}\\nUrgent in 1: {}\\nUrgent in None: {}\".format(len(common_3),len(common_2),len(common_1),len(uncommon)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "846aaaa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "# Selecting 25 data points at random from each common list:\n",
    "no_of_examples = 25\n",
    "random_uncommon = random.choices(uncommon, k=no_of_examples)\n",
    "random_common_1 = random.choices(common_1, k=no_of_examples)\n",
    "random_common_2 = random.choices(common_2, k=no_of_examples)\n",
    "random_common_3 = random.choices(common_3, k=no_of_examples)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2da6653b",
   "metadata": {},
   "source": [
    "### Preparing the testcases:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8eac0a05",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Creating sample df and adding Urgent column to test results:\n",
    "\n",
    "df_random_uncommon = pd.DataFrame(df.iloc[random_uncommon][\"Text\"])\n",
    "df_random_uncommon[\"Urgent\"] = 0\n",
    "\n",
    "df_random_common_1 = pd.DataFrame(df.iloc[random_common_1][\"Text\"])\n",
    "df_random_common_1[\"Urgent\"] = 1\n",
    "\n",
    "df_random_common_2 = pd.DataFrame(df.iloc[random_common_2][\"Text\"])\n",
    "df_random_common_2[\"Urgent\"] = 2\n",
    "\n",
    "df_random_common_3 = pd.DataFrame(df.iloc[random_common_3][\"Text\"])\n",
    "df_random_common_3[\"Urgent\"] = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1066bb0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenate the DataFrames:\n",
    "testing_df = pd.concat([df_random_uncommon, df_random_common_1, df_random_common_2, df_random_common_3], ignore_index=False)\n",
    "\n",
    "# Randomly shuffle the rows:\n",
    "randomized_results_df = testing_df.sample(frac=1, random_state=42)\n",
    "\n",
    "# Save the results for future use:\n",
    "randomized_results_df.to_csv(\"urgency_results_cleaned.csv\", index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1e39191c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create test by removing results:\n",
    "randomized_test_df = randomized_results_df.copy()\n",
    "randomized_test_df[\"Urgent\"] = 1\n",
    "randomized_test_df.to_csv(\"urgency_test_cleaned.csv\", index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01605f47",
   "metadata": {},
   "source": [
    "### Getting Results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "42665a8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Urgent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8866</th>\n",
       "      <td>STATED HIS CONSULTANT SENT A LETTER TO SOCIAL ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6245</th>\n",
       "      <td>URGENT !!!!\\n\\nDAUGHTER ON LUNCH HOUR AT THE M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7205</th>\n",
       "      <td>MRS WILLIAMS NEEDS URGENT CALLBACK PLEASE NOT ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7936</th>\n",
       "      <td>Miss Edwards calling about her neighbour Ms Hu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2264</th>\n",
       "      <td>tcall bethan phillips granddaughter concerned ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8558</th>\n",
       "      <td>TEL CALL WIFE CHERYL -MR LOVERIDGE HAS HAD SEV...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6989</th>\n",
       "      <td>T/C MR WILLIAMS - HE IS REGISTERED BLIND AND N...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6083</th>\n",
       "      <td>tcal suzanne preece granddaughter been furloug...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4252</th>\n",
       "      <td>Please call Mrs Bowen she says that her husban...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6544</th>\n",
       "      <td>MRS JOHN RANG - EXPLAINED TO ME THAT SHE WAS F...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Text  Urgent\n",
       "8866  STATED HIS CONSULTANT SENT A LETTER TO SOCIAL ...       1\n",
       "6245  URGENT !!!!\\n\\nDAUGHTER ON LUNCH HOUR AT THE M...       1\n",
       "7205  MRS WILLIAMS NEEDS URGENT CALLBACK PLEASE NOT ...       1\n",
       "7936  Miss Edwards calling about her neighbour Ms Hu...       1\n",
       "2264  tcall bethan phillips granddaughter concerned ...       1\n",
       "...                                                 ...     ...\n",
       "8558  TEL CALL WIFE CHERYL -MR LOVERIDGE HAS HAD SEV...       1\n",
       "6989  T/C MR WILLIAMS - HE IS REGISTERED BLIND AND N...       1\n",
       "6083  tcal suzanne preece granddaughter been furloug...       1\n",
       "4252  Please call Mrs Bowen she says that her husban...       1\n",
       "6544  MRS JOHN RANG - EXPLAINED TO ME THAT SHE WAS F...       1\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the completed test:\n",
    "completed_test_df = pd.read_csv(\"urgency_test_cleaned.csv\",index_col=0)\n",
    "completed_test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a7b2e27b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Urgent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8866</th>\n",
       "      <td>STATED HIS CONSULTANT SENT A LETTER TO SOCIAL ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6245</th>\n",
       "      <td>URGENT !!!!\\n\\nDAUGHTER ON LUNCH HOUR AT THE M...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7205</th>\n",
       "      <td>MRS WILLIAMS NEEDS URGENT CALLBACK PLEASE NOT ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7936</th>\n",
       "      <td>Miss Edwards calling about her neighbour Ms Hu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2264</th>\n",
       "      <td>tcall bethan phillips granddaughter concerned ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8558</th>\n",
       "      <td>TEL CALL WIFE CHERYL -MR LOVERIDGE HAS HAD SEV...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6989</th>\n",
       "      <td>T/C MR WILLIAMS - HE IS REGISTERED BLIND AND N...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6083</th>\n",
       "      <td>tcal suzanne preece granddaughter been furloug...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4252</th>\n",
       "      <td>Please call Mrs Bowen she says that her husban...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6544</th>\n",
       "      <td>MRS JOHN RANG - EXPLAINED TO ME THAT SHE WAS F...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Text  Urgent\n",
       "8866  STATED HIS CONSULTANT SENT A LETTER TO SOCIAL ...       3\n",
       "6245  URGENT !!!!\\n\\nDAUGHTER ON LUNCH HOUR AT THE M...       2\n",
       "7205  MRS WILLIAMS NEEDS URGENT CALLBACK PLEASE NOT ...       2\n",
       "7936  Miss Edwards calling about her neighbour Ms Hu...       1\n",
       "2264  tcall bethan phillips granddaughter concerned ...       1\n",
       "...                                                 ...     ...\n",
       "8558  TEL CALL WIFE CHERYL -MR LOVERIDGE HAS HAD SEV...       2\n",
       "6989  T/C MR WILLIAMS - HE IS REGISTERED BLIND AND N...       2\n",
       "6083  tcal suzanne preece granddaughter been furloug...       0\n",
       "4252  Please call Mrs Bowen she says that her husban...       3\n",
       "6544  MRS JOHN RANG - EXPLAINED TO ME THAT SHE WAS F...       2\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "randomized_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4c32b162",
   "metadata": {},
   "outputs": [],
   "source": [
    "wrong_0 = 0\n",
    "wrong_1 = 0\n",
    "wrong_2 = 0\n",
    "wrong_3 = 0\n",
    "\n",
    "# Comparing results to test:\n",
    "for df1_row, df2_row in zip(completed_test_df.itertuples(), randomized_results_df.itertuples()):\n",
    "    test = df1_row.Urgent\n",
    "    gt = df2_row.Urgent\n",
    "    if test == 0:\n",
    "        if gt == 1:\n",
    "            wrong_1+=1\n",
    "        elif gt == 2:\n",
    "            wrong_2+=1\n",
    "        elif gt == 3:\n",
    "            wrong_3+=1\n",
    "    else:\n",
    "        if gt == 0:\n",
    "            wrong_0+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "71c93639",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 75%\n",
      "Precision: 0.75\n",
      "Recall: 1.0\n",
      "F1_score: 0.8571428571428571\n"
     ]
    }
   ],
   "source": [
    "# Confusion Matrix:\n",
    "FN = wrong_1 + wrong_2 + wrong_3\n",
    "FP = wrong_0\n",
    "TP = 75-FN\n",
    "TN = 25-FP\n",
    "\n",
    "# Calculating Metrics:\n",
    "accuracy = 100-(FP+FN)\n",
    "precision = TP/(TP+FP)\n",
    "recall = TP/(TP+FN)\n",
    "f1_score = (2*precision*recall)/(precision+recall)\n",
    "\n",
    "print(\"Accuracy: {}%\".format(accuracy))\n",
    "print(\"Precision: {}\".format(precision))\n",
    "print(\"Recall: {}\".format(recall))\n",
    "print(\"F1_score: {}\".format(f1_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a7707b5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}