experiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "nltk.download(\"punkt\")\n",
    "nltk.download(\"wordnet\")\n",
    "nltk.download('omw-1.4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier as RFC, RandomForestRegressor as RFR, GradientBoostingClassifier as XGBoost\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings = pd.read_csv(\"Data/clean_ratings.csv\", infer_datetime_format=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_profs = pd.read_csv(\"Data/clean_prof_info.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_profs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_profs.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_profs[\"firstName\"] = df_profs[\"firstName\"].apply(lambda x: x.strip())\n",
    "df_profs[\"lastName\"] = df_profs[\"lastName\"].apply(lambda x: x.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names = df_profs[[\"profID\", \"firstName\", \"lastName\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add professor first and last names to df_ratings\n",
    "df_ratings = df_ratings.merge(df_names, how=\"inner\", on=\"profID\")\n",
    "df_ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No data lost, all profIDs have match in df_profs\n",
    "df_ratings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "stopword_list = set(stopwords.words(\"english\"))\n",
    "stopword_list.update([',', '.'])\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def preprocess_comment(row):\n",
    "    \"\"\"\n",
    "    Tokenize, remove stopwords and punctuations at word ends, lemmatize, and then reassemble into one string.\n",
    "    \n",
    "    If any token matches the first or last name of the professor, it is dropped.\n",
    "    \n",
    "    All numbers are dropped.\n",
    "    \n",
    "    This is done to eliminate low-impact tokens and reduce vocabulary size.\n",
    "    \n",
    "    String type output required for easier ingestion by sklearn TfidfVectorizer.\n",
    "    \"\"\"\n",
    "    comment = row.loc[\"comment\"]\n",
    "    re.sub(r\"['!\\\"#$%&\\'()*,./:;<=>?@[\\\\]^_`{|}~'] \", ' ', comment)\n",
    "    tokens = word_tokenize(comment)\n",
    "    \n",
    "    ignore_list = stopword_list.copy()\n",
    "    ignore_list.update([row.loc[\"firstName\"], row.loc[\"lastName\"]])\n",
    "    \n",
    "    tokens = [token.lower() for token in tokens if token not in ignore_list and not token.isnumeric()]\n",
    "    tokens = [lemmatizer.lemmatize(token) for token in tokens]\n",
    "    \n",
    "    return \" \".join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings[\"comment\"] = df_ratings.apply(preprocess_comment, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings[\"comment\"][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Denote ratings corresponding to scores of 4 or higher as hot\n",
    "# Encode hot as 1, not hot as 0\n",
    "df_ratings[\"Hot\"] = df_ratings[\"helpfulRating\"] >= 4\n",
    "df_ratings[\"Hot\"] = df_ratings[\"Hot\"].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = df_ratings[\"comment\"], df_ratings[\"Hot\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Frequency and TfIdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(strip_accents=\"ascii\")\n",
    "count_vectorizer = CountVectorizer(strip_accents=\"ascii\")\n",
    "\n",
    "# note to future self: please remember to set the min_df parameter to limit the dimensionality.\n",
    "# High-dimensional encoding drastically dlows down fitting and grid searching."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "test_tfidf = tfidf_vectorizer.transform(X_test)\n",
    "train_count = count_vectorizer.fit_transform(X_train)\n",
    "test_count = count_vectorizer.transform(X_test)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the Effect of Specifiying `min_df`\n",
    "Only terms appearing in at least 5 comments are added to the vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_df_vectorizer = TfidfVectorizer(strip_accents=\"ascii\", min_df=5)\n",
    "min_df_tfidf = min_df_vectorizer.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reduction of dimension by a factor of 3\n",
    "print(train_tfidf.shape, min_df_tfidf.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GloVe (Global Vectors for Word Representation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from gensim.models import KeyedVectors\n",
    "\n",
    "glove_path = \"Data/glove.6B.100d.txt\"\n",
    "word2vec_path = glove_path + \".word2vec\"\n",
    "\n",
    "glove2word2vec(glove_path, word2vec_path)\n",
    "word2vec = KeyedVectors.load_word2vec_format(word2vec_path, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Word2VecVectorizer:\n",
    "    \"\"\"Encode entire comments by taking the average of the word representations\"\"\"\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        self.dimension = 100\n",
    "\n",
    "    def transform(self, data):\n",
    "        X = np.zeros((len(data), self.dimension))\n",
    "        n = 0\n",
    "        empty_count = 0\n",
    "        \n",
    "        for sentence in data:\n",
    "            tokens = sentence.split()\n",
    "            vecs = []\n",
    "\n",
    "            for word in tokens:\n",
    "                try:\n",
    "                  # throws KeyError if word not found\n",
    "                  vec = self.word2vec.get_vector(word)\n",
    "                  vecs.append(vec)\n",
    "                except KeyError:\n",
    "                  pass\n",
    "            \n",
    "            if len(vecs) > 0:\n",
    "                vecs = np.array(vecs)\n",
    "                X[n] = vecs.mean(axis=0)\n",
    "            else:\n",
    "                empty_count += 1\n",
    "                \n",
    "            n += 1\n",
    "            \n",
    "        print(f\"Number of samples with no words found: {empty_count}/{len(data)}\")\n",
    "        return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec_vectorizer = Word2VecVectorizer(word2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_word2vec = word2vec_vectorizer.transform(X_train)\n",
    "test_word2vec = word2vec_vectorizer.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def benchmark(clf, search_grid):\n",
    "    \"\"\"\n",
    "    Train one classifier on the three types of encodings.\n",
    "\n",
    "    Use gridsearch to find the best hyperparameters.\n",
    "\n",
    "    Print the classification report on the test set.\n",
    "\n",
    "    Return the best classifiers.\n",
    "    \"\"\"\n",
    "    # Choose a classifier based on best average f1-score on 5-fold CV\n",
    "    clf = GridSearchCV(estimator=clf, param_grid=search_grid, scoring=\"f1\", n_jobs=-1, verbose=3)\n",
    "    \n",
    "    print(\"Frequency:\")\n",
    "    clf.fit(train_count, y_train)\n",
    "    count_clf = clf.best_estimator_\n",
    "    pred_count = count_clf.predict(test_count)\n",
    "    \n",
    "    # 0 for not hot, 1 for hot\n",
    "    print(classification_report(y_test, pred_count, target_names=[\"Not Hot\", \"Hot\"]))\n",
    "    print(\"-\" * 100)\n",
    "    \n",
    "    print(\"TfIdf:\")\n",
    "    clf.fit(train_tfidf, y_train)\n",
    "    tfidf_clf = clf.best_estimator_\n",
    "    pred_tfidf = tfidf_clf.predict(test_tfidf)\n",
    "    \n",
    "    print(classification_report(y_test, pred_tfidf, target_names=[\"Not Hot\", \"Hot\"]))\n",
    "    print(\"-\" * 100)\n",
    "    \n",
    "    print(\"Word2Vec:\")\n",
    "    clf.fit(train_word2vec, y_train)\n",
    "    word2vec_clf = clf.best_estimator_\n",
    "    pred_word2vec = word2vec_clf.predict(test_word2vec)\n",
    "    \n",
    "    print(classification_report(y_test, pred_word2vec, target_names=[\"Not Hot\", \"Hot\"]))\n",
    "    \n",
    "    return count_clf, tfidf_clf, word2vec_clf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest (Classification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RFC(random_state=42, n_jobs=-1)\n",
    "rf_grid = {\"n_estimators\":[50, 100, 200]}\n",
    "rf_count, rf_tfidf, rf_word2vec = benchmark(rf, rf_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgboost = XGBoost(random_state=42)\n",
    "xgb_grid = {\"learning_rate\":[0.02, 0.1, 0.5]}\n",
    "xgb_count, xgb_tfidf, xgb_word2vec = benchmark(xgboost, xgb_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_reg = LogisticRegression(n_jobs=-1, solver=\"sag\")\n",
    "log_reg_grid = {\"C\":[0.5, 1, 2]}\n",
    "log_reg_count, log_reg_tfidf, log_reg_word2vec = benchmark(log_reg, log_reg_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_bayes = MultinomialNB()\n",
    "\n",
    "# no searchable parameters, set the grid with default value\n",
    "# this simplifies to simple 5-fold CV\n",
    "naive_bayes_grid = {\"alpha\":[1.0]}\n",
    "naive_bayes_count, naive_bayes_tdidf, naive_bayes_word2vec = benchmark(naive_bayes, naive_bayes_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm = LinearSVC(random_state=42)\n",
    "svm_grid = {\"C\":[0.5, 1, 2]}\n",
    "svm_count, svm_tfidf, svm_word2vec = benchmark(svm, svm_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Beware of memory issues!\n",
    "knn = KNeighborsClassifier(n_jobs=-1)\n",
    "knn_grid = {\"n_neighbors\":[1, 3, 5]}\n",
    "knn_count, knn_tfidf, knn_word2vec = benchmark(knn, knn_grid)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest (Regression)\n",
    "\n",
    "Instead of predicting \"hot or not\" 0 or 1 boolean encoding.\n",
    "\n",
    "Predict the integer `helpfulRating` scores. Note this column is technically categorical.\n",
    "\n",
    "For simplicity, we treat it as a continuous variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_regression_train = df_ratings.iloc[X_train.index][\"helpfulRating\"]\n",
    "y_regression_test = df_ratings.iloc[X_test.index][\"helpfulRating\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There are a small numbers of NA values in each series (12 and 8 respectively)\n",
    "# Dropping them and reprocessing X is difficult\n",
    "# Fill NA values with series mean instead\n",
    "\n",
    "y_regression_train = y_regression_train.fillna(y_regression_train.mean())\n",
    "y_regression_test = y_regression_test.fillna(y_regression_test.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error, mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_regression(actual, predicted):\n",
    "    \"\"\"\n",
    "    Infer the accuracy, precision, and recall based on regression results.\n",
    "    \n",
    "    Also calculate the MAE and RMSE.\n",
    "    \"\"\"\n",
    "    MAE = mean_absolute_error(actual, predicted)\n",
    "    RMSE = mean_squared_error(actual, predicted, squared=False)\n",
    "    true_hot = 0\n",
    "    true_not_hot = 0\n",
    "    false_hot = 0\n",
    "    false_not_hot = 0\n",
    "    \n",
    "    for m, n in np.nditer([actual, predicted]):\n",
    "        if m >= 4:\n",
    "            if n >= 4:\n",
    "                true_hot += 1\n",
    "            else:\n",
    "                false_not_hot += 1\n",
    "        else:\n",
    "            if n >= 4:\n",
    "                false_hot += 1\n",
    "            else:\n",
    "                true_not_hot += 1\n",
    "        \n",
    "    total = true_hot + true_not_hot + false_hot + false_not_hot\n",
    "    accuracy = (true_hot + true_not_hot) / total\n",
    "    precision = true_hot / (true_hot + false_hot)\n",
    "    recall = true_hot / (true_hot + false_not_hot)\n",
    "    \n",
    "    print(f\"Accuracy: {accuracy}\")\n",
    "    print(f\"Precision: {precision}\", )\n",
    "    print(f\"Recall: {recall}\")\n",
    "    print(f\"F1 score: {2 * recall * precision / (recall + precision)}\")\n",
    "    print(f\"Mean Absolute Error: {MAE}\")\n",
    "    print(f\"Root Mean Squared Error: {RMSE}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_reg = RFR(n_jobs=-1, random_state=42)\n",
    "rf_reg.fit(train_count, y_regression_train)\n",
    "rf_reg_count_pred = rf_reg.predict(test_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Count:\")\n",
    "eval_regression(y_regression_test, rf_reg_count_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_reg.fit(train_tfidf, y_regression_train)\n",
    "rf_reg_tfidf_pred = rf_reg.predict(test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"TfIdf:\")\n",
    "eval_regression(y_regression_test, rf_reg_tfidf_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_reg.fit(train_word2vec, y_regression_train)\n",
    "rf_reg_word2vec_pred = rf_reg.predict(test_word2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Word2Vec:\")\n",
    "eval_regression(y_regression_test, rf_reg_word2vec_pred)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Persistence\n",
    "Save the trained models to disk so we can load them easily in the future."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import dump, load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dump(rf_count, \"Models/rf_count.joblib\")\n",
    "dump(rf_tfidf, \"Models/rf_tfidf.joblib\")\n",
    "dump(rf_word2vec, \"Models/rf_word2vec.joblib\")\n",
    "dump(xgb_count, \"Models/xgb_count.joblib\")\n",
    "dump(xgb_tfidf, \"Models/xgb_tfidf.joblib\")\n",
    "dump(xgb_word2vec, \"Models/xgb_word2vec.joblib\")\n",
    "dump(log_reg_count, \"Models/log_reg_count.joblib\")\n",
    "dump(log_reg_tfidf, \"Models/log_reg_tfidf.joblib\")\n",
    "dump(log_reg_word2vec, \"Models/log_reg_word2vec.joblib\")\n",
    "dump(svm_count, \"Models/svm_count.joblib\")\n",
    "dump(svm_tfidf, \"Models/svm_tfidf.joblib\")\n",
    "dump(svm_word2vec, \"Models/svm_word2vec.joblib\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}