RandomForestBayes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read and Prep Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Volume in drive C is Windows\n",
      " Volume Serial Number is 4333-1BF2\n",
      "\n",
      " Directory of C:\\Users\\Gaston\\Desktop\\Bayesian_OPt\\BayesianOptimization\n",
      "\n",
      "03/20/2020  04:46 PM    <DIR>          .\n",
      "03/20/2020  04:46 PM    <DIR>          ..\n",
      "03/16/2020  03:20 PM    <DIR>          .ipynb_checkpoints\n",
      "03/16/2020  02:54 PM           640,216 Business_case_dataset.csv\n",
      "03/20/2020  04:43 PM            21,744 RandomForestBayes.ipynb\n",
      "03/20/2020  04:46 PM               453 README.md\n",
      "03/20/2020  04:44 PM            37,984 TensorflowBayesian.ipynb\n",
      "               4 File(s)        700,397 bytes\n",
      "               3 Dir(s)  958,555,918,336 bytes free\n"
     ]
    }
   ],
   "source": [
    "# !ls\n",
    "!dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_labels=[\n",
    "    \"ID\",\n",
    "    \"Book_Length(mins)_overall\",\n",
    "    \"Book_Length(mins)_avg\",\n",
    "    \"Price_overall\",\n",
    "    \"Price_avg\",\n",
    "    \"Gave_Review\",\n",
    "    \"Review 10/10\",\n",
    "    \"Minutes_listened\",\n",
    "    \"%_completion\",\n",
    "    \"Support_Requests\",\n",
    "    \"Last_visited_minus_purchase_date\",\n",
    "    \"targets\"\n",
    "  ];\n",
    "raw_data = pd.read_csv(\"Business_case_dataset.csv\",names= actual_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID                                  14083\n",
       "Book_Length(mins)_overall           14083\n",
       "Book_Length(mins)_avg               14083\n",
       "Price_overall                       14083\n",
       "Price_avg                           14083\n",
       "Gave_Review                         14083\n",
       "Review 10/10                        14083\n",
       "Minutes_listened                    14083\n",
       "%_completion                        14083\n",
       "Support_Requests                    14083\n",
       "Last_visited_minus_purchase_date    14083\n",
       "targets                             14083\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Book_Length(mins)_overall</th>\n",
       "      <th>Book_Length(mins)_avg</th>\n",
       "      <th>Price_overall</th>\n",
       "      <th>Price_avg</th>\n",
       "      <th>Gave_Review</th>\n",
       "      <th>Review 10/10</th>\n",
       "      <th>Minutes_listened</th>\n",
       "      <th>%_completion</th>\n",
       "      <th>Support_Requests</th>\n",
       "      <th>Last_visited_minus_purchase_date</th>\n",
       "      <th>targets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>873</td>\n",
       "      <td>2160.0</td>\n",
       "      <td>2160</td>\n",
       "      <td>10.13</td>\n",
       "      <td>10.13</td>\n",
       "      <td>0</td>\n",
       "      <td>8.91</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>611</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>2808</td>\n",
       "      <td>6.66</td>\n",
       "      <td>13.33</td>\n",
       "      <td>1</td>\n",
       "      <td>6.50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>705</td>\n",
       "      <td>324.0</td>\n",
       "      <td>324</td>\n",
       "      <td>10.13</td>\n",
       "      <td>10.13</td>\n",
       "      <td>1</td>\n",
       "      <td>9.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>334</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>391</td>\n",
       "      <td>1620.0</td>\n",
       "      <td>1620</td>\n",
       "      <td>15.31</td>\n",
       "      <td>15.31</td>\n",
       "      <td>0</td>\n",
       "      <td>9.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>183</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>819</td>\n",
       "      <td>432.0</td>\n",
       "      <td>1296</td>\n",
       "      <td>7.11</td>\n",
       "      <td>21.33</td>\n",
       "      <td>1</td>\n",
       "      <td>9.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ID  Book_Length(mins)_overall  Book_Length(mins)_avg  Price_overall  \\\n",
       "0  873                     2160.0                   2160          10.13   \n",
       "1  611                     1404.0                   2808           6.66   \n",
       "2  705                      324.0                    324          10.13   \n",
       "3  391                     1620.0                   1620          15.31   \n",
       "4  819                      432.0                   1296           7.11   \n",
       "\n",
       "   Price_avg  Gave_Review  Review 10/10  Minutes_listened  %_completion  \\\n",
       "0      10.13            0          8.91               0.0           0.0   \n",
       "1      13.33            1          6.50               0.0           0.0   \n",
       "2      10.13            1          9.00               0.0           0.0   \n",
       "3      15.31            0          9.00               0.0           0.0   \n",
       "4      21.33            1          9.00               0.0           0.0   \n",
       "\n",
       "   Support_Requests  Last_visited_minus_purchase_date  targets  \n",
       "0                 0                                 0        1  \n",
       "1                 0                               182        1  \n",
       "2                 1                               334        1  \n",
       "3                 0                               183        1  \n",
       "4                 0                                 0        1  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID                                  14083\n",
       "Book_Length(mins)_overall           14083\n",
       "Book_Length(mins)_avg               14083\n",
       "Price_overall                       14083\n",
       "Price_avg                           14083\n",
       "Gave_Review                         14083\n",
       "Review 10/10                        14083\n",
       "Minutes_listened                    14083\n",
       "%_completion                        14083\n",
       "Support_Requests                    14083\n",
       "Last_visited_minus_purchase_date    14083\n",
       "targets                             14083\n",
       "dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = raw_data.dropna()\n",
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>targets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   targets\n",
       "0        1\n",
       "1        1\n",
       "2        1\n",
       "3        1\n",
       "4        1"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y= df.iloc[:,-1:].copy()\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Book_Length(mins)_overall</th>\n",
       "      <th>Book_Length(mins)_avg</th>\n",
       "      <th>Price_overall</th>\n",
       "      <th>Price_avg</th>\n",
       "      <th>Gave_Review</th>\n",
       "      <th>Review 10/10</th>\n",
       "      <th>Minutes_listened</th>\n",
       "      <th>%_completion</th>\n",
       "      <th>Support_Requests</th>\n",
       "      <th>Last_visited_minus_purchase_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1404.0</td>\n",
       "      <td>2808</td>\n",
       "      <td>6.66</td>\n",
       "      <td>13.33</td>\n",
       "      <td>1</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>324.0</td>\n",
       "      <td>324</td>\n",
       "      <td>10.13</td>\n",
       "      <td>10.13</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1620.0</td>\n",
       "      <td>1620</td>\n",
       "      <td>15.31</td>\n",
       "      <td>15.31</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>432.0</td>\n",
       "      <td>1296</td>\n",
       "      <td>7.11</td>\n",
       "      <td>21.33</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2160.0</td>\n",
       "      <td>2160</td>\n",
       "      <td>10.13</td>\n",
       "      <td>10.13</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Book_Length(mins)_overall  Book_Length(mins)_avg  Price_overall  Price_avg  \\\n",
       "0                     1404.0                   2808           6.66      13.33   \n",
       "1                      324.0                    324          10.13      10.13   \n",
       "2                     1620.0                   1620          15.31      15.31   \n",
       "3                      432.0                   1296           7.11      21.33   \n",
       "4                     2160.0                   2160          10.13      10.13   \n",
       "\n",
       "   Gave_Review  Review 10/10  Minutes_listened  %_completion  \\\n",
       "0            1           6.5               0.0           0.0   \n",
       "1            1           9.0               0.0           0.0   \n",
       "2            0           9.0               0.0           0.0   \n",
       "3            1           9.0               0.0           0.0   \n",
       "4            1           9.0               0.0           0.0   \n",
       "\n",
       "   Support_Requests  Last_visited_minus_purchase_date  \n",
       "0                 0                               182  \n",
       "1                 1                               334  \n",
       "2                 0                               183  \n",
       "3                 0                                 0  \n",
       "4                 0                                 5  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df.iloc[:,1:-1].copy()\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Book_Length(mins)_overall           14083\n",
       "Book_Length(mins)_avg               14083\n",
       "Price_overall                       14083\n",
       "Price_avg                           14083\n",
       "Gave_Review                         14083\n",
       "Review 10/10                        14083\n",
       "Minutes_listened                    14083\n",
       "%_completion                        14083\n",
       "Support_Requests                    14083\n",
       "Last_visited_minus_purchase_date    14083\n",
       "dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_act= pd.get_dummies(X)\n",
    "X_act.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14083,)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "y_act= np.array(y)\n",
    "y_act= y_act.reshape((-1,))\n",
    "y_act.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split data into Train, Val, Test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "#Perform the split\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_act, y_act, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from skopt import BayesSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = BayesSearchCV(\n",
    "    RandomForestClassifier(n_jobs=-1),\n",
    "    {\n",
    "        'n_estimators': (1, 200), # int range\n",
    "        'max_depth' : [None,1,2,3,4,5,6,7], # categorical\n",
    "        'min_samples_split' : (2,5),\n",
    "        'min_samples_leaf' : (1,10),\n",
    "        \n",
    "\n",
    "    },\n",
    "    n_iter=32,\n",
    "    cv=5\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BayesSearchCV(cv=5, error_score='raise',\n",
       "              estimator=RandomForestClassifier(bootstrap=True,\n",
       "                                               class_weight=None,\n",
       "                                               criterion='gini', max_depth=None,\n",
       "                                               max_features='auto',\n",
       "                                               max_leaf_nodes=None,\n",
       "                                               min_impurity_decrease=0.0,\n",
       "                                               min_impurity_split=None,\n",
       "                                               min_samples_leaf=1,\n",
       "                                               min_samples_split=2,\n",
       "                                               min_weight_fraction_leaf=0.0,\n",
       "                                               n_estimators='warn', n_jobs=-1,\n",
       "                                               oob_score=False,\n",
       "                                               random_state=None, verbose=0,\n",
       "                                               warm_start=False),\n",
       "              fit_params=None, iid=True, n_iter=32, n_jobs=1, n_points=1,\n",
       "              optimizer_kwargs=None, pre_dispatch='2*n_jobs', random_state=None,\n",
       "              refit=True, return_train_score=False, scoring=None,\n",
       "              search_spaces={'max_depth': [None, 1, 2, 3, 4, 5, 6, 7],\n",
       "                             'min_samples_leaf': (1, 10),\n",
       "                             'min_samples_split': (2, 5),\n",
       "                             'n_estimators': (1, 200)},\n",
       "              verbose=0)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore', message='The objective has been evaluated at this point before.')\n",
    "opt.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "val. score: 0.9100834368897568\n",
      "test score: 0.9222577209797657\n"
     ]
    }
   ],
   "source": [
    "print(\"val. score: %s\" % opt.best_score_)\n",
    "print(\"test score: %s\" % opt.score(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}