Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "\n",
    "mnist = fetch_openml(\"mnist_784\", version=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sklearn.utils.Bunch"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(mnist)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'frame', 'categories', 'feature_names', 'target_names', 'DESCR', 'details', 'url'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = mnist[\"data\"], mnist[\"target\"]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f9200fbaf98>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOUElEQVR4nO3dX4xUdZrG8ecFwT8MKiyt2zJEZtGYIRqBlLAJG0Qni38SBS5mAzGIxogXIDMJxEW5gAsvjO7MZBQzplEDbEYmhJEIiRkHCcYQE0OhTAuLLGpapkeEIkTH0QsU373ow6bFrl81VafqlP1+P0mnquup0+dNhYdTXae6fubuAjD0DSt6AACtQdmBICg7EARlB4Kg7EAQF7RyZ+PGjfOJEye2cpdAKD09PTp58qQNlDVUdjO7XdJvJQ2X9Ly7P5G6/8SJE1UulxvZJYCEUqlUNav7abyZDZf0rKQ7JE2WtNDMJtf78wA0VyO/s0+X9IG7f+TupyX9QdLcfMYCkLdGyj5e0l/7fd+b3fYdZrbEzMpmVq5UKg3sDkAjGin7QC8CfO+9t+7e5e4ldy91dHQ0sDsAjWik7L2SJvT7/seSPmlsHADN0kjZ90q61sx+YmYjJS2QtD2fsQDkre5Tb+7+jZktk/Sa+k69vejuB3ObDECuGjrP7u6vSno1p1kANBFvlwWCoOxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiCIhlZxRfs7c+ZMMv/888+buv9169ZVzb766qvktocPH07mzz77bDJfuXJl1Wzz5s3JbS+66KJkvmrVqmS+Zs2aZF6EhspuZj2SvpB0RtI37l7KYygA+cvjyH6Lu5/M4ecAaCJ+ZweCaLTsLunPZrbPzJYMdAczW2JmZTMrVyqVBncHoF6Nln2mu0+TdIekpWY269w7uHuXu5fcvdTR0dHg7gDUq6Gyu/sn2eUJSdskTc9jKAD5q7vsZjbKzEafvS5pjqQDeQ0GIF+NvBp/paRtZnb257zk7n/KZaoh5ujRo8n89OnTyfytt95K5nv27KmaffbZZ8ltt27dmsyLNGHChGT+8MMPJ/Nt27ZVzUaPHp3c9sYbb0zmN998czJvR3WX3d0/kpR+RAC0DU69AUFQdiAIyg4EQdmBICg7EAR/4pqDd999N5nfeuutybzZf2baroYPH57MH3/88WQ+atSoZH7PPfdUza666qrktmPGjEnm1113XTJvRxzZgSAoOxAEZQeCoOxAEJQdCIKyA0FQdiAIzrPn4Oqrr07m48aNS+btfJ59xowZybzW+ejdu3dXzUaOHJncdtGiRckc54cjOxAEZQeCoOxAEJQdCIKyA0FQdiAIyg4EwXn2HIwdOzaZP/XUU8l8x44dyXzq1KnJfPny5ck8ZcqUKcn89ddfT+a1/qb8wIHqSwk8/fTTyW2RL47sQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxAE59lbYN68ecm81ufK11peuLu7u2r2/PPPJ7dduXJlMq91Hr2W66+/vmrW1dXV0M/G+al5ZDezF83shJkd6HfbWDPbaWZHssv0JxgAKNxgnsZvkHT7ObetkrTL3a+VtCv7HkAbq1l2d39T0qlzbp4raWN2faOk9PNUAIWr9wW6K939mCRll1dUu6OZLTGzspmVK5VKnbsD0Kimvxrv7l3uXnL3UkdHR7N3B6CKest+3Mw6JSm7PJHfSACaod6yb5e0OLu+WNIr+YwDoFlqnmc3s82SZksaZ2a9ktZIekLSFjN7QNJRST9v5pBD3aWXXtrQ9pdddlnd29Y6D79gwYJkPmwY78v6oahZdndfWCX6Wc6zAGgi/lsGgqDsQBCUHQiCsgNBUHYgCP7EdQhYu3Zt1Wzfvn3Jbd94441kXuujpOfMmZPM0T44sgNBUHYgCMoOBEHZgSAoOxAEZQeCoOxAEJxnHwJSH/e8fv365LbTpk1L5g8++GAyv+WWW5J5qVSqmi1dujS5rZklc5wfjuxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EATn2Ye4SZMmJfMNGzYk8/vvvz+Zb9q0qe78yy+/TG577733JvPOzs5kju/iyA4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQXCePbj58+cn82uuuSaZr1ixIpmnPnf+0UcfTW778ccfJ/PVq1cn8/HjxyfzaGoe2c3sRTM7YWYH+t221sz+Zmb7s687mzsmgEYN5mn8Bkm3D3D7b9x9Svb1ar5jAchbzbK7+5uSTrVgFgBN1MgLdMvMrDt7mj+m2p3MbImZlc2sXKlUGtgdgEbUW/bfSZokaYqkY5J+Ve2O7t7l7iV3L3V0dNS5OwCNqqvs7n7c3c+4+7eS1kuanu9YAPJWV9nNrP/fFs6XdKDafQG0h5rn2c1ss6TZksaZWa+kNZJmm9kUSS6pR9JDTZwRBbrhhhuS+ZYtW5L5jh07qmb33XdfctvnnnsumR85ciSZ79y5M5lHU7Ps7r5wgJtfaMIsAJqIt8sCQVB2IAjKDgRB2YEgKDsQhLl7y3ZWKpW8XC63bH9obxdeeGEy//rrr5P5iBEjkvlrr71WNZs9e3Zy2x+qUqmkcrk84FrXHNmBICg7EARlB4Kg7EAQlB0IgrIDQVB2IAg+ShpJ3d3dyXzr1q3JfO/evVWzWufRa5k8eXIynzVrVkM/f6jhyA4EQdmBICg7EARlB4Kg7EAQlB0IgrIDQXCefYg7fPhwMn/mmWeS+csvv5zMP/300/OeabAuuCD9z7OzszOZDxvGsaw/Hg0gCMoOBEHZgSAoOxAEZQeCoOxAEJQdCILz7D8Atc5lv/TSS1WzdevWJbft6empZ6Rc3HTTTcl89erVyfzuu+/Oc5whr+aR3cwmmNluMztkZgfN7BfZ7WPNbKeZHckuxzR/XAD1GszT+G8krXD3n0r6V0lLzWyypFWSdrn7tZJ2Zd8DaFM1y+7ux9z9nez6F5IOSRovaa6kjdndNkqa16whATTuvF6gM7OJkqZKelvSle5+TOr7D0HSFVW2WWJmZTMrVyqVxqYFULdBl93MfiTpj5J+6e5/H+x27t7l7iV3L3V0dNQzI4AcDKrsZjZCfUX/vbuf/TOo42bWmeWdkk40Z0QAeah56s3MTNILkg65+6/7RdslLZb0RHb5SlMmHAKOHz+ezA8ePJjMly1blszff//9854pLzNmzEjmjzzySNVs7ty5yW35E9V8DeY8+0xJiyS9Z2b7s9seU1/Jt5jZA5KOSvp5c0YEkIeaZXf3PZIGXNxd0s/yHQdAs/A8CQiCsgNBUHYgCMoOBEHZgSD4E9dBOnXqVNXsoYceSm67f//+ZP7hhx/WNVMeZs6cmcxXrFiRzG+77bZkfvHFF5/3TGgOjuxAEJQdCIKyA0FQdiAIyg4EQdmBICg7EESY8+xvv/12Mn/yySeT+d69e6tmvb29dc2Ul0suuaRqtnz58uS2tT6uedSoUXXNhPbDkR0IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgghznn3btm0N5Y2YPHlyMr/rrruS+fDhw5P5ypUrq2aXX355clvEwZEdCIKyA0FQdiAIyg4EQdmBICg7EARlB4Iwd0/fwWyCpE2S/lnSt5K63P23ZrZW0oOSKtldH3P3V1M/q1QqeblcbnhoAAMrlUoql8sDrro8mDfVfCNphbu/Y2ajJe0zs51Z9ht3/6+8BgXQPINZn/2YpGPZ9S/M7JCk8c0eDEC+zut3djObKGmqpLOf8bTMzLrN7EUzG1NlmyVmVjazcqVSGeguAFpg0GU3sx9J+qOkX7r73yX9TtIkSVPUd+T/1UDbuXuXu5fcvdTR0ZHDyADqMaiym9kI9RX99+7+siS5+3F3P+Pu30paL2l688YE0KiaZTczk/SCpEPu/ut+t3f2u9t8SQfyHw9AXgbzavxMSYskvWdmZ9cefkzSQjObIskl9UhKr1sMoFCDeTV+j6SBztslz6kDaC+8gw4IgrIDQVB2IAjKDgRB2YEgKDsQBGUHgqDsQBCUHQiCsgNBUHYgCMoOBEHZgSAoOxBEzY+SznVnZhVJH/e7aZykky0b4Py062ztOpfEbPXKc7ar3X3Az39radm/t3OzsruXChsgoV1na9e5JGarV6tm42k8EARlB4IouuxdBe8/pV1na9e5JGarV0tmK/R3dgCtU/SRHUCLUHYgiELKbma3m9lhM/vAzFYVMUM1ZtZjZu+Z2X4zK3R96WwNvRNmdqDfbWPNbKeZHckuB1xjr6DZ1prZ37LHbr+Z3VnQbBPMbLeZHTKzg2b2i+z2Qh+7xFwtedxa/ju7mQ2X9L+S/l1Sr6S9kha6+/+0dJAqzKxHUsndC38DhpnNkvQPSZvc/frsticlnXL3J7L/KMe4+3+2yWxrJf2j6GW8s9WKOvsvMy5pnqT7VOBjl5jrP9SCx62II/t0SR+4+0fuflrSHyTNLWCOtufub0o6dc7NcyVtzK5vVN8/lparMltbcPdj7v5Odv0LSWeXGS/0sUvM1RJFlH28pL/2+75X7bXeu0v6s5ntM7MlRQ8zgCvd/ZjU949H0hUFz3Oumst4t9I5y4y3zWNXz/LnjSqi7AMtJdVO5/9muvs0SXdIWpo9XcXgDGoZ71YZYJnxtlDv8ueNKqLsvZIm9Pv+x5I+KWCOAbn7J9nlCUnb1H5LUR8/u4Judnmi4Hn+Xzst4z3QMuNqg8euyOXPiyj7XknXmtlPzGykpAWSthcwx/eY2ajshROZ2ShJc9R+S1Fvl7Q4u75Y0isFzvId7bKMd7VlxlXwY1f48ufu3vIvSXeq7xX5DyWtLmKGKnP9i6S/ZF8Hi55N0mb1Pa37Wn3PiB6Q9E+Sdkk6kl2ObaPZ/lvSe5K61VeszoJm+zf1/WrYLWl/9nVn0Y9dYq6WPG68XRYIgnfQAUFQdiAIyg4EQdmBICg7EARlB4Kg7EAQ/weypTV95ccHFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "some_digit = X[0]\n",
    "some_digit_image = some_digit.reshape(28,28)\n",
    "\n",
    "plt.imshow(some_digit_image, cmap=\"binary\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "y = y.astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a binary classifier first\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False, False, ...,  True, False, False])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_5 = (y_train == 5)\n",
    "y_test_5 = (y_test == 5)\n",
    "y_train_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "SGDClassifier?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(random_state=42)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf = SGDClassifier(shuffle=True, random_state=42)\n",
    "sgd_clf.fit(X_train, y_train_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementing Cross validation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aman.jain/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_split.py:297: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.base import clone\n",
    "\n",
    "skfolds = StratifiedKFold(n_splits=3, random_state=42)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skfolds.get_n_splits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.95035\n",
      "0.96035\n",
      "0.9604\n"
     ]
    }
   ],
   "source": [
    "for train_index, test_index in skfolds.split(X_train, y_train_5):\n",
    "    # print(\"train_index: {} and test_index: {}\", len(train_index), len(test_index))\n",
    "    sgd_clf_clone = clone(sgd_clf)\n",
    "    xt, xtest, yt,ytest = X_train[train_index], X_train[test_index], y_train_5[train_index], y_train_5[test_index]\n",
    "    \n",
    "    sgd_clf_clone.fit(xt, yt)\n",
    "    ypredicted = sgd_clf_clone.predict(xtest)\n",
    "    n_correct = sum(ypredicted == ytest)\n",
    "    print(n_correct / len(ypredicted))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.95035, 0.96035, 0.9604 ])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "scores = cross_val_score(sgd_clf, X_train, y_train_5, cv=3)\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53892,   687],\n",
       "       [ 1891,  3530]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False, False, ...,  True, False, False])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_5.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8370879772350012"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "\n",
    "precision_score(y_train_5, y_train_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6511713705958311"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import recall_score\n",
    "\n",
    "recall_score(y_train_5, y_train_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7325171197343846"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "f1_score(y_train_5, y_train_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3, method=\"decision_function\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_pred_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhTVf7H8fdJ05UWWpYCsiMiFLBgWUQBUXQAN0DQB6w6MjDAoIzozCiKIgiiKDMoq+LgMIoOP0VERDZFBBQBiwtLEUVQdmgp3Zc0yfn9kaTWWmyApDc3+b6ep0+b5Db5Xlo+PTn33O9VWmuEEEKYn8XoAoQQQviGBLoQQgQJCXQhhAgSEuhCCBEkJNCFECJIWI164bp16+rmzZsb9fJCCGFKO3fuzNRa16vsMcMCvXnz5qSlpRn18kIIYUpKqZ/P9ZhMuQghRJCQQBdCiCAhgS6EEEFCAl0IIYKEBLoQQgSJKgNdKfWaUuq0UmrPOR5XSqnZSqkDSqldSqkrfV+mEEKIqngzQl8M9Pudx/sDl7k/RgELLr4sIYQQ56vKdeha681Kqea/s8kA4HXt6sO7TSkVr5RqqLU+4aMahRCi2pTYHeQUlpKZb8OpNU6tcTjLfWiN0wmncouJDLdQ6nBS6tDYHZrDWYXERIRhd2psdiffncwlISYCp9aUOpwcycwj4/gRrr+qI9MHdfB57b44sagRcKTc7aPu+34T6EqpUbhG8TRt2tQHLy2ECAQOp8budOJ0gkNrHA5X8DmcrkAstDmwO5w4NWX3Ocs97rm/qNRBQYkdq0W5QtLpCssjWYXUiLS6tnVq7M5fPjucmvQTuTROiMZe7nU92xw5W0hJqZMfTufToGaUO5DL1VcupG0Op9/+jWynfiRz9Us4C7PJUYsCNtBVJfdVetUMrfVCYCFA586d5coaIuRorSl1uILrbIGtbHRX6nCSVWADKAsjT0g6nK6RX2ykFZvDyamcYnKKSomLCsehNdodRg53SDqd7oB0P/bN4WwuTYxFa8qC1Oks97V21eUJtZyiUo5lF9E4Ibrsexzu53RWeA1PWJvFydzi89q+ae0YakZbCbNYCFMQZlFYlMIa5vr885lCOjWNx2qxEB7muv9EdjHtGtUi0moh0mohIzuftPcWsuKNl6kVX5uxTz/PnXf09sv++SLQjwJNyt1uDBz3wfMKcUG0dgVioc2Bze78VTB6gvKsOzxtDielDifHs4uJsLrePtvsTg5lFhAVHobTqSm2O/j6cDZNa8dgd2rsDqf7sybt5ywuS4z75W251mj3aNMz+nQ4NafzSsqe34iLhB3POb8gAzh6tui8vyfSaiHMon75UK7PSsGp3BJa14/Folxh6ApHsLi3syiFxQKnc0uoGxdJvdhIrGGqLCwPZxXSoVEtrGGe57UQZqHs89nCUprXifnV/VZ3ADu1pmGtKGpFh1Mj0lr2+mHu1/QE9S/3VTZOvTD9+vVj3bp1DB8+nH/+858kJCT47Lkr8kWgrwQeUEotBboBOTJ/LiryjExL7A4KbQ6KbA5XKDqd2B2uoM0utJWN+A6czicmIoxCm4OTOcXkFdspsTvJKy7lu5N5NIqPptju4GBGAXFR1rKw9ryt94e9x3MrvT/9ROX3V2Szu97Oh1kUEWEWEmLCiYm0Eh7mCqzwMAs/nMqjS/PahFlcoz1PKIVZFEfPFtKxSQKRVgvFpQ5qRodTMzrcHYa4A9H1tStEXeGklGteuG5sJBb37fKBqjxh6nkOdwDGRVnLgs4Tthb1S/gqC2Vfu0ansgraIy8vj/DwcKKiopgwYQJ/+9vfuPHGG/3+ulUGulLqf0BvoK5S6ijwFBAOoLV+GVgN3AQcAAqB4f4qVgQGu8PJT2cKOJ5dTG5xKRv2nSYzvwSrRZUdDLI7NTt/PkvT2jEczir0eQ05RaVlX+cV23/zeHiYaw4WoH7NSKwW18jRE442h5PcolKSLqnpDlQLh88UcmUzV2BGWC1k5pXQukEcMRFhhIdZKCl10DA+GqvFFV5W9/MppX4VfmUjT/XLSNWiFDERYUSFhxHmw9GfCDzr1q1j1KhR3H333TzzzDP07t272l7bm1Uuw6p4XAP3+6wiYQinU3Mqr5hvDmez93gu+SV2Pko/Rd3YCPadyMOpNRFWCyXuKQxvVQzzGhFhhFstZBeW0rJeDXfAukaoYRbXnGS3FrUJD7NQUGKnbcOaxESGkRgXRWyklUirBYtFERtppVa0lUhrWFlQet6ee0adQlSnrKwsHn74Yf773//Spk0bbr755mqvwbD2ucK/HE7XgbYSu5NCm52fMgvZezwHm8PJkawivjmSjUW5phFqRYf/asRb3rHsX+ZR7e6DX0qB1pAQE85liXHUrxWF1po+bROpU8M17xnuHsHGRFipXSOCmu7wFSIYbdiwgdTUVM6cOcPEiRN54okniIqKqvY6JNBNyOnUZOaXcCizgNN5JaT9lEV2USln8m18dzKPzPyS83q+8mFeNzaSmlFW2jeqxZVN46kZHU77RrWoEWklPjqcCKulbJpBCOGSmJhIixYtWLt2LR07djSsDgn0AOdwarYfOsMn+07z2YFMvjuZ5/X3RlgtRIZZCLdayCqw0aBmFM3rxnDd5Yk0jI8mIsxC09oxxMeEUyc2QkbQQnhJa81///tfvvrqK2bPnk2HDh3YunWr4QMdCfQA4XRqsgpt7D6Ww9eHszmVU8zPWQXsPppDwTnW+TasFUWT2jG0SoylQc0o2jSII7FmFC3q1KBmtNXwXy4hgtGhQ4cYPXo0H330ET179qSoqIjo6OiA+P8mgW6Q/BI7n3x3mg++PU5mfglfH84+57ZxUVZaJcbSp00irevH0alpAvXiIquxWiGEw+Fg3rx5PPbYY1gsFubPn8/o0aOxWAJnuaYEejXJyCvhjW0/8/aXR373bLWEGNfa4m4tanN9m/q0rFeDVvVifXqigxDi/GVmZjJp0iSuvfZaXn755YBsXyKB7gfZhTbe/+Y4BzPyWbv3JKdyz32Q8sqm8aR2a0b3S+vQsFZUQLxtE0K4lJaW8uabb3LvvfdSv359vvrqK1q0aBGw/08l0H3kZE4x8z89wKpdJ8p6clR0WWIsKc0SGNSpEVc2SyBczqwTImDt3LmTP/3pT+zatYuGDRvSt29fWrZsaXRZv0sC/QJprfn2aA5zP/mBTd9nlJ2V6NEoPpr+7RvQoXEt6teMomOTeKLCZRWJEIGuqKiIKVOmMHPmTBITE3nvvffo27ev0WV5RQL9PNnsTp75MJ3/fvHzbx67tF4NhnZpyh2dGxMfE2FAdUKIizVw4EDWr1/PyJEjeeGFF4iPjze6JK8pbUTrN1ztc9PS0gx57QuRmV/C4s9/Yu7GA7+6PzEukumDOtCzdV1Zxy2ESeXm5hIREUFUVBSbNm3CbrfTp08fo8uqlFJqp9a6c2WPyQi9Ch+ln+LVzQfZ8VPWr+7/R9/LGd2rpXSYE8LkVq9ezZgxY7j77ruZPn061157rdElXTAJ9HM4klXIX5d+/av14U1rxzAkpTHjrm8VsEe5hRDeyczM5KGHHmLJkiUkJSVx2223GV3SRZNAr8TWHzO569XtZbfHXd+KOzs3oUntGAOrEkL4ykcffURqaipnz55l0qRJPP7440RGmv9kPQn0CpbuOMzEFXsA1zLDybe145pWdQ2uSgjhSw0bNqR169YsWLCADh18f21Po0igu53KLWbCu7vYuD8DgD5tEpl/95VyoFOIIKC1ZtGiRXz99dfMmzeP9u3bs2XLlqCbOpVAB9744iemrtpXdsXvf/S9nLG9Lw26H7YQoejgwYP8+c9/5pNPPqF3794B1UzL10I60LXWPP7ebv634wgASQ1r8vCNrbkhqb7BlQkhLpbD4WD27NlMnDgRq9XKK6+8wsiRIwOqmZavhWygO52ae17bzucHzgAwtvelPNKvjcFVCSF8JTMzkylTptCnTx8WLFhA48aNjS7J70I20Geu318W5s8Mak9qt2YGVySEuFg2m40lS5Zw3333Ub9+fb755huaNWsWlNMrlQne9x6/47XPDjH/0x8BePKWJAlzIYLAl19+SUpKCiNGjODjjz8GoHnz5iET5hCCgb75+wyeXpUOwLCuTRjRo4XBFQkhLkZhYSF///vfueqqqzh79iwrV67kD3/4g9FlGSKkplyOZRcxZslOwLUscfqg4Fl/KkSoGjBgAB9//DGjRo3i+eefp1atWkaXZJiQGqFPW5VOoc3BZYmxLLg7JaTeigkRTHJycigudl3568knn+STTz7hlVdeCekwhxAK9DW7T7Bmz0nCLIp/3dmRCGvI7LoQQWXVqlW0a9eOKVOmANCrVy+uu+46g6sKDCGRakU2B1Pd8+Zje19Kh8ah/VdcCDPKyMjgrrvu4tZbb6V27drcfvvtRpcUcEIi0Kd+mM7xnGLqxUXy516BfQkpIcRvrV+/nqSkJJYtW8aUKVNIS0ujS5cuRpcVcIL+oOjx7CKW7jgMwMt3p1AzKtzgioQQ56tRo0a0bduWBQsW0K5dO6PLCVhBP0KfveEHnBquvrQOKc0SjC5HCOEFp9PJwoUL+ctf/gJAu3bt2Lx5s4R5FYI60HOLS1n+9TEAHr+prcHVCCG8ceDAAfr06cPo0aPZv38/RUVFRpdkGkEd6Iu2HMJmd5LcJJ72jeRAqBCBzOFw8M9//pMrrriCr776ildffZUNGzYQHR1tdGmm4VWgK6X6KaX2K6UOKKUmVPJ4LaXUB0qpb5VSe5VSw31f6vkpLnXw5vafAfjTNc2NLUYIUaXMzEymTZvGjTfeSHp6OiNHjpRzRc5TlYGulAoD5gH9gSRgmFIqqcJm9wPpWutkoDfwT6VUhI9rPS9r95wkM99GvbhIbrniEiNLEUKcQ0lJCa+++ipOp7OsmdaKFSto1KiR0aWZkjcj9K7AAa31Qa21DVgKDKiwjQbilOvPaSyQBdh9Wul5emu7a2XLyB4tCLPIX3khAs327dtJSUlh1KhRZc20Qqkzoj94E+iNgCPlbh9131feXKAtcBzYDTyotXZWfCKl1CilVJpSKi0jI+MCS67a4TOFfPlzFhFWC0O7NPXb6wghzl9BQQEPP/ww3bt3Jycnhw8//DBkm2n5mjeBXtmfS13hdl/gG+ASoCMwVylV8zffpPVCrXVnrXXnevXqnXex3npn5xG0hj8k1adWjKw7FyKQDBw4kFmzZjFmzBj27t3LTTfdZHRJQcObQD8KNCl3uzGukXh5w4Hl2uUAcAgw7PI/nx3IBGBAR5mHEyIQZGdnly0/nDRpEps2bWL+/PnUrPmbcZ+4CN4E+pfAZUqpFu4DnUOBlRW2OQz0AVBK1QcuBw76slBvZeaXsOtoDmEWxVUtaxtRghCinJUrV/6qmVbPnj3p1auXwVUFpyoDXWttBx4A1gH7gLe11nuVUmOUUmPcm00FrlZK7QY2AI9qrTP9VfTv2XbwDA6npmvz2sTJaf5CGOb06dMMHTqUAQMGULduXYYMGWJ0SUHPq14uWuvVwOoK971c7uvjQEAc1fjku9MA9Gxd1+BKhAhda9euJTU1lfz8fKZOncqjjz5KeLgMsPwt6JpzbXVf+PmaSyXQhTBKkyZN6NChA/PnzycpqeJpK8JfgurU/9O5xZzMLSYq3CKn+gtRjZxOJwsWLGD06NGAq5nWp59+KmFezYIq0Hf+fBaAlGYJcjKRENXk+++/p3fv3owdO5ZDhw6VXRpOVL+gCvT5n/4IQMcm8QZXIkTws9vtzJgxgyuuuILdu3fzn//8h3Xr1hEVFWV0aSEraObQnU5NRl4JAMmNJdCF8LczZ84wY8YMbrrpJubNm0fDhg2NLinkBc0IPf1ELidzXW/1el+eaHA1QgSnkpISXnnllbJmWt9++y3Lly+XMA8QQRPoOw5lAXBr8iVEWINmt4QIGF988QWdOnVizJgxfPLJJ4BrNYsIHEGTfBv3u9afX9lUpluE8KX8/HzGjx/PNddcQ0FBAWvXruWGG24wuixRiaCZQ/fMn9evKQdkhPClgQMHsmHDBh544AGmT59OXFyc0SWJcwiKEbrWmu9O5gHQublcCFqIi3X27NmyZlqTJ09my5YtzJkzR8I8wAVFoB896/rFiwq3kBgnI3QhLsby5ctJSkpi8uTJAPTo0YMePXoYW5TwSlAE+n736Dw2UnpFCHGhTp48yZAhQxg8eDANGjRg6NChRpckzlNwBPopV6DfmixLp4S4EGvWrCEpKYlVq1Yxffp0duzYQadOnYwuS5ynoDgoeiizAICW9WINrkQIc2rWrBmdOnVi3rx5tGlj2LVpxEUKihH64axCAFrUqWFwJUKYg9PpZO7cufz5z38GICkpiQ0bNkiYm1xQBPqZfNeSxXpxkQZXIkTg279/P7169WLcuHEcOXJEmmkFEdMHutaakzmuX8j6NSXQhTiX0tJSnn32WZKTk0lPT2fx4sWsWbNGmmkFEdPPoZ8tLKXA5iAu0kp8TITR5QgRsM6ePcsLL7zArbfeypw5c2jQoIHRJQkfM/0I/USOaw16g1oyyhCiouLiYubPn4/T6SQxMZFdu3bxzjvvSJgHKdMHenZhKQAJNWR0LkR5n332GcnJydx///1lzbQaN25scFXCn0wf6J4VLo3iow2uRIjAkJeXxwMPPEDPnj2x2WysX79emmmFCNPPoXvWoLdKlDXoQoCrmdbGjRt58MEHmTZtGrGx8n8jVJg+0E+7L2pRL1ZWuIjQlZWVRVRUFDExMUydOhWlFN27dze6LFHNTD/l8vmPZwBoGC8HRUVoWrZsGW3bti1rpnX11VdLmIco0wd6zSjXm4z4aDkoKkLLiRMnuP3227njjjto0qQJqampRpckDGb6QD/rXuUiyxZFKPnwww9JSkpizZo1zJgxg23btpGcnGx0WcJgpp5Ddzg12YU2AOJjpHWuCB0tW7akS5cuzJ07l9atWxtdjggQph6hnykowamhdo0IwsNMvStC/C6Hw8FLL73EiBEjAGjbti3r16+XMBe/YuoU9FxHVFa4iGCWnp5Oz549GT9+PCdPnpRmWuKcTB3omfmu6ZY6sXJAVAQfm83GtGnT6NSpE99//z1Llixh1apV0kxLnJNXga6U6qeU2q+UOqCUmnCObXorpb5RSu1VSm3ybZmVO1vgCXQZoYvgk52dzaxZsxg0aBDp6emkpqailDK6LBHAqjwoqpQKA+YBNwJHgS+VUiu11unltokH5gP9tNaHlVKJ/iq4vFPuk4oS5ICoCBJFRUUsWrSIsWPHkpiYyO7du7nkkkuMLkuYhDcj9K7AAa31Qa21DVgKDKiwzV3Acq31YQCt9Wnfllm5n864+rhEhYdVx8sJ4VebN28mOTmZcePGsXHjRgAJc3FevAn0RsCRcrePuu8rrzWQoJT6VCm1Uyl1b2VPpJQapZRKU0qlZWRkXFjF5URaXeXLm1BhZrm5uYwdO5Zrr70Wu93Oxx9/TJ8+fYwuS5iQN+vQK8tLXcnzpAB9gGjgC6XUNq3197/6Jq0XAgsBOnfuXPE5zltukeukopb15FqiwrwGDhzIp59+ykMPPcTUqVOpUUN+n8WF8SbQjwJNyt1uDByvZJtMrXUBUKCU2gwkA9/jR3kldgBqyWn/wmQyMzOJiYkhJiaGZ555BqUUV111ldFlCZPzZsrlS+AypVQLpVQEMBRYWWGb94GeSimrUioG6Abs822pv5XlXuVSK1oOigpz0FqzdOlS2rZty1NPPQVA9+7dJcyFT1QZ6FprO/AAsA5XSL+ttd6rlBqjlBrj3mYfsBbYBewA/q213uO/sl08p/3LOnRhBseOHWPgwIEMGzaMFi1acO+9lR5qEuKCedXLRWu9Glhd4b6XK9x+AXjBd6VVLcc9hy4jdBHoVq1aRWpqKqWlpcycOZPx48cTFiars4Rvmbo5V0GJA4DYSFPvhggBrVq14uqrr2bOnDm0atXK6HJEkDLtqf82u5OiUgdhFkW0rEMXAcbhcDBr1izuu+8+ANq0acOaNWskzIVfmTbQzxS4GnPFRVmxWGQluggce/fu5ZprruHhhx8mMzNTmmmJamPaQPfMn2e7L3AhhNFsNhtPP/00nTp14scff+Stt97igw8+kGZaotqYN9DdQZ7SLMHgSoRwyc7OZvbs2dxxxx2kp6czbNgwaaYlqpVpAz232HNSkaxwEcYpLCzkpZdewuFwlDXTevPNN6lXr57RpYkQZN5Ad0+5eC4SLUR127hxIx06dGD8+PF8+umnADRs2NDYokRIM22gn3WfVBQrgS6qWU5ODqNHj+b6669HKcXGjRulmZYICKZNw2PZRQCEyRylqGYDBw5k8+bN/OMf/2Dy5MnExMQYXZIQgIkD3XMykd150U0bhahSRkYGNWrUICYmhmeffZawsDC6dOlidFlC/Ippp1z2ncgFoHkdaTUq/EdrzVtvvfWrZlpXXXWVhLkISKYN9Do1XNcRzSuWdejCP44ePcptt91GamoqrVq1KjvrU4hAZdopl1x3kDevKyN04XsrV67k7rvvLjuFf9y4cdJMSwQ80wb6D6fzAYiwmvZNhghgrVu3pkePHsydO5eWLVsaXY4QXjFtGtZ190CPCDPtLogAYrfbmTlzZlmP8jZt2rB69WoJc2Eqpk3DfPfl5xJrSp8McXF27dpF9+7d+cc//kFubq400xKmZdpAzy2SU//FxSkpKeGpp54iJSWFw4cP8/bbb/Pee+9JMy1hWqYNdM8IXS5uIS5Ubm4u8+fPZ9iwYaSnp3PHHXdIMy1haqYMdK11Wfvc+BgZoQvvFRQUMGvWLBwOB/Xq1WPPnj28/vrr1KlTx+jShLhopgz0ErsTh1MTYbUQLgdFhZc2bNhAhw4dePjhh9m0aRMA9evXN7gqIXzHlGlY4J5uqREh64JF1bKzsxk5ciQ33HADVquVTZs2cf311xtdlhA+Z8oJ6KJS18WhYyJMWb6oZoMGDWLLli08+uijPPXUU0RHRxtdkhB+YcpELHYHemS4Kd9giGpw6tQpYmNjqVGjBs899xxWq5WUlBSjyxLCr0yZiEU2JwDR4TLlIn5Na80bb7xBUlJSWTOtbt26SZiLkGDKQC+xu0boURLoopzDhw9z8803c++993L55ZczYsQIo0sSolqZcsqlxO4aoctp/8Lj/fff5+6770ZrzezZsxk7dqw00xIhx5SBbvMEujTmCnlaa5RStGnTht69ezNnzhyaN29udFlCGMKUiehZ5SJz6KHLbrczY8YM7rnnHgAuv/xyPvjgAwlzEdJMGeilDhmhh7Jvv/2Wbt26MWHCBAoLC6WZlhBupkxEz5SLnCUaWoqLi3niiSfo3Lkzx44dY9myZSxfvlyaaQnhZspELHW4LgwdYZVGSqEkLy+PV155hdTUVNLT0xk8eLDRJQkRULwKdKVUP6XUfqXUAaXUhN/ZrotSyqGUGuK7En/Ls2xRVrkEv/z8fGbOnFnWTCs9PZ3FixdTu3Zto0sTIuBUmYhKqTBgHtAfSAKGKaWSzrHdDGCdr4usqLjUNeUSJb1cgtr69etp3749jzzyCJs3bwagXr16BlclRODyZojbFTigtT6otbYBS4EBlWw3DngXOO3D+irlOSgabpERejDKyspi+PDh9O3bl6ioKLZs2cJ1111ndFlCBDxvErERcKTc7aPu+8oopRoBg4CXf++JlFKjlFJpSqm0jIyM8621jN0hB0WD2aBBg3jjjTd4/PHH+eabb7jmmmuMLkkIU/DmxKLKjjzqCrdfBB7VWjt+74ovWuuFwEKAzp07V3wOr50psAFgDZODosHi5MmTxMXFUaNGDV544QUiIiLo2LGj0WUJYSreDHGPAk3K3W4MHK+wTWdgqVLqJ2AIMF8pNdAnFVbC0w/d0wJAmJfWmsWLF5OUlMSkSZMA6Nq1q4S5EBfAm0D/ErhMKdVCKRUBDAVWlt9Aa91Ca91ca90cWAaM1Vqv8Hm1bjXdF4a2WmSEbmY//fQT/fr1Y/jw4bRr145Ro0YZXZIQplbllIvW2q6UegDX6pUw4DWt9V6l1Bj34787b+4PnnXotWtEVPdLCx957733uOeee1BKMXfuXP7yl79gkYPcQlwUr5pzaa1XA6sr3FdpkGut77v4sn5f2an/clDUdDzNtNq1a8cNN9zASy+9RLNmzYwuS4igYMpE9KxykYOi5lFaWsr06dNJTU0FoHXr1qxYsULCXAgfMmWg22TZoql89dVXdO3alYkTJ+JwOCgpKTG6JCGCkikT0dOcS65YFNiKiop47LHH6Nq1KydPnuS9997j//7v/4iMjDS6NCGCkjkD3X1QNFymXAJaQUEBixYt4o9//CPp6ekMHOi3laxCCEwa6KVyCbqAlZeXx/PPP4/D4aBu3bqkp6ezaNEiEhISjC5NiKBnykQs6+UiF7gIKGvXrqV9+/ZMmDCBLVu2AFC3bl2DqxIidJgyET2XoIuyyhx6IDhz5gx//OMf6d+/PzVq1ODzzz+nd+/eRpclRMgx5UWi7e45dFm2GBhuv/12tm7dypNPPsnEiRPloKcQBjFloJc6PcsWJdCNcuLECeLi4oiNjWXmzJlERESQnJxsdFlChDRTTrmUjdDlVPFqp7Xmtddeo23btmXNtLp06SJhLkQAMGUiHs4qBOSgaHU7ePAgf/jDHxgxYgTJycmMGTPG6JKEEOWYMhHruJtySbfF6rN8+XI6dOjA9u3bWbBgARs3bqR169ZGlyWEKMeUc+gO7TmxyJR/j0zF00yrQ4cO9OvXjxdffJEmTZpU/Y1CiGpnykR0OF2BHiYjdL+x2WxMmzaNu+66C601l112Ge+++66EuRABzNSBLlMu/pGWlkaXLl148sknAVe4CyECnykD3S4jdL8oKirikUceoVu3bmRmZvL+++/zv//9T9aVC2ESpgx0pwS6XxQUFLB48WJGjBjB3r17ue2224wuSQhxHkwX6FrrX0boSgL9YuXm5vLcc8+VNdPat28fCxcuJD4+3ujShBDnyXSB7rmeaJhFYZER+kX58MMPadeuHRMnTixrplWnTh2DqxJCXA7kzGQAAAy+SURBVCjTBbrdKa1zL1ZGRgapqanccsst1KpVi61bt0ozLSGCgOnWodtlhctFGzx4MNu2bWPy5Mk89thjREREGF2SEMIHTBfoDs+UizTmOi/Hjh2jVq1axMbGMmvWLCIjI2nfvr3RZQkhfMh08xYyQj8/WmteffVVkpKSyppppaSkSJgLEYRMGOiuOXRZsli1H3/8kT59+jBq1ChSUlK4//77jS5JCOFHpgv0klJXoEvr3N+3bNkyOnTowM6dO1m4cCEbNmzg0ksvNbosIYQfmW4O3eluzHUsu8jgSgKTp5lWcnIyN998M7NmzaJx48ZGlyWEqAamG+Z6+ri0Sow1uJLAYrPZmDJlCkOHDi1rpvXOO+9ImAsRQkwX6HJQ9Ld27NhBSkoKkydPxmq1SjMtIUKU6QJdWuf+orCwkL///e90796ds2fP8sEHH/Dmm29KMy0hQpRpA11G6K7uiEuWLGHUqFGkp6dzyy23GF2SEMJAXgW6UqqfUmq/UuqAUmpCJY+nKqV2uT+2KqX8dsXgUG+dm5OTwzPPPIPdbqdOnTrs27ePBQsWULNmTaNLE0IYrMpAV0qFAfOA/kASMEwplVRhs0PAtVrrK4CpwEJfF+phd4TussUPPvig7AShzz77DICEhASDqxJCBApvUrErcEBrfVBrbQOWAgPKb6C13qq1Puu+uQ3w29IKmzvQw62hM0LPyMhg2LBh3HbbbdSpU4ft27dLMy0hxG94E+iNgCPlbh9133cuI4A1lT2glBqllEpTSqVlZGR4X2U59rL2uaEzQh88eDDvvvsuTz/9NGlpaXTu3NnokoQQAcibE4sqGwrrSjdU6jpcgd6jsse11gtxT8d07ty50ueoimcOPTzI59CPHj1KfHw8sbGxvPjii0RGRtKuXTujyxJCBDBvhrlHgfKXem8MHK+4kVLqCuDfwACt9RnflPdbZXPoQdpt0el08sorr5CUlFR2keYrr7xSwlwIUSVvAv1L4DKlVAulVAQwFFhZfgOlVFNgOXCP1vp735f5i19OLAq+KZcffviB66+/njFjxtC1a1fGjRtndElCCBOpcspFa21XSj0ArAPCgNe01nuVUmPcj78MTALqAPOV6zqfdq21XyZ6T+UWu74IsgH6O++8w7333ktkZCSLFi1i+PDhKLlmqhDiPHjVnEtrvRpYXeG+l8t9PRIY6dvSKhcf47q6zpGswup4Ob/zNNPq1KkTAwYM4F//+heXXHKJ0WUJIUzIdPMW2t1t8bLEOIMruTglJSVMmjSJO++8E601rVq1YunSpRLmQogLZsJAd3028yKXbdu2ceWVVzJ16lSio6OlmZYQwidMF+iefuhmnF4uKCjgoYce4uqrryYvL4/Vq1fz+uuvSzMtIYRPmC7QPYvXLSZM9OLiYpYuXcrYsWPZu3cv/fv3N7okIUQQMe0Vi8yyAiQ7O5s5c+bw2GOPlTXTio+PN7osIUQQMt0I3b0M3RRTLitWrCApKYkpU6awdetWAAlzIYTfmC7QPUdFA/mg6KlTp7jzzjsZNGgQiYmJbN++nV69ehldlhAiyJlwysX1WQXwmUVDhgxhx44dTJs2jUceeYTw8HCjSxJChADTBboO0BH64cOHSUhIIC4ujtmzZxMZGUlSUsW28UII4T+mm3L5ZQ49MBLd6XQyb9482rVrx6RJkwDo1KmThLkQotqZMNADZx36/v37ufbaa3nggQfo3r07Dz74oNElCSFCmOkC3cPodehvv/02ycnJ7Nmzh//85z+sW7eO5s2bG1qTECK0mS7QnQbPoXvm8FNSUrj99tvZt28f9913X8BMAQkhQpcJA931uboDtLi4mIkTJzJkyBC01lx66aW89dZbNGjQoFrrEEKIczFdoGfmlQDVO4e+detWOnXqxPTp04mLi5NmWkKIgGS6QI+wukrOyC3x+2vl5+fz17/+lR49elBYWMjatWtZvHixNNMSQgQk0wV6dHgYAAk1Ivz+WjabjWXLlnH//fezZ88e+vbt6/fXFEKIC2W6E4s8c+gxEWF+ef6srCxmz57NE088Qe3atdm3bx+1atXyy2sJIYQvmW6ErvFft8V3332XpKQkpk2bVtZMS8JcCGEWpgv0X3q5+M6JEycYPHgwQ4YM4ZJLLiEtLU2aaQkhTMd0Uy6/dFv0XaTfeeedfPnllzz33HP87W9/w2o13z+LEEKYLrl81Q/9559/pnbt2sTFxTFnzhyio6O5/PLLL75AIYQwiOmmXMrm0C/w+51OJ3PmzKFdu3Y8+eSTAHTs2FHCXAhheqYbobtnXLBcwLn/3333HSNHjuTzzz+nX79+PPTQQz6uTgghjGO6EbpnyuV8LV26lOTkZPbt28frr7/O6tWradasmW+LE0IIA5ku0D1TLt4eFHU6nQB06dKFO+64g/T0dO655x5ppiWECDrmC3QvD4oWFRUxYcIEBg8eXNZMa8mSJdSvX9//RQohhAFMGOhVt8/dsmULHTt2ZMaMGdSpU4fS0tJqqk4IIYxjukD/vYtE5+Xlcf/999OrVy9KS0v56KOP+Pe//01EhP/7vgghhNFMF+i/N+VSWlrKihUrGD9+PLt37+aGG26o3uKEEMJA5lu2WKGXy5kzZ3jppZeYNGkStWvX5rvvviMuLs7IEoUQwhBejdCVUv2UUvuVUgeUUhMqeVwppWa7H9+llLrS96W6eEboaM0777xDUlISzz77LF988QWAhLkQImRVGehKqTBgHtAfSAKGKaWSKmzWH7jM/TEKWODjOstorbHnnWH+xL9w55130qRJE9LS0ujZs6e/XlIIIUzBmxF6V+CA1vqg1toGLAUGVNhmAPC6dtkGxCulGvq4VsB1UDTz/Rns3r6J559/nm3btpGcnOyPlxJCCFPxZg69EXCk3O2jQDcvtmkEnCi/kVJqFK4RPE2bNj3fWgGIiQyjxYBxjL6uLeOHXHtBzyGEEMHIm0CvbMV3xRPwvdkGrfVCYCFA586dL+gk/sf6t+Wx/m0v5FuFECKoeTPlchRoUu52Y+D4BWwjhBDCj7wJ9C+By5RSLZRSEcBQYGWFbVYC97pXu1wF5GitT1R8IiGEEP5T5ZSL1tqulHoAWAeEAa9prfcqpca4H38ZWA3cBBwACoHh/itZCCFEZbw6sUhrvRpXaJe/7+VyX2vgft+WJoQQ4nyY7tR/IYQQlZNAF0KIICGBLoQQQUICXQghgoTS+gIv0nmxL6xUBvDzBX57XSDTh+WYgexzaJB9Dg0Xs8/NtNb1KnvAsEC/GEqpNK11Z6PrqE6yz6FB9jk0+GufZcpFCCGChAS6EEIECbMG+kKjCzCA7HNokH0ODX7ZZ1POoQshhPgts47QhRBCVCCBLoQQQSKgAz2QLk5dXbzY51T3vu5SSm1VSpn++ntV7XO57boopRxKqSHVWZ8/eLPPSqneSqlvlFJ7lVKbqrtGX/Pid7uWUuoDpdS37n02dddWpdRrSqnTSqk953jc9/mltQ7ID1yten8EWgIRwLdAUoVtbgLW4Lpi0lXAdqPrroZ9vhpIcH/dPxT2udx2n+Dq+jnE6Lqr4eccD6QDTd23E42uuxr2+XFghvvrekAWEGF07Rexz72AK4E953jc5/kVyCP0gLo4dTWpcp+11lu11mfdN7fhujqUmXnzcwYYB7wLnK7O4vzEm32+C1iutT4MoLU2+357s88aiFNKKSAWV6Dbq7dM39Fab8a1D+fi8/wK5EA/14Wnz3cbMznf/RmB6y+8mVW5z0qpRsAg4GWCgzc/59ZAglLqU6XUTqXUvdVWnX94s89zgba4Ll+5G3hQa+2snvIM4fP88uoCFwbx2cWpTcTr/VFKXYcr0Hv4tSL/82afXwQe1Vo7XIM30/Nmn61ACtAHiAa+UEpt01p/7+/i/MSbfe4LfANcD1wKfKSU2qK1zvV3cQbxeX4FcqCH4sWpvdofpdQVwL+B/lrrM9VUm794s8+dgaXuMK8L3KSUsmutV1RPiT7n7e92pta6AChQSm0GkgGzBro3+zwceE67JpgPKKUOAW2AHdVTYrXzeX4F8pRLKF6cusp9Vko1BZYD95h4tFZelfustW6htW6utW4OLAPGmjjMwbvf7feBnkopq1IqBugG7KvmOn3Jm30+jOsdCUqp+sDlwMFqrbJ6+Ty/AnaErkPw4tRe7vMkoA4w3z1itWsTd6rzcp+Dijf7rLXep5RaC+wCnMC/tdaVLn8zAy9/zlOBxUqp3bimIx7VWpu2ra5S6n9Ab6CuUuoo8BQQDv7LLzn1XwghgkQgT7kIIYQ4DxLoQggRJCTQhRAiSEigCyFEkJBAF0KIICGBLoQQQUICXQghgsT/A7IDx1eyWWIuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0,1], [0,1], 'k--')\n",
    "    \n",
    "plot_roc_curve(fpr, tpr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9604938554008616"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_train_5, y_pred_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf_clf = RandomForestClassifier(random_state=42)\n",
    "y_proba_predict = cross_val_predict(rf_clf, X_train, y_train_5, cv=3, method='predict_proba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores_forest = y_proba_predict[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9983436731328145"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-class classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC()"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "svm_clf = SVC()\n",
    "svm_clf.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5], dtype=uint8)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.72501977,  2.72809088,  7.2510018 ,  8.3076379 , -0.31087254,\n",
       "         9.3132482 ,  1.70975103,  2.76765202,  6.23049537,  4.84771048]])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_digit_scores = svm_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=uint8)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = cross_val_predict(svm_clf, X_train, y_train, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5863,    3,   10,    2,    4,    8,   14,    2,   13,    4],\n",
       "       [   1, 6664,   26,    9,   10,    1,    3,   10,   10,    8],\n",
       "       [  19,    9, 5811,   16,   26,    5,    8,   30,   28,    6],\n",
       "       [   3,    9,   54, 5891,    2,   62,    1,   30,   61,   18],\n",
       "       [   5,   11,   13,    0, 5712,    0,   20,    9,    4,   68],\n",
       "       [  11,    6,   10,   57,   14, 5255,   35,    4,   17,   12],\n",
       "       [  20,    8,    7,    0,   10,   32, 5831,    0,   10,    0],\n",
       "       [   5,   21,   42,    6,   35,    2,    1, 6100,    7,   46],\n",
       "       [   8,   29,   19,   38,   20,   39,   21,    9, 5646,   22],\n",
       "       [  14,   11,    4,   31,   72,   16,    2,   59,   19, 5721]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "cm = confusion_matrix(y_train, y_train_pred)\n",
    "cm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f9203126208>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAKU0lEQVR4nO3dwYtd9RnG8edJxqLRiobWhRk1EcRUhKIMJRpwYVy0VRShCwsKdZNNq1EE0W78B0R0UYRB60bRRcyiSLEW1EU3oWMUok5KxKQajSQlVMVNlHm6mGubZFLvGXJ+Off6fj8QyNycnLzc3O+cc++c+7tOIgDfb2uGHgBAe4QOFEDoQAGEDhRA6EABhA4UMFjotn9u+x+2P7D9yFBzdGX7Mttv2F60/Z7tHUPP1IXttbbftv3K0LN0Yfsi2ztt7xvd1zcMPdM4th8cPSbetf2i7XOHnulUg4Rue62kP0j6haRrJP3a9jVDzLIK30h6KMlPJG2R9NspmFmSdkhaHHqIVXhK0qtJNkv6qSZ8dtsbJN0vaS7JtZLWSrpr2KlWGuqI/jNJHyT5MMlxSS9JumOgWTpJcjjJntHvv9TyA3DDsFN9N9uzkm6V9MzQs3Rh+0JJN0l6VpKSHE/y72Gn6mRG0nm2ZyStk/TpwPOsMFToGyR9fMLXhzTh0ZzI9kZJ10naPewkYz0p6WFJS0MP0tGVko5Kem70dOMZ2+cPPdR3SfKJpMclfSTpsKTPk7w27FQrDRW6T3PbVFyLa/sCSS9LeiDJF0PP8//Yvk3SkSRvDT3LKsxIul7S00muk/SVpIl+/cb2xVo+G90k6VJJ59u+e9ipVhoq9EOSLjvh61lN4OnOqWyfo+XIX0iya+h5xtgq6XbbB7X81Ohm288PO9JYhyQdSvLtmdJOLYc/yW6RdCDJ0SRfS9ol6caBZ1phqND/Lukq25ts/0DLL178aaBZOrFtLT93XEzyxNDzjJPk0SSzSTZq+f59PcnEHWlOlOQzSR/bvnp00zZJ7w84UhcfSdpie93oMbJNE/gC4swQ/2iSb2z/TtJftPwq5R+TvDfELKuwVdI9kvbafmd02++T/HnAmb6P7pP0wugA8KGkewee5zsl2W17p6Q9Wv7JzNuS5oedaiXzNlXg+48r44ACCB0ogNCBAggdKIDQgQIGD9329qFnWI1pm1di5rNh0ucdPHRJE30Hnca0zSsx89kw0fNOQugAGmtywcz69eszOzvbadtjx45p/fr1nbbdu3fvmYwFlJBkxZvGmlwCOzs7q1de6X9BkyuuuKL3fQIVcOoOFEDoQAGEDhRA6EABhA4U0Cn0aVuDHcDJxoY+pWuwAzhBlyP61K3BDuBkXUKf6jXYAXQLvdMa7La3216wvXDs2LEznwxAb7qE3mkN9iTzSeaSzHW9dh3A2dEl9Klbgx3Ayca+qWVK12AHcIJO714bfUgBH1QATCmujAMKIHSgAEIHCiB0oABCBwposjik7SYf0dryk1+XP9oamH6nWxySIzpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwV0+pDFSdFySeYDBw402e+mTZua7Bcnm8blulsuX34qjuhAAYQOFEDoQAGEDhRA6EABhA4UQOhAAWNDt32Z7TdsL9p+z/aOszEYgP50uWDmG0kPJdlj+4eS3rL91yTvN54NQE/GHtGTHE6yZ/T7LyUtStrQejAA/VnVc3TbGyVdJ2l3i2EAtNH5WnfbF0h6WdIDSb44zZ9vl7S9x9kA9KRT6LbP0XLkLyTZdbptksxLmh9tf/au1gcwVpdX3S3pWUmLSZ5oPxKAvnV5jr5V0j2Sbrb9zujXLxvPBaBHY0/dk/xN0vS92RfAf3FlHFAAoQMFEDpQAKEDBRA6UIBbrEQ5jRfMrFnT5nvewYMHm+xXki6//PIm+211XywtLTXZr9RuFdiWq8u2uj+SrBiaIzpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwWw3PMU279/f5P9XnXVVU32i5O1WFZ7aWmJ5Z6BqggdKIDQgQIIHSiA0IECCB0ogNCBAjqHbnut7bdtv9JyIAD9W80RfYekxVaDAGinU+i2ZyXdKumZtuMAaKHrEf1JSQ9LWmo4C4BGxoZu+zZJR5K8NWa77bYXbC/0Nh2AXnQ5om+VdLvtg5JeknSz7edP3SjJfJK5JHM9zwjgDI0NPcmjSWaTbJR0l6TXk9zdfDIAveHn6EABM6vZOMmbkt5sMgmAZjiiAwUQOlAAoQMFEDpQAKEDBbAKbGMtVvr81tJSmyuSd+/e3WS/W7ZsabJfSWrxOJ5WrAILFEXoQAGEDhRA6EABhA4UQOhAAYQOFEDoQAGEDhRA6EABhA4UQOhAAYQOFEDoQAGEDhRA6EABhA4UQOhAAYQOFEDoQAGEDhTAKrBYwV6xiGgvDhw40GS/krRx48Zm+542rAILFEXoQAGEDhRA6EABhA4UQOhAAYQOFNApdNsX2d5pe5/tRds3tB4MQH9mOm73lKRXk/zK9g8krWs4E4CejQ3d9oWSbpL0G0lKclzS8bZjAehTl1P3KyUdlfSc7bdtP2P7/MZzAehRl9BnJF0v6ekk10n6StIjp25ke7vtBdsLPc8I4Ax1Cf2QpENJdo++3qnl8E+SZD7JXJK5PgcEcObGhp7kM0kf2756dNM2Se83nQpAr7q+6n6fpBdGr7h/KOnediMB6Fun0JO8I4lTcmBKcWUcUAChAwUQOlAAoQMFEDpQAKEDBbDc80irJY5b3L/fmsaZWzly5EiT/V5yySVN9iu1+f9LwnLPQFWEDhRA6EABhA4UQOhAAYQOFEDoQAGEDhRA6EABhA4UQOhAAYQOFEDoQAGEDhRA6EABhA4UQOhAAYQOFEDoQAGEDhRA6EABU7UKbKtVT6V2K58y8/9M4+qy+/bta7bvzZs3N9kvq8ACRRE6UAChAwUQOlAAoQMFEDpQAKEDBXQK3faDtt+z/a7tF22f23owAP0ZG7rtDZLulzSX5FpJayXd1XowAP3peuo+I+k82zOS1kn6tN1IAPo2NvQkn0h6XNJHkg5L+jzJa60HA9CfLqfuF0u6Q9ImSZdKOt/23afZbrvtBdsL/Y8J4Ex0OXW/RdKBJEeTfC1pl6QbT90oyXySuSRzfQ8J4Mx0Cf0jSVtsr/Py25q2SVpsOxaAPnV5jr5b0k5JeyTtHf2d+cZzAejRTJeNkjwm6bHGswBohCvjgAIIHSiA0IECCB0ogNCBAggdKKDTj9cmxTQuF9zSmjVtvk8vLS012W9Lre6LVksyS9L+/ft73+edd9552ts5ogMFEDpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBRA6UAChAwUQOlAAoQMFEDpQAKEDBbjFyqq2j0r6Z8fNfyTpX70P0c60zSsx89kwKfNekeTHp97YJPTVsL2QZG7QIVZh2uaVmPlsmPR5OXUHCiB0oIBJCH1+6AFWadrmlZj5bJjoeQd/jg6gvUk4ogNojNCBAggdKIDQgQIIHSjgPxZTdFLkIFh6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(cm, cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_sums = cm.sum(axis=1, keepdims=True)\n",
    "norm_cm = cm/ row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f920319c278>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMT0lEQVR4nO3dTYxV9RnH8d+PeZGCRZpgxAKpoqYtaWisEzJI0hjFl7YoC7ugiU3aDZu+UGzS2G66MXFjGrpomqC2m5p2QU00WluaFBZNCHSc0VgcGpAiTgspRYWCLwPM08XcSXBm9J6L58+54/P9JCZwvT4+jvPl3Hvn3HMdEQLw8Tav6QUAlEfoQAKEDiRA6EAChA4kQOhAAo2Fbvse2/+wfcj2Q03tUZXtFbZ32R61vd/2lqZ3qsJ2j+0R2882vUsVthfb3mH7QOtrvbbpndqxvbX1PfF327+1Pb/pnaZrJHTbPZJ+IekrklZJ+obtVU3s0oHzkn4YEZ+XNCjpO3NgZ0naImm06SU68HNJf4yIz0n6orp8d9vLJH1f0kBEfEFSj6RNzW41U1NH9DWSDkXE4YgYl/Q7SRsb2qWSiDgWEcOtX/9Pk9+Ay5rd6sPZXi7pa5Ieb3qXKmwvkvRlSU9IUkSMR8RbzW5VSa+kT9julbRA0r8b3meGpkJfJun1i34/pi6P5mK2r5N0s6S9zW7S1jZJP5I00fQiFa2UdELSr1tPNx63vbDppT5MRPxL0qOSjko6JulUROxsdquZmgrds9w2J87FtX2lpN9L+kFEnG56nw9ie4Ok/0TEC03v0oFeSV+S9MuIuFnSWUld/fqN7U9p8tHo9ZI+LWmh7Qea3WqmpkIfk7Tiot8vVxc+3JnOdp8mI38yIp5qep821km6z/YRTT41ut32b5pdqa0xSWMRMfVIaYcmw+9m6yX9MyJORMQ5SU9JurXhnWZoKvS/SbrJ9vW2+zX54sUzDe1SiW1r8rnjaET8rOl92omIH0fE8oi4TpNf379ERNcdaS4WEcclvW77s62b7pD0SoMrVXFU0qDtBa3vkTvUhS8g9jbxL42I87a/K+lPmnyV8lcRsb+JXTqwTtI3Jb1s+8XWbT+JiD80uNPH0fckPdk6AByW9O2G9/lQEbHX9g5Jw5r8ycyIpO3NbjWTeZsq8PHHmXFAAoQOJEDoQAKEDiRA6EACjYdue3PTO3Riru0rsfPl0O37Nh66pK7+As1iru0rsfPl0NX7dkPoAAorcsKM7SJn4cybV+7Ppb6+vkr3u3Dhgnp6eirPHR8fv9SV2po847K9iKh8X0kd/fd1ore3+omY58+f7+j+77777qWs1Nb8+dWuIdHpvpL03nvvXcpKH2piYkIRMeN/diOnwF6qql/0S3HttdcWmTs2NlZkriRdccUVReYuWrSoyNwlS5YUmStJo6NlTi+/6aabisyVpCNHjtQ+88yZM7PezkN3IAFCBxIgdCABQgcSIHQggUqhz7VrsAN4v7ahz9FrsAO4SJUj+py7BjuA96sS+py+BjuAamfGVboGe+vdO119Yj+QVZXQK12DPSK2q3X1y1LnugO4NFUeus+5a7ADeL+2R/Q5eg12ABep9O611ocU8EEFwBzFmXFAAoQOJEDoQAKEDiRA6EACxa4ZV+ICg2+//XbtM6esXr26yNwSFwCcUuoijqWuc3fy5MkicyVp6dKlReaeOnWqyFxJWrhwYe0z33nnnVlv54gOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EACjqj/o8wXLVoUa9asqX3uoUOHap855bXXXisyd3BwsMhcSbrxxhuLzH3uueeKzC1xeeMp9957b5G54+PjReZK0jPP1P/p42+++abOnTvn6bdzRAcSIHQgAUIHEiB0IAFCBxIgdCABQgcSaBu67RW2d9ketb3f9pbLsRiA+lT5fPTzkn4YEcO2PynpBdt/johXCu8GoCZtj+gRcSwihlu//p+kUUnLSi8GoD4dPUe3fZ2kmyXtLbEMgDIqh277Skm/l/SDiDg9y9/fbHvI9tC5c+fq3BHAR1QpdNt9moz8yYh4arb7RMT2iBiIiIG+vr46dwTwEVV51d2SnpA0GhE/K78SgLpVOaKvk/RNSbfbfrH111cL7wWgRm1/vBYRf5U04/2tAOYOzowDEiB0IAFCBxIgdCABQgcSqPKmlo6dPXtWe/bsqX3u1VdfXfvMKTt37iwy96677ioyV5KGh4eLzB0ZGSkyd/Xq1UXmStLExESRuVu2lHuz5sGDB2uf+UHfExzRgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IwBFR+9De3t646qqrap9b8nLP8+fPLzL3pZdeKjJ3LrrnnnuKzT5w4ECRuW+88UaRuZJ0/Pjx2meuW7dOw8PDMz4UlSM6kAChAwkQOpAAoQMJEDqQAKEDCRA6kEDl0G332B6x/WzJhQDUr5Mj+hZJo6UWAVBOpdBtL5f0NUmPl10HQAlVj+jbJP1I0kTBXQAU0jZ02xsk/SciXmhzv822h2wPlTh/HsClq3JEXyfpPttHJP1O0u22fzP9ThGxPSIGImLAnnFOPYAGtQ09In4cEcsj4jpJmyT9JSIeKL4ZgNrwc3Qggd5O7hwRuyXtLrIJgGI4ogMJEDqQAKEDCRA6kAChAwl09Kp7VX19fVq6dGntc0+ePFn7zCmLFy8uMvfw4cNF5krSpk2biszdt29fkbm33XZbkbmStGfPniJzb7nlliJzJWlwcLD2mQcPHpz1do7oQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EACRa4C29/frxUrVtQ+9/Tp07XPLD175cqVReZK0g033FBk7tatW4vMfeihh4rMLeno0aPFZr/66qvFZk/HER1IgNCBBAgdSIDQgQQIHUiA0IEECB1IoFLothfb3mH7gO1R22tLLwagPlVPmPm5pD9GxNdt90taUHAnADVrG7rtRZK+LOlbkhQR45LGy64FoE5VHrqvlHRC0q9tj9h+3PbCwnsBqFGV0HslfUnSLyPiZklnJc04adn2ZttDtofGxzngA92kSuhjksYiYm/r9zs0Gf77RMT2iBiIiIH+/v46dwTwEbUNPSKOS3rd9mdbN90h6ZWiWwGoVdVX3b8n6cnWK+6HJX273EoA6lYp9Ih4UdJA4V0AFMKZcUAChA4kQOhAAoQOJEDoQAKEDiRQ5HLPZ8+e1d69e9vfsUNr1qypfeaUu+++u8jcRx55pMhcSbr//vuLzN22bVuRuT09PUXmStKFCxeKzD1z5kyRuZL02GOP1T7z4YcfnvV2juhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKOiNqH9vf3xzXXXFP73A0bNtQ+c8qhQ4eKzL3zzjuLzJWkXbt2FZm7fv36InNL7StJ+/fvLzL3yJEjReZK0saNG2ufuXv3br311luefjtHdCABQgcSIHQgAUIHEiB0IAFCBxIgdCCBSqHb3mp7v+2/2/6t7fmlFwNQn7ah214m6fuSBiLiC5J6JG0qvRiA+lR96N4r6RO2eyUtkPTvcisBqFvb0CPiX5IelXRU0jFJpyJiZ+nFANSnykP3T0naKOl6SZ+WtND2A7Pcb7PtIdtDExMT9W8K4JJVeei+XtI/I+JERJyT9JSkW6ffKSK2R8RARAzMm8eL+UA3qVLkUUmDthfYtqQ7JI2WXQtAnao8R98raYekYUkvt/6Z7YX3AlCj3ip3ioifSvpp4V0AFMKTaSABQgcSIHQgAUIHEiB0IAFCBxKo9OO1jof29mrJkiW1zy15ueBjx44Vmbtq1aoicyXp+eefLzJ37dq1ReaOjIwUmStJ+/btKzL3wQcfLDJXkp5++ulis6fjiA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJOCIqH+ofULSaxXvvkTSf2tfopy5tq/EzpdDt+z7mYi4evqNRULvhO2hiBhodIkOzLV9JXa+HLp9Xx66AwkQOpBAN4S+vekFOjTX9pXY+XLo6n0bf44OoLxuOKIDKIzQgQQIHUiA0IEECB1I4P8gCtOdf7PEMQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.fill_diagonal(norm_cm,0)\n",
    "plt.matshow(norm_cm, cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scikit learn uses one-vs-one classifier by default."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-class classifier \n",
    "## Multi-label classifier - > each data point can have multi labels\n",
    "## Multioutput classifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ipython-autotime\n",
      "  Downloading ipython_autotime-0.3.1-py2.py3-none-any.whl (6.8 kB)\n",
      "Requirement already satisfied: ipython in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython-autotime) (7.16.1)\n",
      "Requirement already satisfied: pickleshare in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (0.7.5)\n",
      "Requirement already satisfied: appnope; sys_platform == \"darwin\" in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (0.1.0)\n",
      "Requirement already satisfied: decorator in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (4.4.2)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (4.8.0)\n",
      "Requirement already satisfied: backcall in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (0.2.0)\n",
      "Requirement already satisfied: jedi>=0.10 in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (0.17.1)\n",
      "Requirement already satisfied: pygments in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (2.6.1)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (2.0.10)\n",
      "Requirement already satisfied: traitlets>=4.2 in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (4.3.3)\n",
      "Requirement already satisfied: setuptools>=18.5 in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from ipython->ipython-autotime) (49.2.0.post20200714)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from pexpect; sys_platform != \"win32\"->ipython->ipython-autotime) (0.6.0)\n",
      "Requirement already satisfied: parso<0.8.0,>=0.7.0 in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from jedi>=0.10->ipython->ipython-autotime) (0.7.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->ipython-autotime) (1.15.0)\n",
      "Requirement already satisfied: wcwidth in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->ipython-autotime) (0.2.5)\n",
      "Requirement already satisfied: ipython-genutils in /Users/aman.jain/anaconda3/lib/python3.6/site-packages (from traitlets>=4.2->ipython->ipython-autotime) (0.2.0)\n",
      "Installing collected packages: ipython-autotime\n",
      "Successfully installed ipython-autotime-0.3.1\n",
      "time: 478 µs (started: 2021-04-27 11:09:56 +09:00)\n"
     ]
    }
   ],
   "source": [
    "!pip install ipython-autotime\n",
    "%load_ext autotime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercises\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNeighbors Classifier \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=KNeighborsClassifier())"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2min 43s (started: 2021-04-27 11:11:40 +09:00)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "\n",
    "knn_clf = OneVsRestClassifier(KNeighborsClassifier())\n",
    "knn_clf.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/multiclass.py\u001b[0m in \u001b[0;36m_predict_binary\u001b[0;34m(estimator, X)\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m         \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecision_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mAttributeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'KNeighborsClassifier' object has no attribute 'decision_function'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-60-cf9f9bf63fae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcross_val_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mknn_clf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_val_score\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, error_score)\u001b[0m\n\u001b[1;32m    404\u001b[0m                                 \u001b[0mfit_params\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m                                 \u001b[0mpre_dispatch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpre_dispatch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 406\u001b[0;31m                                 error_score=error_score)\n\u001b[0m\u001b[1;32m    407\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcv_results\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'test_score'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_validate\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    246\u001b[0m             \u001b[0mreturn_times\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_estimator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_estimator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m             error_score=error_score)\n\u001b[0;32m--> 248\u001b[0;31m         for train, test in cv.split(X, y, groups))\n\u001b[0m\u001b[1;32m    249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     \u001b[0mzipped_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1027\u001b[0m             \u001b[0;31m# remaining jobs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1028\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1029\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1030\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1031\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    763\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    764\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 765\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    766\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    767\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    204\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    568\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    569\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 570\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    571\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    572\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 253\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 253\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    558\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m         \u001b[0mfit_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 560\u001b[0;31m         \u001b[0mtest_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    561\u001b[0m         \u001b[0mscore_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mfit_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    562\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mreturn_train_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_score\u001b[0;34m(estimator, X_test, y_test, scorer)\u001b[0m\n\u001b[1;32m    605\u001b[0m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    606\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 607\u001b[0;31m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    609\u001b[0m     error_msg = (\"scoring must return a number, got %s (%s) \"\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/metrics/_scorer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, estimator, *args, **kwargs)\u001b[0m\n\u001b[1;32m     88\u001b[0m                                       *args, **kwargs)\n\u001b[1;32m     89\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m                 \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m             \u001b[0mscores\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/metrics/_scorer.py\u001b[0m in \u001b[0;36m_passthrough_scorer\u001b[0;34m(estimator, *args, **kwargs)\u001b[0m\n\u001b[1;32m    370\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_passthrough_scorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m     \u001b[0;34m\"\"\"Function that wraps estimator.score\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 372\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    373\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/base.py\u001b[0m in \u001b[0;36mscore\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    497\u001b[0m         \"\"\"\n\u001b[1;32m    498\u001b[0m         \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 499\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    500\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    501\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_more_tags\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/multiclass.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    324\u001b[0m             \u001b[0margmaxima\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m                 \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_predict_binary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    327\u001b[0m                 \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaximum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaxima\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmaxima\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m                 \u001b[0margmaxima\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmaxima\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/multiclass.py\u001b[0m in \u001b[0;36m_predict_binary\u001b[0;34m(estimator, X)\u001b[0m\n\u001b[1;32m     97\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mAttributeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m         \u001b[0;31m# probabilities of the positive class\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mscore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/neighbors/_classification.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    217\u001b[0m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csr'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 219\u001b[0;31m         \u001b[0mneigh_dist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mneigh_ind\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkneighbors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    220\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m         \u001b[0mclasses_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/neighbors/_base.py\u001b[0m in \u001b[0;36mkneighbors\u001b[0;34m(self, X, n_neighbors, return_distance)\u001b[0m\n\u001b[1;32m    663\u001b[0m                 delayed_query(\n\u001b[1;32m    664\u001b[0m                     self._tree, X[s], n_neighbors, return_distance)\n\u001b[0;32m--> 665\u001b[0;31m                 \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen_even_slices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    666\u001b[0m             )\n\u001b[1;32m    667\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1027\u001b[0m             \u001b[0;31m# remaining jobs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1028\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1029\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1030\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1031\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    763\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    764\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 765\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    766\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    767\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    204\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    568\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    569\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 570\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    571\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    572\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 253\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 253\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/sklearn/neighbors/_base.py\u001b[0m in \u001b[0;36m_tree_query_parallel_helper\u001b[0;34m(tree, *args, **kwargs)\u001b[0m\n\u001b[1;32m    490\u001b[0m     \u001b[0munder\u001b[0m \u001b[0mPyPy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m     \"\"\"\n\u001b[0;32m--> 492\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    493\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    494\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 21min 11s (started: 2021-04-27 11:33:01 +09:00)\n"
     ]
    }
   ],
   "source": [
    "cross_val_score(knn_clf, X_train, y_train, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = [{'weights': [\"uniform\", \"distance\"], 'n_neighbors': [3, 4, 5]}]\n",
    "\n",
    "grid_search = GridSearchCV(knn_clf, param_grid, cv=5, verbose=3)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search.best_params_\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}