001_retrain.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fbae74ae-2cd5-4cd7-9a33-9f8ec2117853",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import time\n",
    "\n",
    "import torch\n",
    "\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "403b3846-9d50-4090-bde4-0409bd6df012",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_original_dataset, load_deleted_dataset\n",
    "from models import CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2f070e1c-cde4-44db-8533-1aa4e940cdd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = 'Datasets/Features/'\n",
    "BATCH_SIZE = 32\n",
    "EPOCHS = 5\n",
    "PERCENTAGES = [1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 99]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b44c5120-8746-4b94-aef6-5a1d4ae4239e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(model, save_dir, train_set, test_set, forget_set=None):\n",
    "    \n",
    "    os.makedirs(save_dir, exist_ok=True)\n",
    "    \n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "    error = torch.nn.CrossEntropyLoss()\n",
    "    \n",
    "    train_loader = torch.utils.data.DataLoader(train_set, batch_size = BATCH_SIZE, shuffle = True, drop_last=True)\n",
    "    \n",
    "    train_times = list()\n",
    "    train_accs, test_accs, forget_accs = list(), list(), list()\n",
    "    \n",
    "    for epoch in range(EPOCHS):\n",
    "        \n",
    "        # train\n",
    "        \n",
    "        accs = list()\n",
    "        \n",
    "        train_time = 0\n",
    "        \n",
    "        model.train()\n",
    "        \n",
    "        start_time = time.time()\n",
    "        \n",
    "        for x, y in train_loader:\n",
    "            optimizer.zero_grad()\n",
    "            output = model(x.cuda())\n",
    "            y = y.cuda()\n",
    "            loss = error(output, y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            train_time += time.time() - start_time\n",
    "            \n",
    "            predicted = torch.argmax(output.data, dim=-1)\n",
    "            accs.append((predicted == y).float().mean().detach().cpu().numpy())\n",
    "            \n",
    "            start_time = time.time()\n",
    "            \n",
    "        train_times.append(train_time)\n",
    "        train_accs.append(np.mean(accs))\n",
    "        \n",
    "        # eval\n",
    "            \n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            \n",
    "            x, y = test_set.tensors\n",
    "            \n",
    "            accs = list()\n",
    "            \n",
    "            for i in range(0, x.shape[0], BATCH_SIZE):\n",
    "            \n",
    "                output = model(x[i:i+BATCH_SIZE].cuda())\n",
    "\n",
    "                predicted = torch.argmax(output.data, dim=-1)\n",
    "                accs.append((predicted == y[i:i+BATCH_SIZE].cuda()).float().mean().detach().cpu().numpy())\n",
    "            \n",
    "            test_accs.append(np.mean(accs))\n",
    "            \n",
    "            if forget_set is not None:\n",
    "\n",
    "                x, y = forget_set.tensors\n",
    "            \n",
    "                accs = list()\n",
    "\n",
    "                for i in range(0, x.shape[0], BATCH_SIZE):\n",
    "\n",
    "                    output = model(x[i:i+BATCH_SIZE].cuda())\n",
    "\n",
    "                    predicted = torch.argmax(output.data, dim=-1)\n",
    "                    accs.append((predicted == y[i:i+BATCH_SIZE].cuda()).float().mean().detach().cpu().numpy())\n",
    "\n",
    "                forget_accs.append(np.mean(accs))\n",
    "        \n",
    "        # save\n",
    "        torch.save(model.state_dict(), os.path.join(save_dir, f'{(epoch+1):03d}.pt'))\n",
    "        \n",
    "\n",
    "    return train_times, train_accs, test_accs, forget_accs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dfba50e-81fa-485c-b08d-bfd041188a8f",
   "metadata": {},
   "source": [
    "# origin train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "42d4d623-d90f-4b9c-847d-855f883a28b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set, test_set = load_original_dataset(DATA_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cf7a9235-e2cb-4812-8356-6a2201dfce78",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CNN().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aef9173d-7c56-4b2d-9209-a7aff7bff936",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save\n",
    "torch.save(model.state_dict(), 'weights/init.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3430edee-b33d-408f-b66d-c41bac538fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_times, train_accs, test_accs, _ = fit(model, 'weights/original', train_set, test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "540e6f1a-05e8-41ed-9e6c-4fd2737329ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_time</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>test_acc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12.413962</td>\n",
       "      <td>0.868583</td>\n",
       "      <td>0.973143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.085280</td>\n",
       "      <td>0.944733</td>\n",
       "      <td>0.980931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10.715224</td>\n",
       "      <td>0.954433</td>\n",
       "      <td>0.989317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11.184714</td>\n",
       "      <td>0.957367</td>\n",
       "      <td>0.986821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11.101995</td>\n",
       "      <td>0.959433</td>\n",
       "      <td>0.985723</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       train_time  train_acc  test_acc\n",
       "epoch                                 \n",
       "1       12.413962   0.868583  0.973143\n",
       "2       11.085280   0.944733  0.980931\n",
       "3       10.715224   0.954433  0.989317\n",
       "4       11.184714   0.957367  0.986821\n",
       "5       11.101995   0.959433  0.985723"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(zip(train_times, train_accs, test_accs), columns=['train_time', 'train_acc', 'test_acc'])\n",
    "df['epoch'] = range(1, EPOCHS+1)\n",
    "df.set_index('epoch', inplace=True)\n",
    "\n",
    "df.to_csv('results/original.csv')\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "410c9ad5-1281-4186-ba95-56cb10e3903b",
   "metadata": {},
   "source": [
    "# naive retrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "472d663c-e0c7-4f5b-a11d-182f4fec09f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7c72e8492b84c0e890d82553e0a87e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = list()\n",
    "\n",
    "for percentage in tqdm(PERCENTAGES):\n",
    "    \n",
    "    model = CNN().cuda()\n",
    "        \n",
    "    model.load_state_dict(torch.load('./weights/init.pt'))\n",
    "    \n",
    "    train_set, test_set, forget_set = load_deleted_dataset(DATA_DIR, percentage)\n",
    "    \n",
    "    train_times, train_accs, test_accs, forget_accs = fit(model, f'weights/naive_retrain/{percentage}', train_set, test_set, forget_set)\n",
    "    \n",
    "    df = pd.DataFrame(zip(train_times, train_accs, test_accs, forget_accs), columns=['train_time', 'train_acc', 'test_acc', 'forget_acc'])\n",
    "    df['epoch'] = range(1, EPOCHS+1)\n",
    "    df['percentage'] = percentage\n",
    "    \n",
    "    results.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "50af2499-e6d9-48e9-bcf5-2ff69b681c3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>train_time</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>test_acc</th>\n",
       "      <th>forget_acc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percentage</th>\n",
       "      <th>epoch</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>11.065428</td>\n",
       "      <td>0.857018</td>\n",
       "      <td>0.972943</td>\n",
       "      <td>0.972039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.427742</td>\n",
       "      <td>0.940531</td>\n",
       "      <td>0.983327</td>\n",
       "      <td>0.983553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.147157</td>\n",
       "      <td>0.950313</td>\n",
       "      <td>0.985723</td>\n",
       "      <td>0.980263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12.055764</td>\n",
       "      <td>0.952418</td>\n",
       "      <td>0.985923</td>\n",
       "      <td>0.983553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>10.988493</td>\n",
       "      <td>0.958025</td>\n",
       "      <td>0.985623</td>\n",
       "      <td>0.981908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">10</th>\n",
       "      <th>1</th>\n",
       "      <td>9.890369</td>\n",
       "      <td>0.854957</td>\n",
       "      <td>0.973942</td>\n",
       "      <td>0.976396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9.809762</td>\n",
       "      <td>0.939964</td>\n",
       "      <td>0.981230</td>\n",
       "      <td>0.978890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.869044</td>\n",
       "      <td>0.950022</td>\n",
       "      <td>0.981929</td>\n",
       "      <td>0.979721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.747755</td>\n",
       "      <td>0.953523</td>\n",
       "      <td>0.986122</td>\n",
       "      <td>0.984874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9.809713</td>\n",
       "      <td>0.957302</td>\n",
       "      <td>0.987720</td>\n",
       "      <td>0.985539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">20</th>\n",
       "      <th>1</th>\n",
       "      <td>9.203177</td>\n",
       "      <td>0.846146</td>\n",
       "      <td>0.975839</td>\n",
       "      <td>0.972750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.865873</td>\n",
       "      <td>0.940021</td>\n",
       "      <td>0.981030</td>\n",
       "      <td>0.977250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.883193</td>\n",
       "      <td>0.948625</td>\n",
       "      <td>0.981430</td>\n",
       "      <td>0.978000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.964614</td>\n",
       "      <td>0.953500</td>\n",
       "      <td>0.984625</td>\n",
       "      <td>0.982083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9.964748</td>\n",
       "      <td>0.957479</td>\n",
       "      <td>0.987021</td>\n",
       "      <td>0.984500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">30</th>\n",
       "      <th>1</th>\n",
       "      <td>7.790184</td>\n",
       "      <td>0.833794</td>\n",
       "      <td>0.972444</td>\n",
       "      <td>0.971192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.562800</td>\n",
       "      <td>0.937309</td>\n",
       "      <td>0.979333</td>\n",
       "      <td>0.977020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.678257</td>\n",
       "      <td>0.946742</td>\n",
       "      <td>0.982228</td>\n",
       "      <td>0.979518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.073815</td>\n",
       "      <td>0.952887</td>\n",
       "      <td>0.983427</td>\n",
       "      <td>0.981239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7.733404</td>\n",
       "      <td>0.955626</td>\n",
       "      <td>0.985823</td>\n",
       "      <td>0.982571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">40</th>\n",
       "      <th>1</th>\n",
       "      <td>6.567179</td>\n",
       "      <td>0.825694</td>\n",
       "      <td>0.972244</td>\n",
       "      <td>0.970125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.508478</td>\n",
       "      <td>0.932250</td>\n",
       "      <td>0.978235</td>\n",
       "      <td>0.976333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.338063</td>\n",
       "      <td>0.944167</td>\n",
       "      <td>0.985224</td>\n",
       "      <td>0.981125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.731266</td>\n",
       "      <td>0.948944</td>\n",
       "      <td>0.982328</td>\n",
       "      <td>0.979125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.758206</td>\n",
       "      <td>0.956194</td>\n",
       "      <td>0.986821</td>\n",
       "      <td>0.981917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">50</th>\n",
       "      <th>1</th>\n",
       "      <td>5.289086</td>\n",
       "      <td>0.815635</td>\n",
       "      <td>0.970447</td>\n",
       "      <td>0.967784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.597664</td>\n",
       "      <td>0.930563</td>\n",
       "      <td>0.981030</td>\n",
       "      <td>0.978345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.557550</td>\n",
       "      <td>0.943670</td>\n",
       "      <td>0.979034</td>\n",
       "      <td>0.977479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.662526</td>\n",
       "      <td>0.946872</td>\n",
       "      <td>0.976038</td>\n",
       "      <td>0.973181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5.625827</td>\n",
       "      <td>0.953642</td>\n",
       "      <td>0.982827</td>\n",
       "      <td>0.980810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">60</th>\n",
       "      <th>1</th>\n",
       "      <td>4.396187</td>\n",
       "      <td>0.737250</td>\n",
       "      <td>0.967252</td>\n",
       "      <td>0.964139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.404668</td>\n",
       "      <td>0.920250</td>\n",
       "      <td>0.971845</td>\n",
       "      <td>0.967278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.580806</td>\n",
       "      <td>0.937167</td>\n",
       "      <td>0.981729</td>\n",
       "      <td>0.978639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.822374</td>\n",
       "      <td>0.945250</td>\n",
       "      <td>0.979433</td>\n",
       "      <td>0.974611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4.400605</td>\n",
       "      <td>0.945417</td>\n",
       "      <td>0.980931</td>\n",
       "      <td>0.978556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">70</th>\n",
       "      <th>1</th>\n",
       "      <td>3.170623</td>\n",
       "      <td>0.725923</td>\n",
       "      <td>0.942592</td>\n",
       "      <td>0.942665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.333591</td>\n",
       "      <td>0.888123</td>\n",
       "      <td>0.964157</td>\n",
       "      <td>0.962038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.223794</td>\n",
       "      <td>0.917816</td>\n",
       "      <td>0.968351</td>\n",
       "      <td>0.964633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.229980</td>\n",
       "      <td>0.932051</td>\n",
       "      <td>0.973043</td>\n",
       "      <td>0.971820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3.314745</td>\n",
       "      <td>0.941003</td>\n",
       "      <td>0.973742</td>\n",
       "      <td>0.971106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">80</th>\n",
       "      <th>1</th>\n",
       "      <td>2.032088</td>\n",
       "      <td>0.696583</td>\n",
       "      <td>0.940196</td>\n",
       "      <td>0.934521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.153865</td>\n",
       "      <td>0.896833</td>\n",
       "      <td>0.959864</td>\n",
       "      <td>0.957521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.216328</td>\n",
       "      <td>0.918583</td>\n",
       "      <td>0.972045</td>\n",
       "      <td>0.964625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.103795</td>\n",
       "      <td>0.934667</td>\n",
       "      <td>0.975539</td>\n",
       "      <td>0.972583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.099107</td>\n",
       "      <td>0.944917</td>\n",
       "      <td>0.973642</td>\n",
       "      <td>0.970646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">90</th>\n",
       "      <th>1</th>\n",
       "      <td>0.681389</td>\n",
       "      <td>0.543449</td>\n",
       "      <td>0.886182</td>\n",
       "      <td>0.870612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.966778</td>\n",
       "      <td>0.840241</td>\n",
       "      <td>0.928714</td>\n",
       "      <td>0.924152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.982314</td>\n",
       "      <td>0.889873</td>\n",
       "      <td>0.945986</td>\n",
       "      <td>0.941480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.936354</td>\n",
       "      <td>0.917781</td>\n",
       "      <td>0.957668</td>\n",
       "      <td>0.953014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.940168</td>\n",
       "      <td>0.929646</td>\n",
       "      <td>0.966054</td>\n",
       "      <td>0.961067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">99</th>\n",
       "      <th>1</th>\n",
       "      <td>0.061015</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.217652</td>\n",
       "      <td>0.218060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.074171</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.276158</td>\n",
       "      <td>0.274687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.069081</td>\n",
       "      <td>0.328125</td>\n",
       "      <td>0.513478</td>\n",
       "      <td>0.507455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.068599</td>\n",
       "      <td>0.491319</td>\n",
       "      <td>0.710763</td>\n",
       "      <td>0.707223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.068632</td>\n",
       "      <td>0.633681</td>\n",
       "      <td>0.711162</td>\n",
       "      <td>0.710134</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  train_time  train_acc  test_acc  forget_acc\n",
       "percentage epoch                                             \n",
       "1          1       11.065428   0.857018  0.972943    0.972039\n",
       "           2       11.427742   0.940531  0.983327    0.983553\n",
       "           3       11.147157   0.950313  0.985723    0.980263\n",
       "           4       12.055764   0.952418  0.985923    0.983553\n",
       "           5       10.988493   0.958025  0.985623    0.981908\n",
       "10         1        9.890369   0.854957  0.973942    0.976396\n",
       "           2        9.809762   0.939964  0.981230    0.978890\n",
       "           3        9.869044   0.950022  0.981929    0.979721\n",
       "           4        9.747755   0.953523  0.986122    0.984874\n",
       "           5        9.809713   0.957302  0.987720    0.985539\n",
       "20         1        9.203177   0.846146  0.975839    0.972750\n",
       "           2        8.865873   0.940021  0.981030    0.977250\n",
       "           3        8.883193   0.948625  0.981430    0.978000\n",
       "           4        8.964614   0.953500  0.984625    0.982083\n",
       "           5        9.964748   0.957479  0.987021    0.984500\n",
       "30         1        7.790184   0.833794  0.972444    0.971192\n",
       "           2        7.562800   0.937309  0.979333    0.977020\n",
       "           3        7.678257   0.946742  0.982228    0.979518\n",
       "           4        8.073815   0.952887  0.983427    0.981239\n",
       "           5        7.733404   0.955626  0.985823    0.982571\n",
       "40         1        6.567179   0.825694  0.972244    0.970125\n",
       "           2        6.508478   0.932250  0.978235    0.976333\n",
       "           3        6.338063   0.944167  0.985224    0.981125\n",
       "           4        6.731266   0.948944  0.982328    0.979125\n",
       "           5        6.758206   0.956194  0.986821    0.981917\n",
       "50         1        5.289086   0.815635  0.970447    0.967784\n",
       "           2        5.597664   0.930563  0.981030    0.978345\n",
       "           3        5.557550   0.943670  0.979034    0.977479\n",
       "           4        5.662526   0.946872  0.976038    0.973181\n",
       "           5        5.625827   0.953642  0.982827    0.980810\n",
       "60         1        4.396187   0.737250  0.967252    0.964139\n",
       "           2        4.404668   0.920250  0.971845    0.967278\n",
       "           3        4.580806   0.937167  0.981729    0.978639\n",
       "           4        4.822374   0.945250  0.979433    0.974611\n",
       "           5        4.400605   0.945417  0.980931    0.978556\n",
       "70         1        3.170623   0.725923  0.942592    0.942665\n",
       "           2        3.333591   0.888123  0.964157    0.962038\n",
       "           3        3.223794   0.917816  0.968351    0.964633\n",
       "           4        3.229980   0.932051  0.973043    0.971820\n",
       "           5        3.314745   0.941003  0.973742    0.971106\n",
       "80         1        2.032088   0.696583  0.940196    0.934521\n",
       "           2        2.153865   0.896833  0.959864    0.957521\n",
       "           3        2.216328   0.918583  0.972045    0.964625\n",
       "           4        2.103795   0.934667  0.975539    0.972583\n",
       "           5        2.099107   0.944917  0.973642    0.970646\n",
       "90         1        0.681389   0.543449  0.886182    0.870612\n",
       "           2        0.966778   0.840241  0.928714    0.924152\n",
       "           3        0.982314   0.889873  0.945986    0.941480\n",
       "           4        0.936354   0.917781  0.957668    0.953014\n",
       "           5        0.940168   0.929646  0.966054    0.961067\n",
       "99         1        0.061015   0.125000  0.217652    0.218060\n",
       "           2        0.074171   0.208333  0.276158    0.274687\n",
       "           3        0.069081   0.328125  0.513478    0.507455\n",
       "           4        0.068599   0.491319  0.710763    0.707223\n",
       "           5        0.068632   0.633681  0.711162    0.710134"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = pd.concat(results).set_index(['percentage', 'epoch'])\n",
    "\n",
    "results.to_csv('results/naive_retrain.csv')\n",
    "\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93ac3f21-a349-4ab4-b744-79b0ebb746a3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}