notebooks/ToyModelTrackMLComet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": false
   },
   "source": [
    "# Graph Training Sandbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cell_style": "center",
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# System imports\n",
    "import os\n",
    "import sys\n",
    "from pprint import pprint as pp\n",
    "from time import time as tt\n",
    "sys.path.append('..')\n",
    "\n",
    "# sys.path.append('/global/common/cori_cle7/software/jupyter/19-11/lib/python3.7/site-packages')\n",
    "sys.path.append('/global/homes/d/danieltm/.local/lib/python3.7/site-packages')\n",
    "\n",
    "# External imports\n",
    "from comet_ml import OfflineExperiment\n",
    "from comet_ml import Experiment\n",
    "os.environ['COMET_LOGGING_FILE'] = \"/global/homes/d/danieltm/ExaTrkX/GNN-Sandbox/notebooks/comet/log/debug.log\"\n",
    "os.environ['COMET_LOGGING_FILE_LEVEL'] = \"debug\"\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.colors\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch_geometric.data import Data\n",
    "from torch_geometric.data import DataLoader\n",
    "import seaborn as sns\n",
    "\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import interact, interact_manual\n",
    "\n",
    "# Limit CPU usage on Jupyter\n",
    "os.environ['OMP_NUM_THREADS'] = '4'\n",
    "\n",
    "# Local imports\n",
    "from utils.toy_utils import *\n",
    "from datasets.hitgraphs_params import *\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading TrackML Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m\u001b[01;34mhitgraphs_med_000\u001b[0m/\n"
     ]
    }
   ],
   "source": [
    "%ls $SCRATCH/ExaTrkX/node_tracker_data/\n",
    "g1 = np.load(\"/global/cscratch1/sd/danieltm/ExaTrkX/node_tracker_data/hitgraphs_med_000/event000001000_g000.npz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = \"/global/cscratch1/sd/danieltm/ExaTrkX/node_tracker_data/hitgraphs_med_000/\"\n",
    "filenames = [os.path.join(input_dir, f) for f in os.listdir(input_dir)\n",
    "                         if f.endswith('.npz') and not f.endswith('_ID.npz')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_graphs = [load_graph(fi) for fi in filenames[:10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cut_mask = [~(np.isnan(g[3]).any()) for g in full_graphs]\n",
    "cut_full_graphs = np.array(full_graphs)[cut_mask].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cut_full_dataset = [torch_geometric.data.Data(x=torch.from_numpy(di[0]),\n",
    "                                         edge_index=torch.from_numpy(di[1]), y_edges=torch.from_numpy(di[2]), \n",
    "                                         y_params=(torch.from_numpy(di[3][:,0]).unsqueeze(1)), pid=torch.from_numpy(di[4])) for di in full_graphs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Visualise data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2aab7fd80cf8>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAYgklEQVR4nO3df5Dc9X3f8edrd+8kncAIoaM1+hGpqeKaNBgyV+yp4xq7TkbQDGonToJqJ3EHW+NMSNOxm5gkHZyQZDp2ZmLHUxJH4zCkSYAQO3E0Hnkog8nQ1oZwBIeACEYhtnTIsQ6EhI2A0+6+88f3u3dfnXZvv9LtafXZ7+sxc6P98dXu53ucXvfm/f18PquIwMzM0lcb9gDMzGwwHOhmZiPCgW5mNiIc6GZmI8KBbmY2IhrDeuMNGzbE1q1bh/X2ZmZJevTRR5+PiMluzw0t0Ldu3cr09PSw3t7MLEmSvtHrObdczMxGRN9Al3S7pCOSnuhz3L+R1JL07sENz8zMyipTod8B7FjqAEl14GPAvQMYk5mZnYW+gR4RDwJH+xz2s8DngCODGJSZmZ25ZffQJW0E/hPw6RLH7pY0LWl6dnZ2uW9tZmYFg7go+kngIxHR6ndgROyJiKmImJqc7DrrxszMztIgpi1OAXdLAtgAXCepGRGfH8Brm5lZScsO9IjY1rkt6Q7gCw5zM7Nzr8y0xbuArwBvkDQj6UZJH5T0wZUf3tK+8cLLTP36fRw6emLYQzEzG7q+FXpE7Cr7YhHxvmWN5gx944UTPP+dOQ4dPcHm9RPn8q3NzM47Sa8UbeWftjTXag95JGZmw5d2oLeyQG+2/DF6ZmZJB3qznQX5SVfoZmZpB3qrE+htV+hmZkkHerOdVeYnm67QzcySDvR2flG0E+xmZlWWdKB3LobO+aKomVnagd7poTd9UdTMLO1A9ywXM7MFSQf6/CwXt1zMzEYl0F2hm5mNRKB7paiZWeKB7h66mdmCpAO91VlY5ArdzCztQHeFbma2IOlAn++he6WomdloBPpc0y0XM7ORCHRX6GZmiQe6e+hmZguSDnSvFDUzW5B0oM/vh+4K3cysf6BLul3SEUlP9Hj+PZIez7++LOlNgx9md50c90pRM7NyFfodwI4lnv8H4O0RcQXwa8CeAYyrlM7CojlX6GZmNPodEBEPStq6xPNfLtx9CNi0/GGV44uiZmYLBt1DvxH4Yq8nJe2WNC1penZ2dtlv5s25zMwWDCzQJb2DLNA/0uuYiNgTEVMRMTU5Obns93SFbma2oG/LpQxJVwCfAa6NiBcG8ZpltB3oZmbzll2hS9oC/BnwExHxteUPqbym56Gbmc3rW6FLugu4BtggaQb4KDAGEBGfBm4BLgF+RxJAMyKmVmrARf6QaDOzBWVmuezq8/z7gfcPbERnoFOhz7lCNzNLe6VoZx66N+cyM0s+0PMeetOBbmY2GoHedsvFzCzpQPc8dDOzBUkHeqdCj1i4bWZWVUkHenHJv6t0M6u6pAO9HQ50M7OOpAO92S4GulsuZlZtSQd6sW/u1aJmVnVJB3pxQZE/5MLMqi7pQG+1gkZNgPdENzNLO9AjWD1WB3xR1Mws7UBvFwPdFbqZVVvSgd5sB6vHslNwhW5mVZd0oLdawZq8QveOi2ZWdUkHerPQcplruuViZtWWdKC3Ci0XV+hmVnVpB7pnuZiZzUs20CPCs1zMzAqSDfTOsn9X6GZmmWQDvbMx15pOD90VuplVXN9Al3S7pCOSnujxvCR9StIBSY9L+v7BD/N0iyt07+ViZlVXpkK/A9ixxPPXAtvzr93A7y5/WP214tRAd4VuZlXXN9Aj4kHg6BKH7AT+d2QeAtZJev2gBthLKw/w1Q2vFDUzg8H00DcChwr3Z/LHVlSnh7563BdFzcxgMIGuLo917X9I2i1pWtL07Ozsst50vofe8LRFMzMYTKDPAJsL9zcBh7sdGBF7ImIqIqYmJyeX9aadlaFrxjs9dFfoZlZtgwj0vcBP5rNd3gIcj4hvDuB1l9RZ6T9edw/dzAyg0e8ASXcB1wAbJM0AHwXGACLi08A+4DrgAHAC+C8rNdiiToXeqIvxeo05t1zMrOL6BnpE7OrzfAA/M7ARldTpoddrolGXWy5mVnnJrxRt1MRYveaWi5lVXrKBvlCh1xiri5Ntt1zMrNpGINDJKvSmK3Qzq7ZkA71ZqNAbdc3fNzOrqmQDvbWoh+7Nucys6pIN9M60xXpNjNVqnuViZpWXbKCfUqE35KX/ZlZ5yQd6rSYaNU9bNDNLPtAbtWylqAPdzKou2UBvnrZS1C0XM6u2ZAN9oUKveaWomRkJB3qxQh+r+6KomVmygd4+JdBdoZuZJRvoxc25GvWaV4qaWeUlG+it4sKiupjzXi5mVnHJBvop2+fWavMrR83MqirZQC9+wIVXipqZJRzonXnnda8UNTMDEg70diwE+njDgW5mlmygd3ron3v0Ob72j99mrtnmzocPcufDB4c8MjOz4Ug20Bc258o26GoHRLiPbmbVlWygd3roNYlGTQC0HOhmVmGlAl3SDklPSzog6eYuz2+R9ICkxyQ9Lum6wQ/1VJ156DWJmvJA9+IiM6uwvoEuqQ7cBlwLXA7sknT5osP+B3BPRFwF3AD8zqAHulgrgrwwp57f8FR0M6uyMhX61cCBiHg2IuaAu4Gdi44J4HX57YuAw4MbYnfNdsxX5p1A9+IiM6uyMoG+EThUuD+TP1b0K8B7Jc0A+4Cf7fZCknZLmpY0PTs7exbDXdBqBbXaqYHujouZVVmZQFeXxxZH5y7gjojYBFwH/KGk0147IvZExFRETE1OTp75aAuyCj27XXcP3cysVKDPAJsL9zdxekvlRuAegIj4CrAa2DCIAfbS6tJycaCbWZWVCfRHgO2StkkaJ7vouXfRMQeBfw8g6Y1kgb68nkof2UXRLMhrDnQzs/6BHhFN4CbgXuApstksT0q6VdL1+WEfBj4g6W+Au4D3xQqv8mm1FlounoduZgaNMgdFxD6yi53Fx24p3N4PvHWwQ1tas71wUdTz0M3MEl4p2mq33UM3MytINtC7zUN3oJtZlSUb6O0uK0Ud6GZWZckGerNVqNDVWVjkQDez6ko20FvtoJaPfmHpvwPdzKor2UAv9tA7wd52oJtZhSUb6MWVoo080d1DN7MqSzzQs9udP72wyMyqLPFA97RFM7OOZAO92W6ftn2uA93MqizZQC+2XBzoZmYJB/opK0U9D93MLN1Ab7VP3z7X89DNrMoSD/Tsdk2iJs9DN7NqSzvQO4lO1kd3D93MqizZQC/20CGr0j0P3cyqLNlAby0KdFfoZlZ1iQf6wn0HuplVXbKBvrjl4kA3s6pLNtBb7fb8LouQzUV3D93MqizZQHeFbmZ2qlKBLmmHpKclHZB0c49jfkzSfklPSrpzsMM8XbeLop6HbmZV1uh3gKQ6cBvwg8AM8IikvRGxv3DMduAXgbdGxIuSLl2pAXc0u10UdcvFzCqsTIV+NXAgIp6NiDngbmDnomM+ANwWES8CRMSRwQ7zdO1u89BdoZtZhZUJ9I3AocL9mfyxou8BvkfS/5f0kKQd3V5I0m5J05KmZ2dnz27EQERkFbpXipqZzSsT6Ory2OLkbADbgWuAXcBnJK077S9F7ImIqYiYmpycPNOxzuvktuehm5ktKBPoM8Dmwv1NwOEux/xFRJyMiH8AniYL+BXRbLcBTr0o6mmLZlZxZQL9EWC7pG2SxoEbgL2Ljvk88A4ASRvIWjDPDnKgRZ1K/PRZLiv1jmZm57++gR4RTeAm4F7gKeCeiHhS0q2Srs8Puxd4QdJ+4AHg5yPihZUa9EKgLzxWr8n7oZtZpfWdtggQEfuAfYseu6VwO4AP5V8rbj7QF10U9ScWmVmVJblStNmt5eJpi2ZWcUkGeq8eugPdzKosyUBvdumh1xzoZlZxSQZ6u0uF3vDSfzOruCQDfb5CL4zeS//NrOqSDPRWt4VF+W6L4SrdzCoqyUDvOsullu1H4CLdzKoqzUBvdVtYlJ2K2y5mVlVJBnpnAdGp89BPfc7MrGqSDPRmj5WixefMzKomyUDvtrCoE+7+GDozq6okA71bD72R33EP3cyqKslA71qh57e9uMjMqirNQI/u2+eCK3Qzq640A72zsKjLRVEHuplVVZKBvtBDd6CbmXUkGehdt8/Nb3seuplVVZKB3m37XM9DN7OqSzLQu64U9Tx0M6u4JAN9vofui6JmZvOSDPTWEi2Xkw50M6uoUoEuaYekpyUdkHTzEse9W1JImhrcEE/XbfvcifEGAK/MtVbyrc3Mzlt9A11SHbgNuBa4HNgl6fIux10I/Ffg4UEPcrFu89AnxusAnJhrrvTbm5mdl8pU6FcDByLi2YiYA+4GdnY57teAjwOvDnB8Xc1X6IXHxuo1xurihCt0M6uoMoG+EThUuD+TPzZP0lXA5oj4wlIvJGm3pGlJ07Ozs2c82I5Wl+1zAdaON3j5NVfoZlZNZQJdXR6bv/IoqQZ8AvhwvxeKiD0RMRURU5OTk+VHuUi3hUWQtV1coZtZVZUJ9Blgc+H+JuBw4f6FwL8G/lLS14G3AHtX8sJot4VFABOrGu6hm1lllQn0R4DtkrZJGgduAPZ2noyI4xGxISK2RsRW4CHg+oiYXpER07vl4grdzKqsb6BHRBO4CbgXeAq4JyKelHSrpOtXeoDddCr0xb2gifEGL7tCN7OKapQ5KCL2AfsWPXZLj2OvWf6wltZuB/WaUJce+qsn2zRbbRr1JNdMmZmdtSRTr5kH+mJr87nox145ea6HZGY2dEkGeqvdnv8M0aKJVdn/cBw7MXeuh2RmNnRJBnqvCr2zWvToy67Qzax6kgz0Vs+WS1ahv+gK3cwqKNlA79pyySv0F192oJtZ9SQb6N1bLlmFftQVuplVUJKB3mwHjdrpQx9v1GjUxLET7qGbWfUkGei9KnSAtasaHHXLxcwqKMlA7zXLBbI+uqctmlkVJRno7T6B7grdzKooyUBv9lhYBNmFUffQzayKkgz0pXroE+N1z3Ixs0pKMtCbPeahQ3ZR9PgrJ+e32DUzq4okA73VjtP2Qu+YGK8TAce9QZeZVUyygb5UDx28/N/MqifJQO83bRG8/N/MqifJQG/1WCkKCxt0eeqimVVNkoFepkL31EUzq5okA73XB1wATKzK90R3D93MKibRQKfnLJfxeo3xes0XRc2schIN9N4VuiQuXjvmi6JmVjmlAl3SDklPSzog6eYuz39I0n5Jj0u6X9J3DX6oC5bqoQNcPDHOi+6hm1nF9A10SXXgNuBa4HJgl6TLFx32GDAVEVcAnwU+PuiBFi01Dx3yQHeFbmYVU6ZCvxo4EBHPRsQccDews3hARDwQESfyuw8BmwY7zFM1W0G9x7RFgPVrx91DN7PKKRPoG4FDhfsz+WO93Ah8cTmD6ifbnKv38+smxtxyMbPKaZQ4pltvo+vOV5LeC0wBb+/x/G5gN8CWLVtKDvF0rehfoR87MUd7iT1fzMxGTZkKfQbYXLi/CTi8+CBJ7wJ+Gbg+Il7r9kIRsScipiJianJy8mzGC/Tvoa+bGKcd8NKrrtLNrDrKBPojwHZJ2ySNAzcAe4sHSLoK+D2yMD8y+GGeqtlqLznLZf3aMcDL/82sWvoGekQ0gZuAe4GngHsi4klJt0q6Pj/sN4ELgD+V9FVJe3u83ECUqdAB99HNrFLK9NCJiH3AvkWP3VK4/a4Bj2tJ/eahr+8Euit0M6uQJFeKtqNPoK/NAv2Fl7u28s3MRlKSgb7UR9ABXLZuDRPjdfYffukcjsrMbLiSC/R2O4hgyWmL9Zp406Z1PHbo2DkcmZnZcCUX6M38w58b9aXnl1+5ZR37D7/Eqydb52JYZmZDl1ygt/JAr2npQL9q8zqa7eDJw8fPxbDMzIYuvUCPvELvswL0yi3rAHjsoNsuZlYN6QV6Kwv0pWa5AFx64Wo2XbzGgW5mlZFcoDfbbaB/Dx3gys3reOzgiys9JDOz80Jygd7pofer0AGu2nIxh4+/yrdeenWlh2VmNnSlVoqeTzqzXOoS7a57PsKdDx8EYPbb2cKiT93/DN972UX85zef/Q6PZmbnu5Gu0C+7aDX1mjh09ETfY83MUpdsoJfpoTfqNS67aDUHj76y0sMyMxu65AJ9vuWyxErRok3rJ3ju2In5XwRmZqMquUCfr9BLfhLRlosnONkKXxg1s5GXXKB3pi32WynasXn9BAB/P/udFRuTmdn5ILlAz/O8dIV+8cQY2zas5Ut/d4TnjrmXbmajK7lA71To9RIXRQEk8SPfv4kI+MhnHyfCvXQzG03JBfqZ9tAh+8CLa7/vn/P/DjzPH+Vz1M3MRk1ygd48g3noRVdvXc/btm/gf+57im+88PJKDM3MbKiSC/RWYaXomZDEx37kCuo18Z7PPOyLpGY2cpIN9DILixa7bN0a/vj9b+aVuRbv/t0v89feuMvMRkiygV52YVHRnQ8f5InnXuJ9/3Yrkvjx3/sKP/1Hj3prADMbCaVSUdIOSU9LOiDp5i7Pr5L0J/nzD0vaOuiBdjTP4qLoYpdcsIoPvv272bhuDV984h9528cfYMcnH+QT932NA0fcijGzNPXdbVFSHbgN+EFgBnhE0t6I2F847EbgxYj4l5JuAD4G/PhKDLjVmba4jEAHuGBVg93/7rt54TuvsWa8zn37v8WnvvQMv33/M7zx9a/jzdvWc9GaMS5c3eB1q8eYWFVn7XiDVY0anYmPAlaN1VjVqANw/JWTHH15jldOtthwwTiXXriaSy9cxZrxOqsadcbqQkv0/rtNqVzq+MWarTatCBq1GjWd+ncjgnZk0z4jYKxeW/b3sKgz9jMZr5kNVpntc68GDkTEswCS7gZ2AsVA3wn8Sn77s8D/kqRYgUnfZzvLpZdLLlgFwM4rN/KOf3UpTzx3nMdnjnP3Iwd59WR7IO/R0cm6M/2u1LSwMjbIwlMSdYlaLXu9k632adsJF7O123s2amKsXqMVQbsd8x/vB9kvq3pN8+8TZL8QFv8nbQen7JPTqIl6/nXKWM7gfANoRxCR3RbZ+UvdXyfIxtCOoNkOasrHIDHA31mlx935Pil//5p0Rud/LmQ/R/n3mez7qvN0rKnqfI+DmP/3l/0Miw+8bRsf+qE3DPw9ywT6RuBQ4f4M8OZex0REU9Jx4BLg+eJBknYDu/O735H09FmMeQPw/Bs+dhZ/M00bWPR9HGE+19FVpfPte64fzr/O0nf1eqJMoPcqis70GCJiD7CnxHv2How0HRFTy3mNlFTpfH2uo6tK5zvMcy1zUXQG2Fy4vwk43OsYSQ3gIuDoIAZoZmbllAn0R4DtkrZJGgduAPYuOmYv8FP57XcDX1qJ/rmZmfXWt+WS98RvAu4F6sDtEfGkpFuB6YjYC/w+8IeSDpBV5jes4JiX1bJJUJXO1+c6uqp0vkM7V7mQNjMbDcmtFDUzs+4c6GZmIyKpQO+3BUHqJN0u6YikJwqPrZd0n6Rn8j8vHuYYB0HSZkkPSHpK0pOSfi5/fOTOFUDSakl/Jelv8vP91fzxbflWGc/kW2eMD3usgyKpLukxSV/I74/kuUr6uqS/lfRVSdP5Y0P7OU4m0AtbEFwLXA7sknT5cEc1cHcAOxY9djNwf0RsB+7P76euCXw4It4IvAX4mfy/5SieK8BrwDsj4k3AlcAOSW8h2yLjE/n5vki2hcao+DngqcL9UT7Xd0TElYW550P7OU4m0ClsQRARc0BnC4KREREPcvr8/Z3AH+S3/wD4j+d0UCsgIr4ZEX+d3/422T/8jYzguQJEprPr21j+FcA7ybbKgBE6X0mbgP8AfCa/L0b0XHsY2s9xSoHebQuCjUMay7n0zyLim5AFIXDpkMczUPnOnFcBDzPC55q3IL4KHAHuA/4eOBYRzfyQUfp5/iTwC0BnM6RLGN1zDeD/SHo039oEhvhzXGbp//mi1PYClg5JFwCfA/5bRLw0yjs1RkQLuFLSOuDPgTd2O+zcjmrwJP0wcCQiHpV0TefhLocmf665t0bEYUmXAvdJ+rthDialCr3MFgSj6FuSXg+Q/3lkyOMZCEljZGH+xxHxZ/nDI3muRRFxDPhLsmsH6/KtMmB0fp7fClwv6etkbdF3klXso3iuRMTh/M8jZL+or2aIP8cpBXqZLQhGUXFbhZ8C/mKIYxmIvKf6+8BTEfFbhadG7lwBJE3mlTmS1gDvIrtu8ADZVhkwIucbEb8YEZsiYivZv9EvRcR7GMFzlbRW0oWd28APAU8wxJ/jpFaKSrqO7Ld9ZwuC3xjykAZK0l3ANWTbb34L+CjweeAeYAtwEPjRiEh64zNJPwD8X+BvWeiz/hJZH32kzhVA0hVkF8fqZEXUPRFxq6R/QVbFrgceA94bEa8Nb6SDlbdc/ntE/PAonmt+Tn+e320Ad0bEb0i6hCH9HCcV6GZm1ltKLRczM1uCA93MbEQ40M3MRoQD3cxsRDjQzcxGhAPdzGxEONDNzEbEPwFoQ7RGwQHipAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(np.concatenate(np.array([di[3][:,0] for di in full_graphs]))[:100000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Constructing PyG Datastructure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_dataset = [gen_graph_class(event_size_min, event_size_max, max_curve, min_curve, height, num_layers, max_angle) for i in range(1000)]\n",
    "test_dataset = [gen_graph_class(event_size_min, event_size_max, max_curve, min_curve, height, num_layers, max_angle) for i in range(500)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True)\n",
    "test_loader = DataLoader(test_dataset, batch_size=100, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using  cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(\"Using \", device)\n",
    "# model = Edge_Class_Net( input_dim=2, hidden_dim=64, n_graph_iters=4).to(device)\n",
    "model = Net(train_dataset).to(device)\n",
    "# data = dataset[0].to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.1, weight_decay=1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Edge Classification Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true,
    "hidden": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  1 , loss:  0.5512356758117676 , accuracy:  0.6864395625058761\n",
      "Epoch:  2 , loss:  0.5046652555465698 , accuracy:  0.7436710645899276\n",
      "Epoch:  3 , loss:  0.48637688159942627 , accuracy:  0.7629634272462315\n",
      "Epoch:  4 , loss:  0.47630026936531067 , accuracy:  0.7705349587890564\n",
      "Epoch:  5 , loss:  0.4798726439476013 , accuracy:  0.7736814064997336\n",
      "Epoch:  6 , loss:  0.4695682227611542 , accuracy:  0.7731611770973706\n",
      "Epoch:  7 , loss:  0.4612908959388733 , accuracy:  0.7738506377511047\n",
      "Epoch:  8 , loss:  0.44322115182876587 , accuracy:  0.7768779968034097\n",
      "Epoch:  9 , loss:  0.46184107661247253 , accuracy:  0.7758438058228087\n",
      "Epoch:  10 , loss:  0.43595728278160095 , accuracy:  0.7730358206148735\n",
      "Epoch:  11 , loss:  0.40324726700782776 , accuracy:  0.7812466702184336\n",
      "Epoch:  12 , loss:  0.3373474180698395 , accuracy:  0.7961014133943402\n",
      "Epoch:  13 , loss:  0.27982667088508606 , accuracy:  0.8321476699363816\n",
      "Epoch:  14 , loss:  0.2253979742527008 , accuracy:  0.8571813594910527\n",
      "Epoch:  15 , loss:  0.21640172600746155 , accuracy:  0.8812498041304961\n",
      "Epoch:  16 , loss:  0.1846497803926468 , accuracy:  0.9013381804506565\n",
      "Epoch:  17 , loss:  0.2350691705942154 , accuracy:  0.8991068350622081\n",
      "Epoch:  18 , loss:  0.1935807168483734 , accuracy:  0.8991256385345827\n",
      "Epoch:  19 , loss:  0.15857245028018951 , accuracy:  0.912563853458272\n",
      "Epoch:  20 , loss:  0.18233557045459747 , accuracy:  0.9181923595223918\n",
      "Epoch:  21 , loss:  0.16060705482959747 , accuracy:  0.9187752671660033\n",
      "Epoch:  22 , loss:  0.15544316172599792 , accuracy:  0.9248111817982387\n",
      "Epoch:  23 , loss:  0.12819615006446838 , accuracy:  0.9281519320567865\n",
      "Epoch:  24 , loss:  0.1715494692325592 , accuracy:  0.9166567426118023\n",
      "Epoch:  25 , loss:  0.15350672602653503 , accuracy:  0.9195086025886113\n",
      "Epoch:  26 , loss:  0.13381382822990417 , accuracy:  0.9310163276818453\n",
      "Epoch:  27 , loss:  0.16854779422283173 , accuracy:  0.9178727004920242\n",
      "Epoch:  28 , loss:  0.12585857510566711 , accuracy:  0.9288288570622708\n",
      "Epoch:  29 , loss:  0.1297493726015091 , accuracy:  0.936086997398853\n",
      "Epoch:  30 , loss:  0.13670764863491058 , accuracy:  0.9336112068695352\n",
      "Epoch:  31 , loss:  0.12563517689704895 , accuracy:  0.9378733272744367\n",
      "Epoch:  32 , loss:  0.10643548518419266 , accuracy:  0.9423736249960826\n",
      "Epoch:  33 , loss:  0.11343897879123688 , accuracy:  0.9475069729543389\n",
      "Epoch:  34 , loss:  0.10112588107585907 , accuracy:  0.9496004262120404\n",
      "Epoch:  35 , loss:  0.10293048620223999 , accuracy:  0.9479018458742048\n",
      "Epoch:  36 , loss:  0.10879960656166077 , accuracy:  0.9497132470462879\n",
      "Epoch:  37 , loss:  0.08479493856430054 , accuracy:  0.9517252185903664\n",
      "Epoch:  38 , loss:  0.08469492197036743 , accuracy:  0.9548340593562945\n",
      "Epoch:  39 , loss:  0.09417087584733963 , accuracy:  0.9555423234824031\n",
      "Epoch:  40 , loss:  0.09632786363363266 , accuracy:  0.9555172521859037\n",
      "Epoch:  41 , loss:  0.09234034270048141 , accuracy:  0.9530728007772102\n",
      "Epoch:  42 , loss:  0.07967119663953781 , accuracy:  0.9559309285781441\n",
      "Epoch:  43 , loss:  0.08349929749965668 , accuracy:  0.957924096649848\n",
      "Epoch:  44 , loss:  0.07990405708551407 , accuracy:  0.957710990629603\n",
      "Epoch:  45 , loss:  0.09039483219385147 , accuracy:  0.9594722492086872\n",
      "Epoch:  46 , loss:  0.0847826823592186 , accuracy:  0.9588266633238272\n",
      "Epoch:  47 , loss:  0.07450012862682343 , accuracy:  0.9607258140336582\n",
      "Epoch:  48 , loss:  0.08207990229129791 , accuracy:  0.9606819392647842\n",
      "Epoch:  49 , loss:  0.08625155687332153 , accuracy:  0.9572221003478643\n",
      "Epoch:  50 , loss:  0.08353188633918762 , accuracy:  0.9602118524554201\n",
      "Epoch:  51 , loss:  0.07035135477781296 , accuracy:  0.9608699739885299\n",
      "Epoch:  52 , loss:  0.07894345372915268 , accuracy:  0.9623930552508697\n",
      "Epoch:  53 , loss:  0.08261425048112869 , accuracy:  0.9617098624212604\n",
      "Epoch:  54 , loss:  0.07314658910036087 , accuracy:  0.9608449026920305\n",
      "Epoch:  55 , loss:  0.07553153485059738 , accuracy:  0.9610956156570246\n",
      "Epoch:  56 , loss:  0.08274070173501968 , accuracy:  0.9619856466827541\n",
      "Epoch:  57 , loss:  0.07393041253089905 , accuracy:  0.9628004638189852\n",
      "Epoch:  58 , loss:  0.07949676364660263 , accuracy:  0.9619104327932558\n",
      "Epoch:  59 , loss:  0.07388246804475784 , accuracy:  0.9609201165815288\n",
      "Epoch:  60 , loss:  0.07234286516904831 , accuracy:  0.9622676987683726\n",
      "Epoch:  61 , loss:  0.07553829252719879 , accuracy:  0.964154313829954\n",
      "Epoch:  62 , loss:  0.07114172726869583 , accuracy:  0.9638973330408349\n",
      "Epoch:  63 , loss:  0.07683456689119339 , accuracy:  0.9638597260960857\n",
      "Epoch:  64 , loss:  0.07580337673425674 , accuracy:  0.9636027453069667\n",
      "Epoch:  65 , loss:  0.07201939821243286 , accuracy:  0.9629508915979818\n",
      "Epoch:  66 , loss:  0.06553474068641663 , accuracy:  0.9643987589708233\n",
      "Epoch:  67 , loss:  0.0714544951915741 , accuracy:  0.9651320943934313\n",
      "Epoch:  68 , loss:  0.06275434046983719 , accuracy:  0.9649503274938105\n",
      "Epoch:  69 , loss:  0.06658802181482315 , accuracy:  0.966097339308659\n",
      "Epoch:  70 , loss:  0.0704989805817604 , accuracy:  0.9662540349117804\n",
      "Epoch:  71 , loss:  0.059061162173748016 , accuracy:  0.9654705568961734\n",
      "Epoch:  72 , loss:  0.06813564896583557 , accuracy:  0.9660597323639099\n",
      "Epoch:  73 , loss:  0.06905221194028854 , accuracy:  0.9658716976401642\n",
      "Epoch:  74 , loss:  0.0689922496676445 , accuracy:  0.9655771099062961\n",
      "Epoch:  75 , loss:  0.06837943941354752 , accuracy:  0.9663229809771537\n",
      "Epoch:  76 , loss:  0.05394389107823372 , accuracy:  0.9671127268168855\n",
      "Epoch:  77 , loss:  0.07251249998807907 , accuracy:  0.9667930677865179\n",
      "Epoch:  78 , loss:  0.07537279278039932 , accuracy:  0.9664169983390266\n",
      "Epoch:  79 , loss:  0.06626568734645844 , accuracy:  0.9660221254191608\n",
      "Epoch:  80 , loss:  0.07760695368051529 , accuracy:  0.9664734087561503\n",
      "Epoch:  81 , loss:  0.06580779701471329 , accuracy:  0.9625748221504905\n",
      "Epoch:  82 , loss:  0.05862663313746452 , accuracy:  0.9656147168510452\n",
      "Epoch:  83 , loss:  0.060530368238687515 , accuracy:  0.9675577423297502\n",
      "Epoch:  84 , loss:  0.05909406766295433 , accuracy:  0.969168573129838\n",
      "Epoch:  85 , loss:  0.06882461905479431 , accuracy:  0.9686734150239744\n",
      "Epoch:  86 , loss:  0.0680612325668335 , accuracy:  0.9690244131749663\n",
      "Epoch:  87 , loss:  0.05981859937310219 , accuracy:  0.971330972452913\n",
      "Epoch:  88 , loss:  0.05889563262462616 , accuracy:  0.9706289761509292\n",
      "Epoch:  89 , loss:  0.06496661901473999 , accuracy:  0.9716819706039048\n",
      "Epoch:  90 , loss:  0.05609319731593132 , accuracy:  0.9718637375035256\n",
      "Epoch:  91 , loss:  0.05172021687030792 , accuracy:  0.9737190134444828\n",
      "Epoch:  92 , loss:  0.061566054821014404 , accuracy:  0.972841518067003\n",
      "Epoch:  93 , loss:  0.05868617817759514 , accuracy:  0.9713121689805384\n",
      "Epoch:  94 , loss:  0.05601102486252785 , accuracy:  0.9713936506941615\n",
      "Epoch:  95 , loss:  0.0565975159406662 , accuracy:  0.972885392835877\n",
      "Epoch:  96 , loss:  0.06436861306428909 , accuracy:  0.9743332602087186\n",
      "Epoch:  97 , loss:  0.05844764783978462 , accuracy:  0.9742831176157197\n",
      "Epoch:  98 , loss:  0.04329262301325798 , accuracy:  0.972797643298129\n",
      "Epoch:  99 , loss:  0.04300661012530327 , accuracy:  0.9760318405465542\n",
      "Epoch:  100 , loss:  0.060396987944841385 , accuracy:  0.9762888213356733\n",
      "Epoch:  101 , loss:  0.050678230822086334 , accuracy:  0.9777053495878906\n",
      "Epoch:  102 , loss:  0.0520532988011837 , accuracy:  0.976345231752797\n",
      "Epoch:  103 , loss:  0.04601125046610832 , accuracy:  0.9768403898586606\n",
      "Epoch:  104 , loss:  0.05154242366552353 , accuracy:  0.9773230123162744\n",
      "Epoch:  105 , loss:  0.04324488714337349 , accuracy:  0.9760757153154282\n",
      "Epoch:  106 , loss:  0.05357293039560318 , accuracy:  0.9770597637030305\n",
      "Epoch:  107 , loss:  0.05393035709857941 , accuracy:  0.977210191482027\n",
      "Epoch:  108 , loss:  0.06744545698165894 , accuracy:  0.9722147356545175\n",
      "Epoch:  109 , loss:  0.050511591136455536 , accuracy:  0.9722084678303927\n",
      "Epoch:  110 , loss:  0.04840807616710663 , accuracy:  0.9769281393964085\n",
      "Epoch:  111 , loss:  0.04603151977062225 , accuracy:  0.9788460935786142\n",
      "Epoch:  112 , loss:  0.049852970987558365 , accuracy:  0.9779121877840108\n",
      "Epoch:  113 , loss:  0.06091667711734772 , accuracy:  0.977517314864145\n",
      "Epoch:  114 , loss:  0.045730751007795334 , accuracy:  0.9755742893854399\n",
      "Epoch:  115 , loss:  0.046588025987148285 , accuracy:  0.9791594847848569\n",
      "Epoch:  116 , loss:  0.04268786311149597 , accuracy:  0.979761195900843\n",
      "Epoch:  117 , loss:  0.05151399224996567 , accuracy:  0.9765896768936664\n",
      "Epoch:  118 , loss:  0.045346617698669434 , accuracy:  0.9789275752922373\n",
      "Epoch:  119 , loss:  0.040606942027807236 , accuracy:  0.9785201667241217\n",
      "Epoch:  120 , loss:  0.03887306526303291 , accuracy:  0.9797486602525933\n",
      "Epoch:  121 , loss:  0.043499015271663666 , accuracy:  0.9801874079413332\n",
      "Epoch:  122 , loss:  0.042412832379341125 , accuracy:  0.9791469491366073\n",
      "Epoch:  123 , loss:  0.04148733615875244 , accuracy:  0.980193675765458\n",
      "Epoch:  124 , loss:  0.03858195245265961 , accuracy:  0.9796609107148453\n",
      "Epoch:  125 , loss:  0.03824522718787193 , accuracy:  0.9811714563289354\n",
      "Epoch:  126 , loss:  0.04555254429578781 , accuracy:  0.97946034034285\n",
      "Epoch:  127 , loss:  0.04094495251774788 , accuracy:  0.977899652135761\n",
      "Epoch:  128 , loss:  0.03822688013315201 , accuracy:  0.9811401172083112\n",
      "Epoch:  129 , loss:  0.030357101932168007 , accuracy:  0.9822557899025354\n",
      "Epoch:  130 , loss:  0.03848995640873909 , accuracy:  0.9800933905794603\n",
      "Epoch:  131 , loss:  0.034991804510354996 , accuracy:  0.9812341345701839\n",
      "Epoch:  132 , loss:  0.03558206930756569 , accuracy:  0.9811965276254349\n",
      "Epoch:  133 , loss:  0.047384537756443024 , accuracy:  0.9814033658215551\n",
      "Epoch:  134 , loss:  0.03825727105140686 , accuracy:  0.9815224544799273\n",
      "Epoch:  135 , loss:  0.03913990408182144 , accuracy:  0.9816979535554232\n",
      "Epoch:  136 , loss:  0.050426892936229706 , accuracy:  0.9813344197561816\n",
      "Epoch:  137 , loss:  0.04604649543762207 , accuracy:  0.9807640477608198\n",
      "Epoch:  138 , loss:  0.04506449028849602 , accuracy:  0.9819486665204175\n",
      "Epoch:  139 , loss:  0.07405521720647812 , accuracy:  0.9720267009307719\n",
      "Epoch:  140 , loss:  0.04675091430544853 , accuracy:  0.9755868250336895\n",
      "Epoch:  141 , loss:  0.03860006853938103 , accuracy:  0.980538406092325\n",
      "Epoch:  142 , loss:  0.03254537656903267 , accuracy:  0.9821555047165377\n",
      "Epoch:  143 , loss:  0.040432099252939224 , accuracy:  0.9814973831834278\n",
      "Epoch:  144 , loss:  0.04703542962670326 , accuracy:  0.9801372653483343\n",
      "Epoch:  145 , loss:  0.03535279259085655 , accuracy:  0.9815099188316776\n",
      "Epoch:  146 , loss:  0.03875648230314255 , accuracy:  0.9833150521796359\n",
      "Epoch:  147 , loss:  0.04009599611163139 , accuracy:  0.9814033658215551\n",
      "Epoch:  148 , loss:  0.045232970267534256 , accuracy:  0.9813845623491805\n",
      "Epoch:  149 , loss:  0.03929991275072098 , accuracy:  0.9841612084364912\n",
      "Epoch:  150 , loss:  0.03452068567276001 , accuracy:  0.9841047980193676\n",
      "Epoch:  151 , loss:  0.03560980409383774 , accuracy:  0.9832962487072613\n",
      "Epoch:  152 , loss:  0.041717901825904846 , accuracy:  0.9828010906013978\n",
      "Epoch:  153 , loss:  0.057955581694841385 , accuracy:  0.9778056347738883\n",
      "Epoch:  154 , loss:  0.04936639219522476 , accuracy:  0.9731674449214955\n",
      "Epoch:  155 , loss:  0.04441019147634506 , accuracy:  0.977122441944279\n",
      "Epoch:  156 , loss:  0.03909330815076828 , accuracy:  0.981967469992792\n",
      "Epoch:  157 , loss:  0.030724773183465004 , accuracy:  0.9836660503306277\n",
      "Epoch:  158 , loss:  0.0435817688703537 , accuracy:  0.9832774452348867\n",
      "Epoch:  159 , loss:  0.033314041793346405 , accuracy:  0.9824124855056567\n",
      "Epoch:  160 , loss:  0.0374787338078022 , accuracy:  0.983264909586637\n",
      "Epoch:  161 , loss:  0.030533701181411743 , accuracy:  0.9833213200037607\n",
      "Epoch:  162 , loss:  0.035522811114788055 , accuracy:  0.9842301545018647\n",
      "Epoch:  163 , loss:  0.0327361561357975 , accuracy:  0.9842364223259895\n",
      "Epoch:  164 , loss:  0.03695467859506607 , accuracy:  0.9837914068131248\n",
      "Epoch:  165 , loss:  0.03675539046525955 , accuracy:  0.9829201792597699\n",
      "Epoch:  166 , loss:  0.039733801037073135 , accuracy:  0.9826381271741514\n",
      "Epoch:  167 , loss:  0.03238556906580925 , accuracy:  0.9838791563508728\n",
      "Epoch:  168 , loss:  0.030454277992248535 , accuracy:  0.9841925475571155\n",
      "Epoch:  169 , loss:  0.043676987290382385 , accuracy:  0.983007928797518\n",
      "Epoch:  170 , loss:  0.04005815088748932 , accuracy:  0.9820050769375411\n",
      "Epoch:  171 , loss:  0.045554470270872116 , accuracy:  0.9817418283242972\n",
      "Epoch:  172 , loss:  0.03457340970635414 , accuracy:  0.9826694662947758\n",
      "Epoch:  173 , loss:  0.033884212374687195 , accuracy:  0.9835406938481306\n",
      "Epoch:  174 , loss:  0.030815908685326576 , accuracy:  0.9840483876022439\n",
      "Epoch:  175 , loss:  0.03124821186065674 , accuracy:  0.9849258829797236\n",
      "Epoch:  176 , loss:  0.03896614909172058 , accuracy:  0.9772603340750259\n",
      "Epoch:  177 , loss:  0.034430284053087234 , accuracy:  0.9812529380425585\n",
      "Epoch:  178 , loss:  0.037297699600458145 , accuracy:  0.9817731674449215\n",
      "Epoch:  179 , loss:  0.0333557203412056 , accuracy:  0.9834090695415086\n",
      "Epoch:  180 , loss:  0.0382612943649292 , accuracy:  0.9839042276473722\n",
      "Epoch:  181 , loss:  0.03331493213772774 , accuracy:  0.9845811526528565\n",
      "Epoch:  182 , loss:  0.024810975417494774 , accuracy:  0.9847002413112288\n",
      "Epoch:  183 , loss:  0.0436268150806427 , accuracy:  0.9776990817637657\n",
      "Epoch:  184 , loss:  0.04014063626527786 , accuracy:  0.9790341283023598\n",
      "Epoch:  185 , loss:  0.037939392030239105 , accuracy:  0.9835156225516312\n",
      "Epoch:  186 , loss:  0.026451334357261658 , accuracy:  0.9848882760349744\n",
      "Epoch:  187 , loss:  0.02467665821313858 , accuracy:  0.9855338619198345\n",
      "Epoch:  188 , loss:  0.03608778864145279 , accuracy:  0.9855338619198345\n",
      "Epoch:  189 , loss:  0.02868589572608471 , accuracy:  0.9848067943213513\n",
      "Epoch:  190 , loss:  0.026818392798304558 , accuracy:  0.9853708984925883\n",
      "Epoch:  191 , loss:  0.030235012993216515 , accuracy:  0.9854649158544612\n",
      "Epoch:  192 , loss:  0.029142208397388458 , accuracy:  0.9861230373875709\n",
      "Epoch:  193 , loss:  0.02885563299059868 , accuracy:  0.9865680529004356\n",
      "Epoch:  194 , loss:  0.030767781659960747 , accuracy:  0.9868438371619292\n",
      "Epoch:  195 , loss:  0.031723205000162125 , accuracy:  0.9850136325174715\n",
      "Epoch:  196 , loss:  0.03449144959449768 , accuracy:  0.985471183678586\n",
      "Epoch:  197 , loss:  0.03040768951177597 , accuracy:  0.985991413080949\n",
      "Epoch:  198 , loss:  0.02778073027729988 , accuracy:  0.9841800119088658\n",
      "Epoch:  199 , loss:  0.031113574281334877 , accuracy:  0.984737848255978\n",
      "Epoch:  200 , loss:  0.0350588783621788 , accuracy:  0.985038703813971\n",
      "Epoch:  201 , loss:  0.03575316071510315 , accuracy:  0.9850261681657213\n",
      "Epoch:  202 , loss:  0.027306491509079933 , accuracy:  0.9857469679400797\n",
      "Epoch:  203 , loss:  0.02635180577635765 , accuracy:  0.9861669121564449\n",
      "Epoch:  204 , loss:  0.05310515686869621 , accuracy:  0.9747782757215833\n",
      "Epoch:  205 , loss:  0.04839653521776199 , accuracy:  0.9793851264533517\n",
      "Epoch:  206 , loss:  0.03698990121483803 , accuracy:  0.9839481024162462\n",
      "Epoch:  207 , loss:  0.03777533397078514 , accuracy:  0.9848255977937259\n",
      "Epoch:  208 , loss:  0.03729405999183655 , accuracy:  0.9835845686170046\n",
      "Epoch:  209 , loss:  0.031062213703989983 , accuracy:  0.9845059387633583\n",
      "Epoch:  210 , loss:  0.023279579356312752 , accuracy:  0.9856529505782068\n",
      "Epoch:  211 , loss:  0.03791983425617218 , accuracy:  0.9857657714124541\n",
      "Epoch:  212 , loss:  0.03225843608379364 , accuracy:  0.9841549406123664\n",
      "Epoch:  213 , loss:  0.035936351865530014 , accuracy:  0.9843993857532357\n",
      "Epoch:  214 , loss:  0.034864574670791626 , accuracy:  0.9852768811307154\n",
      "Epoch:  215 , loss:  0.028552347794175148 , accuracy:  0.9857908427089536\n",
      "Epoch:  216 , loss:  0.02753402292728424 , accuracy:  0.9852267385377166\n",
      "Epoch:  217 , loss:  0.031228698790073395 , accuracy:  0.9858535209502022\n",
      "Epoch:  218 , loss:  0.028560616075992584 , accuracy:  0.9865429816039362\n",
      "Epoch:  219 , loss:  0.031074870377779007 , accuracy:  0.9857156288194553\n",
      "Epoch:  220 , loss:  0.03851897269487381 , accuracy:  0.9836785859788775\n",
      "Epoch:  221 , loss:  0.039986029267311096 , accuracy:  0.983396533893259\n",
      "Epoch:  222 , loss:  0.033866241574287415 , accuracy:  0.9857281644677051\n",
      "Epoch:  223 , loss:  0.026249274611473083 , accuracy:  0.9867122128553073\n",
      "Epoch:  224 , loss:  0.0259819608181715 , accuracy:  0.9858033783572033\n",
      "Epoch:  225 , loss:  0.02425507828593254 , accuracy:  0.9853708984925883\n",
      "Epoch:  226 , loss:  0.03318915516138077 , accuracy:  0.9861105017393212\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-51-f515b0ae1d58>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m         \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinary_cross_entropy_with_logits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/global/u2/d/danieltm/ExaTrkX/eta-tracker/notebooks/toy_utils.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m    206\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_graph_iters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m             \u001b[0;31m# Apply edge network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 208\u001b[0;31m             \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msigmoid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0medge_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0medge_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    209\u001b[0m             \u001b[0;31m# Apply node network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnode_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0medge_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/global/u2/d/danieltm/ExaTrkX/eta-tracker/notebooks/toy_utils.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, edge_index)\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0medge_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnetwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0medge_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mNodeNetwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/nn/modules/normalization.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    150\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         return F.layer_norm(\n\u001b[0;32m--> 152\u001b[0;31m             input, self.normalized_shape, self.weight, self.bias, self.eps)\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlayer_norm\u001b[0;34m(input, normalized_shape, weight, bias, eps)\u001b[0m\n\u001b[1;32m   1680\u001b[0m     \"\"\"\n\u001b[1;32m   1681\u001b[0m     return torch.layer_norm(input, normalized_shape, weight, bias, eps,\n\u001b[0;32m-> 1682\u001b[0;31m                             torch.backends.cudnn.enabled)\n\u001b[0m\u001b[1;32m   1683\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1684\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.train()\n",
    "loss_v = []\n",
    "acc_v = []\n",
    "ep = 0\n",
    "for epoch in range(500):\n",
    "    ep += 1\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch in train_loader:\n",
    "#         print(batch.x)\n",
    "        optimizer.zero_grad()\n",
    "        data = batch.to(device)\n",
    "        pred = model(data)\n",
    "        loss = F.binary_cross_entropy_with_logits(pred.float(), data.y.float())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        correct += ((pred > 0.5) == (data.y > 0.5)).sum().item()\n",
    "#         print(correct, pred, data.y)\n",
    "        total += len(pred)\n",
    "#         print(out, data.y, )\n",
    "    acc = correct/total\n",
    "    print(\"Epoch: \" , ep, \", loss: \", loss.item(), \", accuracy: \", acc)\n",
    "    loss_v.append(loss)\n",
    "    acc_v.append(acc)\n",
    "plt.plot(np.arange(len(loss_v)), loss_v)\n",
    "plt.plot(np.arange(len(acc_v)), acc_v)\n",
    "plt.ylim(0.1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch(batch=[5930], edge_index=[2, 14580], x=[5930, 2], y=[14580])\n",
      "Accuracy: 0.9812\n",
      "Batch(batch=[5880], edge_index=[2, 14637], x=[5880, 2], y=[14637])\n",
      "Accuracy: 0.9832\n",
      "Batch(batch=[5580], edge_index=[2, 14190], x=[5580, 2], y=[14190])\n",
      "Accuracy: 0.9862\n",
      "Batch(batch=[5710], edge_index=[2, 14652], x=[5710, 2], y=[14652])\n",
      "Accuracy: 0.9838\n",
      "Batch(batch=[5470], edge_index=[2, 12952], x=[5470, 2], y=[12952])\n",
      "Accuracy: 0.9866\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "for batch in test_loader:\n",
    "    print(batch)\n",
    "    data = batch.to(device)\n",
    "    pred = model(data)\n",
    "    correct = ((pred > 0.5) == (data.y > 0.5)).sum().item()\n",
    "    acc = correct / len(pred)\n",
    "    print('Accuracy: {:.4f}'.format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Track Count Testing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_dataset = [gen_graph_class(event_size_min, event_size_max, max_curve, min_curve, height, num_layers, max_angle) for i in range(1000)]\n",
    "test_dataset = [gen_graph_class(event_size_min, event_size_max, max_curve, min_curve, height, num_layers, max_angle) for i in range(500)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True)\n",
    "test_loader = DataLoader(test_dataset, batch_size=100, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using  cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(\"Using \", device)\n",
    "# model = Edge_Class_Net( input_dim=2, hidden_dim=64, n_graph_iters=4).to(device)\n",
    "model = Net(train_dataset).to(device)\n",
    "# data = dataset[0].to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.005, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "hidden": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 9,  9,  7,  4,  7,  3,  5,  7,  6,  7,  4,  7,  2,  6,  5,  6,  7,  9,\n",
      "         5,  9,  6,  9,  5,  7,  8,  8,  4,  4,  7,  5,  6,  4,  4,  8, 10,  1,\n",
      "         5,  8,  8,  5,  8,  4,  9,  5,  9,  6,  2,  9,  5,  4,  7,  4,  9,  5,\n",
      "         9,  5,  5,  4,  9,  6,  5,  7,  6,  4,  7,  4, 11,  4, 11,  5,  7,  4,\n",
      "         7,  7,  9,  5,  3,  7,  3,  6,  3,  8,  3,  6,  6,  4,  6,  4,  8,  3,\n",
      "         7,  7,  8,  2,  6,  6,  3,  4,  8,  2], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 3, 10,  9,  3,  7,  7,  5,  8,  4,  6, 10,  4,  7,  5,  3,  5, 10,  7,\n",
      "         4,  6,  9,  5,  7,  4,  8,  4,  3,  6,  3,  6,  5,  4,  9,  7,  6,  8,\n",
      "         7,  8,  9,  3,  3,  7,  9,  8,  7,  4, 11,  6,  6,  4,  4,  8,  5,  7,\n",
      "         4,  5,  6,  6,  7,  4,  6,  3,  7,  5,  6,  8,  6,  8,  8,  9,  4,  8,\n",
      "         4,  7,  7,  5,  6,  7,  5,  7,  3,  7,  6,  5,  8,  5,  2,  5,  7,  5,\n",
      "         7,  3,  9,  9,  5,  9,  7,  4,  6,  7], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 8,  4,  2,  7,  7,  4,  5,  6,  9,  6,  3,  7,  7,  5,  9,  3,  5,  5,\n",
      "         4,  5,  9,  9,  7,  7,  8,  3,  7,  5,  9,  6,  7,  3,  6,  5,  9,  3,\n",
      "        10,  3,  7,  5,  9,  6,  5,  8,  4,  3,  3,  6,  4,  5,  4, 10,  7,  5,\n",
      "         5,  6,  9,  5,  6,  5,  6,  4,  7,  4,  3,  3,  6,  8,  6, 10,  7,  7,\n",
      "         8,  4,  5,  3,  8,  8,  4,  2,  5,  5,  3,  9,  5, 11,  9,  5,  3,  6,\n",
      "         9,  4,  9,  8,  7,  7,  5,  4,  6,  7], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 4,  4,  6,  7,  9,  8,  4,  8,  4,  4,  3, 11,  4,  6,  9,  6,  7,  5,\n",
      "         7,  8,  6,  5,  4,  7,  9,  9,  4,  4,  9,  9,  4,  6,  5,  4,  6,  7,\n",
      "         4,  9,  4,  4,  6,  8,  6,  2,  6,  6,  3,  7,  6,  7,  2,  7,  7,  3,\n",
      "         3,  6,  7,  7,  7,  3,  7,  7,  5,  2,  9,  7, 10,  8,  7,  7,  3,  8,\n",
      "         3,  5,  6,  7,  7,  6,  5,  6,  7, 11,  2,  8, 10,  7,  8,  7,  6,  5,\n",
      "         4, 10, 10,  4,  3,  7,  8,  6,  5,  7], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 4,  4,  7,  8,  7,  5,  2,  6,  6,  2,  7,  4,  7,  4,  6,  5,  7,  7,\n",
      "         5,  6,  2, 10,  4,  5,  7,  8,  5,  7,  5,  8, 10,  5,  8,  4,  7,  8,\n",
      "         6,  9,  5,  5,  9,  8,  9,  5,  3,  7,  7,  6, 10,  7,  6,  3,  7,  7,\n",
      "         5,  9,  4,  6,  5,  9,  5,  5,  5,  9,  9,  7,  7,  6,  6,  5,  6,  7,\n",
      "         4,  4,  6,  4,  3,  7,  3,  8,  6,  3,  5,  5,  5,  8,  6,  5,  3,  8,\n",
      "        11,  5,  3,  4,  4,  8,  5,  8,  9,  3], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 4,  8,  3,  5,  4,  4,  8,  7,  7,  7,  8,  9,  4,  9,  9,  6,  5,  7,\n",
      "         7,  7,  8,  5,  5,  9,  6,  5,  3,  8,  4,  5,  7,  5,  3, 10,  6,  5,\n",
      "         3,  6,  5,  2,  5, 11,  5,  6,  4,  6,  8,  7,  5,  7,  9,  6,  8,  3,\n",
      "         7,  8,  2,  8,  4,  4,  4,  9,  6,  4,  6,  6,  6,  5,  9,  4,  7,  8,\n",
      "         4,  5,  3,  7,  5,  7,  3,  8,  4,  8,  8,  8,  5,  6,  9,  8,  6,  5,\n",
      "         6, 10,  7,  9,  4,  7,  2,  4,  7, 10], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 5,  3,  7,  6,  6,  7,  8,  3,  2,  6,  3,  4,  3,  8,  3,  4,  4,  4,\n",
      "         8,  5,  6,  3,  4,  8,  4,  7,  6,  9,  7,  9,  9,  9,  5,  6,  7,  4,\n",
      "         5,  8,  3,  6,  9,  3,  8,  6,  9,  5,  5,  6,  7,  1, 10,  4,  8,  6,\n",
      "         5,  6, 10,  4,  9,  8,  4,  7,  4,  5,  4,  5,  9,  5,  7,  9,  7,  6,\n",
      "         5, 10,  9,  8,  3,  6,  2,  4,  8,  6,  7,  7,  5,  3,  4,  6,  8,  9,\n",
      "         5,  8,  6,  6,  6,  5,  5,  4,  6,  4], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 7,  7,  4,  8,  6,  6,  7,  3,  6,  5,  3,  5,  7, 10,  6,  7,  9, 10,\n",
      "         4,  4,  5,  6,  5,  7,  8,  9,  7,  3,  7,  3,  5,  8,  8,  9,  6,  5,\n",
      "         7, 10,  8,  7,  7,  5,  8,  4,  7, 10,  8,  5,  8,  7,  7,  9,  7,  7,\n",
      "         4,  8,  5,  8,  5,  6, 10,  3,  3,  7,  7, 10,  5,  8,  7,  3,  9, 10,\n",
      "         3,  4, 10,  8,  5,  5,  4, 10,  7,  6,  5, 10,  9,  6,  8,  5,  9,  5,\n",
      "         7,  3,  4,  6,  3,  6,  5,  7,  8,  8], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([10,  6,  6,  8,  9,  6,  4,  6,  9,  6,  9,  8,  7,  4,  7, 10,  9,  5,\n",
      "         4,  9,  4,  6,  4,  4,  8,  4,  5,  4,  8, 10,  2,  2,  7,  5,  5,  7,\n",
      "         6,  3, 11,  9,  7, 10,  4,  5,  3,  2,  8,  4,  5,  3,  8,  9,  4,  8,\n",
      "         5, 11,  4,  9,  6,  7,  8,  4,  5,  3,  3, 10,  9,  7,  2,  7,  9, 10,\n",
      "         5,  4,  6,  3,  9,  3,  7,  5,  7,  8,  5,  7,  4,  4,  5,  4,  8,  8,\n",
      "         6,  3,  7,  9, 10,  8, 10,  4,  6,  5], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 9,  5,  5,  8,  9,  4,  5,  8,  5,  3,  8,  6,  9,  1,  6,  6,  7,  6,\n",
      "         4,  5,  6,  4,  8,  4,  8,  7,  7,  8,  9,  6,  9, 10,  7,  8,  3,  3,\n",
      "         8,  5,  6,  5,  8,  3,  9,  7,  7,  8,  5, 10,  4,  3,  4,  7,  6,  8,\n",
      "         4,  3,  3,  1,  4,  8,  9, 10,  6,  4,  8,  2, 10,  7,  4,  2,  6,  4,\n",
      "         8, 10,  4,  8,  2, 10,  8,  8,  3,  3,  9,  4,  8,  7,  6,  3,  3,  6,\n",
      "         9,  7,  9,  7,  5, 10,  8,  7,  3,  8], device='cuda:0')\n",
      "Epoch:  1 , loss:  2.449791431427002 , accuracy:  0.14\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([ 6,  7,  9, 10,  6,  8,  2,  3,  4,  9,  7,  7,  5,  8,  3,  5, 10,  3,\n",
      "         4,  7,  4,  6,  5,  7,  7,  4,  4,  4,  5,  7,  7,  6,  7,  3,  5,  5,\n",
      "         5,  8,  4,  7,  4,  7,  5,  4,  6,  9,  6,  8,  4,  6,  8,  6, 10,  8,\n",
      "         4,  5,  6,  6,  5,  4,  7,  7,  6,  3,  4,  5,  5,  4,  4,  9,  5,  3,\n",
      "         9,  3,  7,  5,  5,  3,  4,  5,  7,  4,  8,  5,  7,  5,  7,  7,  6,  7,\n",
      "         5,  7,  5,  8,  8,  4,  6, 10,  6,  3], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([6, 6, 4, 6, 4, 6, 6, 4, 4, 6, 6, 6, 6, 6, 4, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 4, 6, 4, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 6, 6, 6, 6, 4, 6, 4, 4, 6, 6, 4, 4, 6, 4, 4, 6, 6, 6, 6,\n",
      "        6, 6, 6, 6, 6, 4, 4, 6, 6, 6, 4, 6, 6, 6, 6, 6, 6, 4, 6, 6, 6, 4, 6, 4,\n",
      "        6, 6, 6, 6], device='cuda:0') tensor([11,  7,  2,  9,  3,  8,  8,  4,  3,  9,  6,  6,  6,  8,  5,  7,  3,  5,\n",
      "         9,  8,  8,  5,  5,  8,  7,  8,  7,  9,  6,  4,  8, 10,  3,  5,  4,  9,\n",
      "         9,  6,  3,  7,  9,  7,  9,  9,  7,  7,  5,  8,  4,  7,  7,  9,  7,  7,\n",
      "         7,  9,  9,  2,  9,  3,  4, 10,  5,  5,  5,  8,  5,  3,  6,  4,  9,  7,\n",
      "         7,  7,  8, 10,  4,  5,  2,  9,  3,  8,  4,  7,  8,  8,  8,  6,  6,  3,\n",
      "         4,  5,  8,  5,  9,  6,  8,  7,  9,  4], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([4, 4, 6, 4, 4, 6, 4, 4, 6, 6, 4, 4, 4, 6, 6, 4, 4, 6, 4, 4, 6, 4, 6, 4,\n",
      "        4, 6, 6, 4, 4, 6, 4, 6, 4, 4, 6, 4, 6, 4, 6, 4, 4, 4, 4, 4, 4, 4, 6, 4,\n",
      "        4, 6, 6, 4, 4, 4, 4, 6, 4, 4, 4, 4, 6, 4, 4, 6, 4, 4, 4, 4, 6, 6, 4, 4,\n",
      "        4, 4, 4, 4, 6, 4, 4, 4, 6, 6, 4, 6, 4, 4, 4, 6, 6, 4, 6, 4, 6, 4, 4, 4,\n",
      "        4, 4, 6, 6], device='cuda:0') tensor([ 4,  6,  8,  5,  4, 10,  8,  5,  6,  8,  6,  8,  6,  6,  8,  5,  4,  8,\n",
      "         6,  3,  7,  5,  9,  4,  5,  7,  5,  6,  3,  6,  3,  8,  5,  3,  9,  4,\n",
      "         8,  5, 10,  5,  9,  5,  3,  7,  6,  4,  8,  6,  7, 11,  5,  3,  5,  7,\n",
      "         9,  7,  5,  5,  3,  5,  9,  4,  3, 10,  2,  7,  8,  9, 10,  9,  6,  6,\n",
      "         6,  4,  3,  1,  7,  3,  4,  7,  8,  8,  7,  9,  6,  5,  4,  4,  9,  7,\n",
      "         7,  6,  8,  7,  5,  5,  3,  5,  9, 10], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([4, 5, 5, 5, 5, 4, 5, 5, 5, 4, 5, 6, 5, 5, 5, 5, 5, 6, 4, 5, 5, 6, 5, 4,\n",
      "        5, 5, 4, 5, 5, 6, 5, 5, 4, 5, 5, 6, 5, 6, 5, 4, 5, 5, 4, 5, 5, 5, 4, 6,\n",
      "        4, 4, 5, 5, 5, 5, 5, 5, 4, 4, 5, 4, 5, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 5, 5, 4, 5, 5, 4, 5, 5, 5, 5, 6, 5, 4, 4,\n",
      "        5, 4, 4, 5], device='cuda:0') tensor([ 3,  5,  9,  5, 10,  4,  9,  6,  7,  5,  9,  8,  5,  8,  8,  5,  6,  5,\n",
      "         3,  8,  6,  5,  9,  2,  5,  7,  6,  7,  6,  3,  6,  7,  3, 10,  8,  7,\n",
      "        10,  3,  7,  5,  8,  7,  5,  8,  9,  7,  5,  8,  1,  4,  6,  8,  2, 11,\n",
      "         7,  8,  3,  4,  7,  3,  5,  9,  7,  4,  4,  8,  4,  9,  6,  8,  4,  5,\n",
      "         4,  5,  8,  4,  9,  5,  5,  6,  3,  4,  6,  9,  4,  7,  6,  5,  8,  4,\n",
      "         7,  4,  6, 11,  4,  4,  4,  4,  4,  4], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([ 4,  7,  4,  2, 10,  7,  4,  7,  3,  7,  5,  9,  5,  8,  6,  7,  6, 10,\n",
      "         6,  6,  4,  4,  3,  6,  3, 10,  8,  7,  9,  5,  6,  7,  4,  5,  7,  8,\n",
      "         6,  3,  7,  9,  8,  5,  5,  3,  4,  5, 10,  4,  6,  4,  7,  6,  4,  5,\n",
      "         5,  1,  7,  4,  3,  8,  6,  6,  2,  5,  4,  7,  8,  7,  9,  8,  7, 10,\n",
      "         3,  7,  8,  7,  9,  9,  3, 10,  4,  6,  8,  5,  6,  7,  8,  4,  9,  4,\n",
      "         4,  2,  8,  5,  5,  9,  7,  4,  6,  6], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([ 8,  8,  6,  6,  7,  9,  8,  7,  7,  7,  7,  8,  5,  6,  5,  8,  6,  6,\n",
      "         6,  6,  6,  7,  7,  4,  8,  3,  5,  6,  9,  3,  4,  5,  5,  4,  2,  7,\n",
      "         6,  5,  4,  9,  8,  8,  5,  7,  8,  5,  4, 10,  5,  5,  7,  9,  4,  6,\n",
      "         7,  9,  4,  6,  5,  7,  6,  3,  4,  6,  9,  4,  5,  5,  8,  3,  5,  5,\n",
      "         8,  4,  9,  3,  6,  3,  8,  4,  7,  4,  7,  8,  9,  9,  9,  4,  9,  2,\n",
      "        10,  6,  8,  4,  3,  6, 10,  8,  4,  6], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([ 5,  2,  5,  8,  8,  5,  8,  4,  5,  4,  9,  9,  8,  7,  3,  4,  5,  4,\n",
      "         7,  4,  7,  4,  8,  9,  7,  3,  7,  9,  3,  8,  7,  7,  8,  8,  8,  7,\n",
      "         9,  6,  9,  6,  3,  7,  6,  8,  6,  7,  6,  5,  8, 10,  8,  6,  9,  5,\n",
      "         7,  7,  7,  9,  6,  3,  7,  6,  5,  7,  9,  7,  3,  7,  8,  6,  6,  4,\n",
      "         8,  7,  8,  3,  7,  4,  3,  3,  9,  7,  4,  6,  6,  4,  7,  4,  4,  8,\n",
      "         3,  6,  7,  3,  6,  7,  5,  6,  8, 10], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([10,  5,  5,  4,  5,  7,  5,  8,  5,  2,  9,  3,  5,  4,  7,  2, 10,  2,\n",
      "         4,  9,  4,  3,  5,  5,  6,  7, 11,  7,  8,  7,  8,  8,  9,  3,  4,  6,\n",
      "         6,  3,  9,  4,  3, 10,  3,  6,  9,  3,  2,  8, 10,  6,  5,  9,  6,  3,\n",
      "         5,  7,  6, 11,  5,  3,  5,  7,  3,  5,  4,  4,  8,  5,  7,  4, 10,  4,\n",
      "         6,  9,  7,  9,  6,  2,  8,  5,  7,  7,  7, 11,  8,  2,  9,  5,  7,  2,\n",
      "         4,  7, 11, 11,  5,  6,  7,  4,  9,  4], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([ 3,  7,  5,  9,  5,  4,  3,  2,  3,  5,  6,  7,  8,  8,  5, 10,  7,  7,\n",
      "         7,  7,  6,  7,  3,  7,  7,  5,  3,  6,  6,  7,  4,  5,  5, 10,  6,  6,\n",
      "        10,  8,  5,  4,  6,  8, 10,  3,  7,  3,  8,  7,  3,  5,  9,  9,  6,  5,\n",
      "         4, 10,  6, 10,  9,  3,  7,  7,  6,  5,  3,  7, 10,  4,  6,  9,  9,  6,\n",
      "         7,  8,  5,  5, 10,  8,  7, 11,  7,  4,  8,  6,  6,  4,  8,  8,  5,  9,\n",
      "         6,  5,  7,  6,  6,  8, 10,  6,  7,  4], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([ 4,  2,  8,  3, 10,  7,  5, 10,  7,  8, 10,  5,  6,  6,  4,  7,  1,  6,\n",
      "         5, 10,  7,  2,  6,  7,  9,  9,  9,  7,  9,  4,  3,  4,  7,  9,  9, 10,\n",
      "         5,  4,  7,  5,  3,  8,  5,  3,  5,  3,  2,  6,  7,  3,  8,  5,  4,  4,\n",
      "         9,  5,  6,  3,  9,  6,  8,  5,  9,  8,  4,  7,  3,  6,  5,  7,  3,  2,\n",
      "         6,  6,  9,  4,  6,  4,  5,  6,  4,  4,  3,  2,  4,  6,  4,  4,  4,  4,\n",
      "         9,  7,  5,  4,  7,  6,  7,  8,  8,  7], device='cuda:0')\n",
      "Epoch:  2 , loss:  2.38297176361084 , accuracy:  0.141\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([ 8,  9,  5,  4,  7,  7,  7,  7,  7,  4,  8,  7,  5,  2,  6,  3,  7,  9,\n",
      "         6,  5,  7,  4,  4,  5,  8,  8,  4,  3,  5,  8,  7,  4,  7,  8,  9,  2,\n",
      "         3,  9,  4,  4, 11,  9,  4,  4,  3,  9,  7,  7,  3,  7,  7,  8,  6,  8,\n",
      "         4,  5,  5,  2,  7,  9,  5,  7,  6,  3,  7,  7,  8,  7,  5,  8,  6,  3,\n",
      "         4,  3,  7, 11,  7,  7,  5,  6,  7,  5,  6,  3,  6,  8,  7,  3,  7,  7,\n",
      "         6,  7, 10,  7,  6,  9,  5,  9,  7,  5], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n",
      "        5, 5, 5, 5], device='cuda:0') tensor([ 3,  4,  4,  7,  4,  7,  7,  5,  8,  5,  5,  8,  9,  8,  5,  4,  7,  6,\n",
      "         6,  7,  4,  7,  5,  5,  5, 11,  3,  6,  8,  8,  8,  9,  4,  7,  3,  5,\n",
      "         4,  9,  4,  6,  9, 10,  1,  5,  4, 10,  8,  3,  8,  7,  5,  9,  6,  7,\n",
      "         4,  7,  9,  8,  5,  6,  8,  8,  3,  9,  9,  2,  5,  3,  5,  7,  6,  9,\n",
      "         3,  3,  6,  7,  6,  4,  9,  4,  2,  4,  4,  7,  9,  7,  9,  3,  3,  5,\n",
      "         7,  5,  6,  6,  5,  4,  7,  7,  4,  5], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([4, 4, 4, 5, 5, 5, 4, 4, 4, 4, 4, 5, 5, 4, 5, 4, 4, 5, 5, 4, 4, 5, 4, 4,\n",
      "        5, 4, 5, 4, 5, 5, 5, 4, 4, 4, 4, 5, 4, 5, 5, 5, 4, 4, 5, 5, 4, 5, 4, 4,\n",
      "        4, 5, 5, 5, 4, 4, 5, 4, 4, 5, 4, 4, 5, 4, 4, 4, 5, 5, 4, 5, 5, 5, 4, 4,\n",
      "        4, 5, 5, 4, 5, 5, 5, 5, 5, 5, 5, 4, 5, 4, 4, 5, 5, 4, 5, 4, 5, 4, 5, 4,\n",
      "        5, 4, 5, 5], device='cuda:0') tensor([ 6,  4,  5, 10,  7, 10,  6,  4,  5,  6,  3,  2,  7,  6,  6,  6,  5,  5,\n",
      "         6,  6,  4,  5,  8,  6,  4,  2,  8,  6,  8,  8,  9,  4,  3,  6,  2,  6,\n",
      "         4,  6,  7,  6,  5,  4,  7,  7,  3,  9,  5,  4,  4,  8,  5, 10,  4,  4,\n",
      "         6,  4,  5,  8,  6,  5,  6,  6,  6,  6,  5,  6,  3,  6,  7,  9,  4,  4,\n",
      "         2,  7,  5,  3,  7,  6,  6,  5,  5,  9, 10,  5,  6,  3,  5,  5, 10,  6,\n",
      "         7,  7,  8,  3,  7,  6,  9,  7,  9,  8], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([4, 5, 5, 4, 4, 5, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 5, 4, 5, 4, 5, 4, 4, 5,\n",
      "        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4,\n",
      "        4, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5,\n",
      "        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 5, 4, 4,\n",
      "        4, 4, 4, 5], device='cuda:0') tensor([ 6,  9,  9,  7,  4, 10,  8,  4,  4,  6,  4, 11,  9,  7,  2,  2,  5,  7,\n",
      "         7,  4,  5,  4,  4,  9,  7,  5,  6,  5,  7,  3,  5,  3,  3,  5,  6, 10,\n",
      "         8,  5,  5,  6,  5, 10,  4,  3,  6,  9,  4,  4,  4,  3,  3,  4,  6,  5,\n",
      "         5,  5,  5,  7,  9,  6,  9,  9,  9,  5,  4,  3,  3,  7,  7,  9,  3,  9,\n",
      "         5,  8,  6,  3,  4,  4,  3,  7,  6,  7,  3,  8,  4,  3,  6,  8,  5,  9,\n",
      "         7,  7,  2,  7,  8,  6,  4,  9,  4, 10], device='cuda:0')\n",
      "torch.Size([100]) torch.Size([100]) tensor([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n",
      "        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4,\n",
      "        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n",
      "        4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n",
      "        4, 4, 4, 4], device='cuda:0') tensor([ 4,  4,  4,  6,  4,  4,  8,  6,  7,  6,  9,  5,  8,  7,  8, 10,  4,  9,\n",
      "         4,  7,  5,  8,  6,  5, 10,  7,  9,  3,  2,  4,  5,  5,  4, 10,  3,  4,\n",
      "         3,  8,  5,  9, 10,  8,  8,  6,  8,  5,  7,  5,  8,  5,  7,  7,  9,  5,\n",
      "         3,  5, 10,  4,  6,  6,  7,  9,  3, 10,  6,  3,  6,  7,  5,  5,  8,  8,\n",
      "        10,  6,  5,  7,  6,  6,  4,  9,  2,  8,  9,  4,  9,  7,  5,  4,  8,  4,\n",
      "         4,  5,  8,  8,  2,  7,  8,  9,  4,  9], device='cuda:0')\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-31fb8df39f0f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mcorrect\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                 \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.train()\n",
    "loss_v = []\n",
    "acc_v = []\n",
    "ep = 0\n",
    "for epoch in range(500):\n",
    "    ep += 1\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch in train_loader:\n",
    "#         print(batch.x)\n",
    "        optimizer.zero_grad()\n",
    "        data = batch.to(device)\n",
    "        out = model(data)\n",
    "        _, pred = out.max(dim=1)\n",
    "        print(pred.shape, data.y.shape, pred, data.y)\n",
    "        loss = F.cross_entropy(out, data.y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        correct += float(pred.eq(data.y).sum().item())\n",
    "#         print(correct, pred, data.y)\n",
    "        total += len(pred)\n",
    "#         print(out, data.y, )\n",
    "    acc = correct/total\n",
    "    print(\"Epoch: \" , ep, \", loss: \", loss.item(), \", accuracy: \", acc)\n",
    "    loss_v.append(loss)\n",
    "    acc_v.append(acc)\n",
    "plt.plot(np.arange(len(loss_v)), loss_v)\n",
    "plt.plot(np.arange(len(acc_v)), acc_v)\n",
    "plt.ylim(0.1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "hidden": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch(batch=[6110], edge_index=[2, 16297], x=[6110, 2], y=[100])\n",
      "tensor([ 7,  7,  7,  4,  6,  4,  6,  9,  7,  9,  8,  2,  5,  6,  5,  6,  7,  7,\n",
      "         9,  7,  4,  7,  9,  9,  5,  8,  5,  5,  4,  9,  7,  4,  9,  3,  9,  4,\n",
      "         6,  5,  4,  6,  4,  7,  3,  5,  5,  4,  5,  5, 10,  5,  2,  7,  9,  6,\n",
      "        10,  4,  3,  9,  4,  9,  4,  8,  6,  9,  9,  5,  1,  9,  7,  9,  5,  4,\n",
      "         9,  6,  4,  9,  4,  4,  3,  7,  7,  9,  4,  4,  3,  7,  7,  4,  6,  9,\n",
      "         7,  9, 10,  7,  9,  4, 10,  5,  7,  6], device='cuda:0') tensor([ 7,  7,  8,  4,  6,  4,  5, 11,  8, 10,  8,  3,  5,  6,  4,  6,  7,  6,\n",
      "         8,  7,  4,  7,  9,  8,  5,  8,  5,  5,  2, 10,  7,  5,  9,  2,  8,  4,\n",
      "         6,  5,  4,  6,  4,  7,  3,  5,  6,  5,  5,  6, 10,  5,  2,  6,  7,  6,\n",
      "        11,  4,  3,  9,  3,  9,  4,  8,  6,  8,  6,  6,  2,  8,  7,  8,  5,  4,\n",
      "         6,  5,  3,  9,  4,  3,  3,  6,  8, 10,  5,  4,  3,  7,  9,  4,  6,  5,\n",
      "         7,  9,  9,  7, 10,  4, 11,  5,  6,  6], device='cuda:0')\n",
      "Accuracy: 0.5800\n",
      "Batch(batch=[6030], edge_index=[2, 15800], x=[6030, 2], y=[100])\n",
      "tensor([ 4,  6,  8,  3,  8,  6, 10,  6,  7,  9,  3,  6,  7,  6,  5,  8,  4,  4,\n",
      "         4,  6,  4,  6,  7,  5,  6, 10,  8,  5,  9,  2,  5,  6,  4,  9,  7,  9,\n",
      "         9,  8,  6,  9,  4,  7,  3,  4,  4,  4,  6,  4,  7,  9,  3,  4,  9,  4,\n",
      "         6,  8,  4,  7,  9,  5,  5,  8,  3,  7,  5,  7,  4,  9,  5,  3,  7,  8,\n",
      "         9,  4,  9,  8,  9,  4,  3,  2,  7,  8,  7,  6,  5,  7,  7,  5,  5,  7,\n",
      "         7,  7,  6,  4,  7,  9,  7,  7,  5,  2], device='cuda:0') tensor([ 4,  5,  8,  2,  6,  6, 10,  6,  8,  6,  2,  6,  8,  5,  6,  7,  4,  4,\n",
      "         4,  5,  4,  6,  7,  5,  6,  9,  7,  5, 10,  2,  5,  6,  3,  9,  6,  8,\n",
      "        10,  8,  6,  9,  4,  8,  3,  4,  5,  4,  5,  4,  7,  8,  3,  3,  8,  3,\n",
      "         6,  8,  4,  7,  9,  5,  5,  8,  3,  8,  6,  7,  4,  8,  4,  3,  7,  9,\n",
      "         9,  4,  9,  8,  8,  5,  4,  2,  7,  8,  6,  6,  6,  9,  7,  3,  6,  8,\n",
      "         8,  8,  6,  4,  8,  9,  6,  9,  4,  3], device='cuda:0')\n",
      "Accuracy: 0.5400\n",
      "Batch(batch=[6180], edge_index=[2, 16398], x=[6180, 2], y=[100])\n",
      "tensor([ 7,  8,  5,  4,  9,  9,  9,  4,  9,  9,  6,  7,  5,  9,  3, 10,  3,  8,\n",
      "         4,  3,  3,  1,  5,  5,  4,  7,  9,  6,  6,  9,  2,  5,  4,  5,  5,  7,\n",
      "         5,  4,  6,  7,  9,  8,  4,  7,  4,  5,  6,  5,  6,  9,  7,  2,  7,  4,\n",
      "         2,  9,  6,  9,  3,  4, 10,  9,  5,  9,  6,  5,  7,  7,  6,  9,  5,  6,\n",
      "         5,  5,  5, 10,  8,  9,  6,  6,  7, 10,  7,  9,  4,  3,  5,  7,  5,  9,\n",
      "         4,  9,  9,  9, 10,  4,  5,  5,  7,  7], device='cuda:0') tensor([ 6,  7,  5,  4,  7,  9, 10,  3,  9,  8,  6,  6,  5,  7,  3, 11,  3,  7,\n",
      "         4,  3,  3,  1,  4,  5,  4,  7,  7,  6,  6, 10,  5,  5,  4,  5,  5,  7,\n",
      "         5,  3,  5,  9,  9,  9,  4,  7,  4,  6,  6,  6,  6,  8,  6,  2,  6,  4,\n",
      "         2,  8,  5,  9,  3,  4,  8,  9,  6,  7,  7,  6,  7,  7,  6, 10,  6,  6,\n",
      "         5,  7,  5, 10,  8,  9,  6,  6,  7, 10,  7,  8,  5,  3,  5,  7,  6,  9,\n",
      "         4,  9,  8,  9,  9,  4,  4,  6,  7,  7], device='cuda:0')\n",
      "Accuracy: 0.6000\n",
      "Batch(batch=[6120], edge_index=[2, 16169], x=[6120, 2], y=[100])\n",
      "tensor([ 7,  9,  3,  4,  6,  2,  4,  9,  5,  7,  8, 10,  9,  7,  6,  6,  9,  9,\n",
      "         6,  5,  4,  5,  5,  4,  9,  9,  3,  7,  4,  4,  4,  8,  7,  9,  1,  4,\n",
      "         9, 10,  7,  9, 10,  7,  9,  5,  7,  9,  4,  4,  5,  4,  9,  7,  4,  7,\n",
      "         5,  8,  9,  9,  4,  6,  9,  7,  7,  7,  6,  9,  5,  7,  4,  7,  5,  5,\n",
      "         4,  4, 10,  5,  9,  8,  9,  9,  2,  9,  4,  5,  3,  6,  6,  9,  6,  9,\n",
      "         7,  9,  7,  6,  4,  7,  6,  5,  7,  6], device='cuda:0') tensor([ 7,  8,  2,  4,  7,  2,  4,  9,  6,  7,  7, 10,  8,  6,  6,  5,  8,  8,\n",
      "         5,  5,  4,  5,  5,  5, 10,  9,  4,  8,  4,  5,  4,  8,  6,  8,  1,  4,\n",
      "         8,  8,  6,  9, 11,  6,  8,  5,  6,  8,  3,  3,  3,  3,  9,  7,  3,  7,\n",
      "         3,  8,  8,  8,  4,  6,  9,  7,  7,  7,  6,  8,  5,  7,  4,  8,  5,  5,\n",
      "         5,  4,  8,  5,  9,  9,  8,  7,  2,  8,  3,  5,  3,  6,  6,  8,  6,  9,\n",
      "         7,  8,  6,  5,  4,  6,  6,  5,  6,  6], device='cuda:0')\n",
      "Accuracy: 0.5100\n",
      "Batch(batch=[6080], edge_index=[2, 15661], x=[6080, 2], y=[100])\n",
      "tensor([ 7,  4,  6,  7,  6,  4,  4,  7,  7,  3,  7,  4,  9,  9, 10,  6,  9,  5,\n",
      "         7,  8,  9,  9,  7, 10,  4,  7,  9,  7,  5,  6,  6,  5,  9,  9,  4,  9,\n",
      "         4, 10,  5,  5,  4,  5,  7,  6,  4,  7,  9,  7,  7,  8,  5,  5,  2,  4,\n",
      "         7,  9,  3,  6,  9,  9,  4,  3,  7,  6,  4,  4,  4,  7,  5,  4, 10,  5,\n",
      "         7,  5,  4,  5,  8,  9,  6,  5,  5, 10,  5,  7,  1,  4,  4,  7,  5,  6,\n",
      "         7,  9,  7,  5, 10,  9,  6,  9,  9,  6], device='cuda:0') tensor([ 8,  4,  5,  8,  5,  4,  3,  6,  7,  3,  7,  4,  9,  9, 10,  7,  9,  6,\n",
      "         7,  8,  8,  9,  6, 10,  4,  6,  3,  8,  5,  6,  5,  5,  8,  9,  4,  9,\n",
      "         4,  9,  5,  6,  4,  5,  6,  5,  3,  8,  9,  7,  7,  9,  5,  5,  3,  3,\n",
      "         8,  8,  2,  5,  8,  8,  4,  2,  6,  6,  3,  5,  3,  7,  5,  5,  9,  5,\n",
      "         6,  5,  4,  5,  8,  9,  7,  6,  3,  9,  5,  7,  1,  4,  5,  7,  5,  6,\n",
      "         6,  7,  6,  5, 10,  8,  6,  8,  8,  6], device='cuda:0')\n",
      "Accuracy: 0.5100\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "for batch in test_loader:\n",
    "    print(batch)\n",
    "    data = batch.to(device)\n",
    "    _, pred = model(data).max(dim=1)\n",
    "    print(pred, data.y)\n",
    "    correct = float(pred.eq(data.y).sum().item())\n",
    "    acc = correct / len(pred)\n",
    "    print('Accuracy: {:.4f}'.format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Combined Counter & Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_dataset = [gen_edge_graph_class(event_size_min, event_size_max, max_curve, min_curve, height, num_layers, max_angle) for i in range(1000)]\n",
    "test_dataset = [gen_edge_graph_class(event_size_min, event_size_max, max_curve, min_curve, height, num_layers, max_angle) for i in range(500)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset[0].y_graph.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True)\n",
    "test_loader = DataLoader(test_dataset, batch_size=100, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using  cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(\"Using \", device)\n",
    "# model = Edge_Class_Net( input_dim=2, hidden_dim=64, n_graph_iters=4).to(device)\n",
    "model = Edge_Graph_Class_Net(input_dim=2, hidden_dim=16, n_graph_iters=4, output_dim=12).to(device)\n",
    "# data = dataset[0].to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "hidden": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  1 , loss:  3.1576285362243652 , count accuracy:  0.122 , edge accuracy:  0.6513064133016627\n",
      "Epoch:  2 , loss:  3.1742045879364014 , count accuracy:  0.134 , edge accuracy:  0.6614945111382166\n",
      "Epoch:  3 , loss:  3.0324511528015137 , count accuracy:  0.134 , edge accuracy:  0.6614945111382166\n",
      "Epoch:  4 , loss:  3.011707305908203 , count accuracy:  0.158 , edge accuracy:  0.6614945111382166\n",
      "Epoch:  5 , loss:  3.0208494663238525 , count accuracy:  0.196 , edge accuracy:  0.6620851255055531\n",
      "Epoch:  6 , loss:  3.003718137741089 , count accuracy:  0.182 , edge accuracy:  0.6734865506837003\n",
      "Epoch:  7 , loss:  2.8945674896240234 , count accuracy:  0.193 , edge accuracy:  0.6919881877126532\n",
      "Epoch:  8 , loss:  2.9283816814422607 , count accuracy:  0.194 , edge accuracy:  0.7084547730628491\n",
      "Epoch:  9 , loss:  2.8948309421539307 , count accuracy:  0.204 , edge accuracy:  0.7297875072221865\n",
      "Epoch:  10 , loss:  2.914076328277588 , count accuracy:  0.208 , edge accuracy:  0.746857546382487\n",
      "Epoch:  11 , loss:  2.802792549133301 , count accuracy:  0.216 , edge accuracy:  0.7610258714771779\n",
      "Epoch:  12 , loss:  2.8270790576934814 , count accuracy:  0.247 , edge accuracy:  0.7690762020928291\n",
      "Epoch:  13 , loss:  2.7655274868011475 , count accuracy:  0.247 , edge accuracy:  0.7779418373242601\n",
      "Epoch:  14 , loss:  2.808720111846924 , count accuracy:  0.254 , edge accuracy:  0.7958464402644925\n",
      "Epoch:  15 , loss:  2.6588196754455566 , count accuracy:  0.259 , edge accuracy:  0.8237080310714515\n",
      "Epoch:  16 , loss:  2.656951427459717 , count accuracy:  0.266 , edge accuracy:  0.8548436797842973\n",
      "Epoch:  17 , loss:  2.6260929107666016 , count accuracy:  0.273 , edge accuracy:  0.8773512229569237\n",
      "Epoch:  18 , loss:  2.5608267784118652 , count accuracy:  0.272 , edge accuracy:  0.8878089490916095\n",
      "Epoch:  19 , loss:  2.58125376701355 , count accuracy:  0.273 , edge accuracy:  0.9003723438402773\n",
      "Epoch:  20 , loss:  2.567147731781006 , count accuracy:  0.273 , edge accuracy:  0.9057199717532259\n",
      "Epoch:  21 , loss:  2.507758617401123 , count accuracy:  0.276 , edge accuracy:  0.9089234127238878\n",
      "Epoch:  22 , loss:  2.519458770751953 , count accuracy:  0.291 , edge accuracy:  0.9125954933555883\n",
      "Epoch:  23 , loss:  2.466383457183838 , count accuracy:  0.287 , edge accuracy:  0.9154458496501252\n",
      "Epoch:  24 , loss:  2.516859531402588 , count accuracy:  0.307 , edge accuracy:  0.9169865827823073\n",
      "Epoch:  25 , loss:  2.4069888591766357 , count accuracy:  0.288 , edge accuracy:  0.9167233742055595\n",
      "Epoch:  26 , loss:  2.520430326461792 , count accuracy:  0.288 , edge accuracy:  0.9171534955382936\n",
      "Epoch:  27 , loss:  2.5352001190185547 , count accuracy:  0.297 , edge accuracy:  0.9195673107787122\n",
      "Epoch:  28 , loss:  2.4262051582336426 , count accuracy:  0.31 , edge accuracy:  0.9197855813057714\n",
      "Epoch:  29 , loss:  2.428661584854126 , count accuracy:  0.31 , edge accuracy:  0.9215189060794762\n",
      "Epoch:  30 , loss:  2.467913866043091 , count accuracy:  0.292 , edge accuracy:  0.9207164409064647\n",
      "Epoch:  31 , loss:  2.4559147357940674 , count accuracy:  0.291 , edge accuracy:  0.9211786608461193\n",
      "Epoch:  32 , loss:  2.435981273651123 , count accuracy:  0.323 , edge accuracy:  0.9214739680297875\n",
      "Epoch:  33 , loss:  2.349578380584717 , count accuracy:  0.324 , edge accuracy:  0.922417667073249\n",
      "Epoch:  34 , loss:  2.4482226371765137 , count accuracy:  0.319 , edge accuracy:  0.9236630930217629\n",
      "Epoch:  35 , loss:  2.389862537384033 , count accuracy:  0.335 , edge accuracy:  0.9241317326828016\n",
      "Epoch:  36 , loss:  2.457862377166748 , count accuracy:  0.353 , edge accuracy:  0.924439879309238\n",
      "Epoch:  37 , loss:  2.384885787963867 , count accuracy:  0.352 , edge accuracy:  0.9241381524041856\n",
      "Epoch:  38 , loss:  2.327662944793701 , count accuracy:  0.335 , edge accuracy:  0.923887783270206\n",
      "Epoch:  39 , loss:  2.3603055477142334 , count accuracy:  0.328 , edge accuracy:  0.9231880336393401\n",
      "Epoch:  40 , loss:  2.4143574237823486 , count accuracy:  0.323 , edge accuracy:  0.9241830904538743\n",
      "Epoch:  41 , loss:  2.3928160667419434 , count accuracy:  0.314 , edge accuracy:  0.923605315529306\n",
      "Epoch:  42 , loss:  2.3331708908081055 , count accuracy:  0.341 , edge accuracy:  0.9241959298966425\n",
      "Epoch:  43 , loss:  2.352027416229248 , count accuracy:  0.359 , edge accuracy:  0.9256467869294472\n",
      "Epoch:  44 , loss:  2.3379623889923096 , count accuracy:  0.359 , edge accuracy:  0.927444308916993\n",
      "Epoch:  45 , loss:  2.403022289276123 , count accuracy:  0.374 , edge accuracy:  0.9267573987288952\n",
      "Epoch:  46 , loss:  2.396306037902832 , count accuracy:  0.378 , edge accuracy:  0.9265583873659883\n",
      "Epoch:  47 , loss:  2.395338535308838 , count accuracy:  0.38 , edge accuracy:  0.9251460486614881\n",
      "Epoch:  48 , loss:  2.359173536300659 , count accuracy:  0.348 , edge accuracy:  0.9255376516659177\n",
      "Epoch:  49 , loss:  2.243870258331299 , count accuracy:  0.366 , edge accuracy:  0.925345060024395\n",
      "Epoch:  50 , loss:  2.382564067840576 , count accuracy:  0.378 , edge accuracy:  0.9261346857546382\n",
      "Epoch:  51 , loss:  2.3130781650543213 , count accuracy:  0.388 , edge accuracy:  0.9288181292931886\n",
      "Epoch:  52 , loss:  2.3007421493530273 , count accuracy:  0.395 , edge accuracy:  0.9275406047377543\n",
      "Epoch:  53 , loss:  2.38906192779541 , count accuracy:  0.381 , edge accuracy:  0.9281247993837067\n",
      "Epoch:  54 , loss:  2.4518377780914307 , count accuracy:  0.375 , edge accuracy:  0.9269628298131861\n",
      "Epoch:  55 , loss:  2.1836540699005127 , count accuracy:  0.387 , edge accuracy:  0.9280349232843295\n",
      "Epoch:  56 , loss:  2.303584337234497 , count accuracy:  0.393 , edge accuracy:  0.9277973935931181\n",
      "Epoch:  57 , loss:  2.3001396656036377 , count accuracy:  0.389 , edge accuracy:  0.9290235603774796\n",
      "Epoch:  58 , loss:  2.2036349773406982 , count accuracy:  0.398 , edge accuracy:  0.9304551582461321\n",
      "Epoch:  59 , loss:  2.3271288871765137 , count accuracy:  0.393 , edge accuracy:  0.9297554086152661\n",
      "Epoch:  60 , loss:  2.3661508560180664 , count accuracy:  0.376 , edge accuracy:  0.9301341721769275\n",
      "Epoch:  61 , loss:  2.4601023197174072 , count accuracy:  0.33 , edge accuracy:  0.9298838030429479\n",
      "Epoch:  62 , loss:  2.3030624389648438 , count accuracy:  0.322 , edge accuracy:  0.9283880079604545\n",
      "Epoch:  63 , loss:  2.2585561275482178 , count accuracy:  0.346 , edge accuracy:  0.9281183796623227\n",
      "Epoch:  64 , loss:  2.3640875816345215 , count accuracy:  0.367 , edge accuracy:  0.9275983822302112\n",
      "Epoch:  65 , loss:  2.283921241760254 , count accuracy:  0.363 , edge accuracy:  0.9294151633819092\n",
      "Epoch:  66 , loss:  2.2880241870880127 , count accuracy:  0.399 , edge accuracy:  0.9302433074404571\n",
      "Epoch:  67 , loss:  2.1699166297912598 , count accuracy:  0.393 , edge accuracy:  0.9289401039994865\n",
      "Epoch:  68 , loss:  2.3117504119873047 , count accuracy:  0.404 , edge accuracy:  0.9309687359568595\n",
      "Epoch:  69 , loss:  2.1328814029693604 , count accuracy:  0.402 , edge accuracy:  0.9310842909417731\n",
      "Epoch:  70 , loss:  2.395967483520508 , count accuracy:  0.407 , edge accuracy:  0.9302176285549207\n",
      "Epoch:  71 , loss:  2.3635895252227783 , count accuracy:  0.381 , edge accuracy:  0.930378121589523\n",
      "Epoch:  72 , loss:  2.29846453666687 , count accuracy:  0.371 , edge accuracy:  0.9311549078769982\n",
      "Epoch:  73 , loss:  2.1668996810913086 , count accuracy:  0.382 , edge accuracy:  0.932047249149387\n",
      "Epoch:  74 , loss:  2.2934577465057373 , count accuracy:  0.398 , edge accuracy:  0.9325800860242666\n",
      "Epoch:  75 , loss:  2.3736393451690674 , count accuracy:  0.395 , edge accuracy:  0.9321949027412211\n",
      "Epoch:  76 , loss:  2.371438980102539 , count accuracy:  0.382 , edge accuracy:  0.9291326956410092\n",
      "Epoch:  77 , loss:  2.2368662357330322 , count accuracy:  0.374 , edge accuracy:  0.9309430570713231\n",
      "Epoch:  78 , loss:  2.23089337348938 , count accuracy:  0.367 , edge accuracy:  0.931077871220389\n",
      "Epoch:  79 , loss:  2.2200896739959717 , count accuracy:  0.407 , edge accuracy:  0.930654169609039\n",
      "Epoch:  80 , loss:  2.272797107696533 , count accuracy:  0.41 , edge accuracy:  0.9314694742248186\n",
      "Epoch:  81 , loss:  2.177095651626587 , count accuracy:  0.41 , edge accuracy:  0.9321499646915324\n",
      "Epoch:  82 , loss:  2.1739630699157715 , count accuracy:  0.406 , edge accuracy:  0.9331257623419144\n",
      "Epoch:  83 , loss:  2.115454912185669 , count accuracy:  0.4 , edge accuracy:  0.9315272517172755\n",
      "Epoch:  84 , loss:  2.3608763217926025 , count accuracy:  0.397 , edge accuracy:  0.9334210695255826\n",
      "Epoch:  85 , loss:  2.078286647796631 , count accuracy:  0.402 , edge accuracy:  0.9339731655646145\n",
      "Epoch:  86 , loss:  2.271789312362671 , count accuracy:  0.407 , edge accuracy:  0.9319445336072415\n",
      "Epoch:  87 , loss:  2.3249642848968506 , count accuracy:  0.393 , edge accuracy:  0.9322270013481415\n",
      "Epoch:  88 , loss:  2.2421343326568604 , count accuracy:  0.397 , edge accuracy:  0.9326763818450279\n",
      "Epoch:  89 , loss:  2.205183744430542 , count accuracy:  0.405 , edge accuracy:  0.9325800860242666\n",
      "Epoch:  90 , loss:  2.22875714302063 , count accuracy:  0.408 , edge accuracy:  0.9313346600757527\n",
      "Epoch:  91 , loss:  2.149848699569702 , count accuracy:  0.405 , edge accuracy:  0.9319573730500096\n",
      "Epoch:  92 , loss:  2.1380088329315186 , count accuracy:  0.402 , edge accuracy:  0.9338897091866213\n",
      "Epoch:  93 , loss:  2.304644823074341 , count accuracy:  0.402 , edge accuracy:  0.9320729280349233\n",
      "Epoch:  94 , loss:  2.3695461750030518 , count accuracy:  0.408 , edge accuracy:  0.9317391025229506\n",
      "Epoch:  95 , loss:  2.2972190380096436 , count accuracy:  0.406 , edge accuracy:  0.9335623033960326\n",
      "Epoch:  96 , loss:  2.261439800262451 , count accuracy:  0.402 , edge accuracy:  0.9329524298645439\n",
      "Epoch:  97 , loss:  2.201672077178955 , count accuracy:  0.415 , edge accuracy:  0.9345830390961032\n",
      "Epoch:  98 , loss:  2.2863404750823975 , count accuracy:  0.412 , edge accuracy:  0.9342106952558259\n",
      "Epoch:  99 , loss:  2.221977710723877 , count accuracy:  0.413 , edge accuracy:  0.9334916864608076\n",
      "Epoch:  100 , loss:  2.222644329071045 , count accuracy:  0.412 , edge accuracy:  0.9339218077935417\n",
      "Epoch:  101 , loss:  2.3099451065063477 , count accuracy:  0.419 , edge accuracy:  0.9340309430570714\n",
      "Epoch:  102 , loss:  2.3079631328582764 , count accuracy:  0.409 , edge accuracy:  0.9325030493676575\n",
      "Epoch:  103 , loss:  2.1578786373138428 , count accuracy:  0.395 , edge accuracy:  0.9344161263401168\n",
      "Epoch:  104 , loss:  2.235250234603882 , count accuracy:  0.412 , edge accuracy:  0.9348205687873147\n",
      "Epoch:  105 , loss:  2.147027015686035 , count accuracy:  0.418 , edge accuracy:  0.9348783462797715\n",
      "Epoch:  106 , loss:  2.2201507091522217 , count accuracy:  0.419 , edge accuracy:  0.9355138986967966\n",
      "Epoch:  107 , loss:  2.1187169551849365 , count accuracy:  0.411 , edge accuracy:  0.9336329203312577\n",
      "Epoch:  108 , loss:  2.2093658447265625 , count accuracy:  0.411 , edge accuracy:  0.93379341336586\n",
      "Epoch:  109 , loss:  2.1756436824798584 , count accuracy:  0.405 , edge accuracy:  0.9343711882904282\n",
      "Epoch:  110 , loss:  2.1418650150299072 , count accuracy:  0.414 , edge accuracy:  0.9358413044873852\n",
      "Epoch:  111 , loss:  2.1606314182281494 , count accuracy:  0.414 , edge accuracy:  0.9356615522886307\n",
      "Epoch:  112 , loss:  2.1200220584869385 , count accuracy:  0.406 , edge accuracy:  0.934351929126276\n",
      "Epoch:  113 , loss:  2.2652487754821777 , count accuracy:  0.421 , edge accuracy:  0.935822045323233\n",
      "Epoch:  114 , loss:  2.0983119010925293 , count accuracy:  0.425 , edge accuracy:  0.9338383514155486\n",
      "Epoch:  115 , loss:  2.0581417083740234 , count accuracy:  0.429 , edge accuracy:  0.9341721769275213\n",
      "Epoch:  116 , loss:  2.2800710201263428 , count accuracy:  0.428 , edge accuracy:  0.935822045323233\n",
      "Epoch:  117 , loss:  2.1589226722717285 , count accuracy:  0.43 , edge accuracy:  0.9354689606471079\n",
      "Epoch:  118 , loss:  2.2353904247283936 , count accuracy:  0.429 , edge accuracy:  0.9372022854208127\n",
      "Epoch:  119 , loss:  2.226597547531128 , count accuracy:  0.403 , edge accuracy:  0.935828465044617\n",
      "Epoch:  120 , loss:  2.053783416748047 , count accuracy:  0.416 , edge accuracy:  0.9358156256018488\n",
      "Epoch:  121 , loss:  2.3179473876953125 , count accuracy:  0.407 , edge accuracy:  0.9352956281697374\n",
      "Epoch:  122 , loss:  2.0529134273529053 , count accuracy:  0.414 , edge accuracy:  0.9362906849842717\n",
      "Epoch:  123 , loss:  2.335205554962158 , count accuracy:  0.427 , edge accuracy:  0.9383257366630289\n",
      "Epoch:  124 , loss:  2.206385374069214 , count accuracy:  0.427 , edge accuracy:  0.9374141362264877\n",
      "Epoch:  125 , loss:  2.0820634365081787 , count accuracy:  0.429 , edge accuracy:  0.9370610515503627\n",
      "Epoch:  126 , loss:  1.9886621236801147 , count accuracy:  0.432 , edge accuracy:  0.9374012967837196\n",
      "Epoch:  127 , loss:  2.1789045333862305 , count accuracy:  0.416 , edge accuracy:  0.9378121589523015\n",
      "Epoch:  128 , loss:  2.05041766166687 , count accuracy:  0.427 , edge accuracy:  0.935269949284201\n",
      "Epoch:  129 , loss:  2.181492328643799 , count accuracy:  0.434 , edge accuracy:  0.936386980805033\n",
      "Epoch:  130 , loss:  2.1278584003448486 , count accuracy:  0.416 , edge accuracy:  0.9367079668742376\n",
      "Epoch:  131 , loss:  2.1479368209838867 , count accuracy:  0.416 , edge accuracy:  0.9355652564678693\n",
      "Epoch:  132 , loss:  2.1656486988067627 , count accuracy:  0.421 , edge accuracy:  0.9355780959106375\n",
      "Epoch:  133 , loss:  2.199862003326416 , count accuracy:  0.425 , edge accuracy:  0.9360659947358284\n",
      "Epoch:  134 , loss:  2.10709285736084 , count accuracy:  0.427 , edge accuracy:  0.9364704371830262\n",
      "Epoch:  135 , loss:  2.0422310829162598 , count accuracy:  0.434 , edge accuracy:  0.9368042626949991\n",
      "Epoch:  136 , loss:  2.1500422954559326 , count accuracy:  0.428 , edge accuracy:  0.9384605508120948\n",
      "Epoch:  137 , loss:  2.084538698196411 , count accuracy:  0.424 , edge accuracy:  0.9367143865956218\n",
      "Epoch:  138 , loss:  2.2892906665802 , count accuracy:  0.431 , edge accuracy:  0.9367850035308468\n",
      "Epoch:  139 , loss:  2.1157445907592773 , count accuracy:  0.431 , edge accuracy:  0.9348398279514669\n",
      "Epoch:  140 , loss:  2.2012758255004883 , count accuracy:  0.426 , edge accuracy:  0.9361815497207421\n",
      "Epoch:  141 , loss:  2.0916194915771484 , count accuracy:  0.435 , edge accuracy:  0.9329845284714643\n",
      "Epoch:  142 , loss:  1.9550838470458984 , count accuracy:  0.427 , edge accuracy:  0.9348398279514669\n",
      "Epoch:  143 , loss:  2.240842342376709 , count accuracy:  0.432 , edge accuracy:  0.9371124093214355\n",
      "Epoch:  144 , loss:  2.1204588413238525 , count accuracy:  0.431 , edge accuracy:  0.9379726519869037\n",
      "Epoch:  145 , loss:  2.136545419692993 , count accuracy:  0.434 , edge accuracy:  0.9362200680490467\n",
      "Epoch:  146 , loss:  2.195040702819824 , count accuracy:  0.439 , edge accuracy:  0.9365346343968671\n",
      "Epoch:  147 , loss:  2.142554759979248 , count accuracy:  0.439 , edge accuracy:  0.9371701868138923\n",
      "Epoch:  148 , loss:  2.1372339725494385 , count accuracy:  0.429 , edge accuracy:  0.9364704371830262\n",
      "Epoch:  149 , loss:  1.9903781414031982 , count accuracy:  0.431 , edge accuracy:  0.9381716633498106\n",
      "Epoch:  150 , loss:  2.008729934692383 , count accuracy:  0.429 , edge accuracy:  0.9383000577774925\n",
      "Epoch:  151 , loss:  2.160226821899414 , count accuracy:  0.436 , edge accuracy:  0.9382615394491879\n",
      "Epoch:  152 , loss:  2.074380874633789 , count accuracy:  0.445 , edge accuracy:  0.9368235218591513\n",
      "Epoch:  153 , loss:  2.027723789215088 , count accuracy:  0.436 , edge accuracy:  0.9385953649611607\n",
      "Epoch:  154 , loss:  2.1855876445770264 , count accuracy:  0.441 , edge accuracy:  0.9384220324837902\n",
      "Epoch:  155 , loss:  2.1038031578063965 , count accuracy:  0.438 , edge accuracy:  0.9373306798484946\n",
      "Epoch:  156 , loss:  2.154909610748291 , count accuracy:  0.435 , edge accuracy:  0.9362906849842717\n",
      "Epoch:  157 , loss:  2.0912325382232666 , count accuracy:  0.422 , edge accuracy:  0.9384926494190152\n",
      "Epoch:  158 , loss:  2.0611534118652344 , count accuracy:  0.432 , edge accuracy:  0.9382101816781152\n",
      "Epoch:  159 , loss:  2.0063302516937256 , count accuracy:  0.444 , edge accuracy:  0.9389548693586698\n",
      "Epoch:  160 , loss:  1.9991614818572998 , count accuracy:  0.443 , edge accuracy:  0.9373114206843423\n",
      "Epoch:  161 , loss:  1.958442211151123 , count accuracy:  0.444 , edge accuracy:  0.9383835141554856\n",
      "Epoch:  162 , loss:  2.21334171295166 , count accuracy:  0.436 , edge accuracy:  0.938691660781922\n",
      "Epoch:  163 , loss:  2.0490927696228027 , count accuracy:  0.446 , edge accuracy:  0.9391153623932721\n",
      "Epoch:  164 , loss:  2.175071954727173 , count accuracy:  0.444 , edge accuracy:  0.938897091866213\n",
      "Epoch:  165 , loss:  2.1463544368743896 , count accuracy:  0.448 , edge accuracy:  0.9385119085831675\n",
      "Epoch:  166 , loss:  2.122861623764038 , count accuracy:  0.445 , edge accuracy:  0.9366501893817808\n",
      "Epoch:  167 , loss:  2.05952525138855 , count accuracy:  0.448 , edge accuracy:  0.9378249983950696\n",
      "Epoch:  168 , loss:  2.1320407390594482 , count accuracy:  0.442 , edge accuracy:  0.9380753675290492\n",
      "Epoch:  169 , loss:  2.0140438079833984 , count accuracy:  0.445 , edge accuracy:  0.9368812993516081\n",
      "Epoch:  170 , loss:  2.022045135498047 , count accuracy:  0.446 , edge accuracy:  0.9374269756692559\n",
      "Epoch:  171 , loss:  1.9167137145996094 , count accuracy:  0.444 , edge accuracy:  0.9384220324837902\n",
      "Epoch:  172 , loss:  2.0613515377044678 , count accuracy:  0.448 , edge accuracy:  0.9379405533799833\n",
      "Epoch:  173 , loss:  2.018104076385498 , count accuracy:  0.451 , edge accuracy:  0.9382615394491879\n",
      "Epoch:  174 , loss:  1.9494966268539429 , count accuracy:  0.454 , edge accuracy:  0.9382422802850356\n",
      "Epoch:  175 , loss:  1.7587087154388428 , count accuracy:  0.446 , edge accuracy:  0.9383449958271811\n",
      "Epoch:  176 , loss:  1.8550444841384888 , count accuracy:  0.458 , edge accuracy:  0.9371830262566605\n",
      "Epoch:  177 , loss:  2.022266387939453 , count accuracy:  0.456 , edge accuracy:  0.9381716633498106\n",
      "Epoch:  178 , loss:  2.0113096237182617 , count accuracy:  0.464 , edge accuracy:  0.9381010464145856\n",
      "Epoch:  179 , loss:  1.8105573654174805 , count accuracy:  0.478 , edge accuracy:  0.9376324067535469\n",
      "Epoch:  180 , loss:  1.9163634777069092 , count accuracy:  0.488 , edge accuracy:  0.9385568466328561\n",
      "Epoch:  181 , loss:  1.863350510597229 , count accuracy:  0.501 , edge accuracy:  0.9364704371830262\n",
      "Epoch:  182 , loss:  1.6614629030227661 , count accuracy:  0.498 , edge accuracy:  0.938685241060538\n",
      "Epoch:  183 , loss:  1.807140588760376 , count accuracy:  0.493 , edge accuracy:  0.9379533928227515\n",
      "Epoch:  184 , loss:  1.8507134914398193 , count accuracy:  0.503 , edge accuracy:  0.9382936380561083\n",
      "Epoch:  185 , loss:  1.717226266860962 , count accuracy:  0.507 , edge accuracy:  0.9382807986133401\n",
      "Epoch:  186 , loss:  1.9944990873336792 , count accuracy:  0.489 , edge accuracy:  0.9387365988316108\n",
      "Epoch:  187 , loss:  1.9143891334533691 , count accuracy:  0.51 , edge accuracy:  0.9357257495024716\n",
      "Epoch:  188 , loss:  1.8687174320220947 , count accuracy:  0.491 , edge accuracy:  0.9355973550747898\n",
      "Epoch:  189 , loss:  1.9194934368133545 , count accuracy:  0.492 , edge accuracy:  0.9359632791936829\n",
      "Epoch:  190 , loss:  2.0154356956481934 , count accuracy:  0.51 , edge accuracy:  0.9353469859408101\n",
      "Epoch:  191 , loss:  1.9457921981811523 , count accuracy:  0.496 , edge accuracy:  0.9377543814598447\n",
      "Epoch:  192 , loss:  1.8166245222091675 , count accuracy:  0.526 , edge accuracy:  0.9372472234705014\n",
      "Epoch:  193 , loss:  1.979361891746521 , count accuracy:  0.516 , edge accuracy:  0.9381010464145856\n",
      "Epoch:  194 , loss:  1.8314417600631714 , count accuracy:  0.52 , edge accuracy:  0.9396738781536881\n",
      "Epoch:  195 , loss:  1.827699899673462 , count accuracy:  0.521 , edge accuracy:  0.9390447454580472\n",
      "Epoch:  196 , loss:  1.8227808475494385 , count accuracy:  0.518 , edge accuracy:  0.9363163638698081\n",
      "Epoch:  197 , loss:  1.7523926496505737 , count accuracy:  0.521 , edge accuracy:  0.935475380368492\n",
      "Epoch:  198 , loss:  1.9599902629852295 , count accuracy:  0.495 , edge accuracy:  0.937080310714515\n",
      "Epoch:  199 , loss:  2.0010786056518555 , count accuracy:  0.469 , edge accuracy:  0.9353213070552738\n",
      "Epoch:  200 , loss:  1.863943099975586 , count accuracy:  0.514 , edge accuracy:  0.9339731655646145\n",
      "Epoch:  201 , loss:  1.879685640335083 , count accuracy:  0.504 , edge accuracy:  0.9360788341785966\n",
      "Epoch:  202 , loss:  1.9052863121032715 , count accuracy:  0.519 , edge accuracy:  0.9384862296976311\n",
      "Epoch:  203 , loss:  1.8513412475585938 , count accuracy:  0.52 , edge accuracy:  0.9362457469345831\n",
      "Epoch:  204 , loss:  2.059642791748047 , count accuracy:  0.537 , edge accuracy:  0.9356487128458625\n",
      "Epoch:  205 , loss:  1.7278437614440918 , count accuracy:  0.514 , edge accuracy:  0.9382487000064197\n",
      "Epoch:  206 , loss:  1.863663673400879 , count accuracy:  0.507 , edge accuracy:  0.9374911728830969\n",
      "Epoch:  207 , loss:  1.857136607170105 , count accuracy:  0.471 , edge accuracy:  0.9354625409257238\n",
      "Epoch:  208 , loss:  2.0660808086395264 , count accuracy:  0.498 , edge accuracy:  0.9337227964306349\n",
      "Epoch:  209 , loss:  1.7690074443817139 , count accuracy:  0.519 , edge accuracy:  0.935545997303717\n",
      "Epoch:  210 , loss:  1.7688298225402832 , count accuracy:  0.516 , edge accuracy:  0.9378763561661424\n",
      "Epoch:  211 , loss:  1.742173433303833 , count accuracy:  0.543 , edge accuracy:  0.9393914104127881\n",
      "Epoch:  212 , loss:  1.870680570602417 , count accuracy:  0.531 , edge accuracy:  0.9386595621750016\n",
      "Epoch:  213 , loss:  1.8515676259994507 , count accuracy:  0.518 , edge accuracy:  0.9384605508120948\n",
      "Epoch:  214 , loss:  1.7366515398025513 , count accuracy:  0.542 , edge accuracy:  0.9385311677473198\n",
      "Epoch:  215 , loss:  1.8430802822113037 , count accuracy:  0.533 , edge accuracy:  0.9387879566026834\n",
      "Epoch:  216 , loss:  1.6865381002426147 , count accuracy:  0.54 , edge accuracy:  0.9396353598253836\n",
      "Epoch:  217 , loss:  1.8010212182998657 , count accuracy:  0.539 , edge accuracy:  0.9392244976568017\n",
      "Epoch:  218 , loss:  1.9431835412979126 , count accuracy:  0.551 , edge accuracy:  0.939314373756179\n",
      "Epoch:  219 , loss:  1.7670519351959229 , count accuracy:  0.544 , edge accuracy:  0.9393785709700199\n",
      "Epoch:  220 , loss:  1.75729238986969 , count accuracy:  0.535 , edge accuracy:  0.938126725300122\n",
      "Epoch:  221 , loss:  1.857451319694519 , count accuracy:  0.503 , edge accuracy:  0.9373884573409514\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-16cb4621fbe1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mcount_total\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0medge_total\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;31m#         print(batch.x)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch_geometric/data/dataloader.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(data_list)\u001b[0m\n\u001b[1;32m     30\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m             collate_fn=lambda data_list: Batch.from_data_list(\n\u001b[0;32m---> 32\u001b[0;31m                 data_list, follow_batch),\n\u001b[0m\u001b[1;32m     33\u001b[0m             **kwargs)\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/common/software/pytorch/v1.2.0-gpu/lib/python3.6/site-packages/torch_geometric/data/batch.py\u001b[0m in \u001b[0;36mfrom_data_list\u001b[0;34m(data_list, follow_batch)\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m                 \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcumsum\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcumsum\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m                     \u001b[0mcumsum\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__inc__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.train()\n",
    "loss_v = []\n",
    "acc_v_count = []\n",
    "acc_v_edge = []\n",
    "ep = 0\n",
    "for epoch in range(500):\n",
    "    ep += 1\n",
    "    count_correct = 0\n",
    "    edge_correct = 0\n",
    "    count_total = 0\n",
    "    edge_total = 0 \n",
    "    for batch in train_loader:\n",
    "#         print(batch.x)\n",
    "        optimizer.zero_grad()\n",
    "        data = batch.to(device)\n",
    "        edge_pred, graph_pred = model(data)\n",
    "        _, graph_pred_max = graph_pred.max(dim=1)\n",
    "        losses = [F.binary_cross_entropy_with_logits(edge_pred.float(), data.y.float()), F.cross_entropy(graph_pred, data.y_graph)]\n",
    "#         print(losses[0].item(), losses[1].item())\n",
    "        loss = sum(losses)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        edge_correct += ((edge_pred > 0.5) == (data.y > 0.5)).sum().item()\n",
    "        count_correct += float(graph_pred_max.eq(data.y_graph).sum().item())\n",
    "#         print(correct, pred, data.y)\n",
    "        count_total += len(graph_pred_max)\n",
    "        edge_total += len(edge_pred)\n",
    "#         print(out, data.y, )\n",
    "    count_acc = count_correct/count_total\n",
    "    edge_acc = edge_correct / edge_total\n",
    "    print(\"Epoch: \" , ep, \", loss: \", loss.item(), \", count accuracy: \", count_acc, \", edge accuracy: \", edge_acc)\n",
    "    loss_v.append(loss)\n",
    "    acc_v_count.append(count_acc)\n",
    "    acc_v_edge.append(edge_acc)\n",
    "plt.plot(np.arange(len(loss_v)), loss_v)\n",
    "plt.plot(np.arange(len(acc_v_count)), acc_v_count)\n",
    "plt.plot(np.arange(len(acc_v_edge)), acc_v_edge)\n",
    "plt.ylim(0.1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch(batch=[5910], edge_index=[2, 14768], x=[5910, 2], y=[14768], y_graph=[100])\n",
      "Accuracy: 0.6900\n",
      "Batch(batch=[6130], edge_index=[2, 15735], x=[6130, 2], y=[15735], y_graph=[100])\n",
      "Accuracy: 0.7000\n",
      "Batch(batch=[6460], edge_index=[2, 17175], x=[6460, 2], y=[17175], y_graph=[100])\n",
      "Accuracy: 0.6700\n",
      "Batch(batch=[6180], edge_index=[2, 15944], x=[6180, 2], y=[15944], y_graph=[100])\n",
      "Accuracy: 0.6800\n",
      "Batch(batch=[6110], edge_index=[2, 15557], x=[6110, 2], y=[15557], y_graph=[100])\n",
      "Accuracy: 0.8600\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "for batch in test_loader:\n",
    "    print(batch)\n",
    "    data = batch.to(device)\n",
    "    edge_pred, graph_pred = model(data)\n",
    "    _, graph_pred_max = graph_pred.max(dim=1)\n",
    "    correct = float(graph_pred_max.eq(data.y_graph).sum().item())\n",
    "    acc = correct / len(graph_pred_max)\n",
    "    print('Accuracy: {:.4f}'.format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combined Edge & Track Param Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How to do a double scatter..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [
     26,
     52,
     121
    ]
   },
   "outputs": [],
   "source": [
    "class TwoHopAttNetwork(nn.Module):\n",
    "    \"\"\"\n",
    "    A module which computes new node features on the graph.\n",
    "    For each node, it aggregates the neighbor node features\n",
    "    (separately on the input and output side), and combines\n",
    "    them with the node's previous features in a fully-connected\n",
    "    network to compute the new features.\n",
    "    \"\"\"\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim, hidden_activation=nn.ReLU,\n",
    "                 layer_norm=True):\n",
    "        super(TwoHopAttNetwork, self).__init__()\n",
    "        self.network = make_mlp(input_dim*5, [hidden_dim, hidden_dim, hidden_dim, output_dim],\n",
    "                                hidden_activation=hidden_activation,\n",
    "                                output_activation=hidden_activation,\n",
    "                                layer_norm=layer_norm)\n",
    "\n",
    "    def forward(self, x, e, edge_index):\n",
    "        start, end = edge_index\n",
    "        # Aggregate edge-weighted incoming/outgoing features\n",
    "        mi = scatter_add(e[:, None] * x[start], end, dim=0, dim_size=x.shape[0])\n",
    "        mi2 = scatter_add(e[:, None]*scatter_add(e[:, None] * x[start], end, dim=0, dim_size=x.shape[0])[start], end, dim=0, dim_size=x.shape[0])\n",
    "        mo = scatter_add(e[:, None] * x[end], start, dim=0, dim_size=x.shape[0])\n",
    "        mo2 = scatter_add(e[:, None]*scatter_add(e[:, None] * x[end], start, dim=0, dim_size=x.shape[0])[end], start, dim=0, dim_size=x.shape[0])\n",
    "        node_inputs = torch.cat([mi, mi2, mo, mo2, x], dim=1)\n",
    "        return self.network(node_inputs)\n",
    "\n",
    "class TwoHopNetwork(nn.Module):\n",
    "    \"\"\"\n",
    "    A module which computes new node features on the graph.\n",
    "    For each node, it aggregates the neighbor node features\n",
    "    (separately on the input and output side), and combines\n",
    "    them with the node's previous features in a fully-connected\n",
    "    network to compute the new features.\n",
    "    \"\"\"\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim, hidden_activation=nn.ReLU,\n",
    "                 layer_norm=True):\n",
    "        super(TwoHopNetwork, self).__init__()\n",
    "        self.network = make_mlp(input_dim*5, [hidden_dim, hidden_dim, hidden_dim, output_dim],\n",
    "                                hidden_activation=hidden_activation,\n",
    "                                output_activation=hidden_activation,\n",
    "                                layer_norm=layer_norm)\n",
    "\n",
    "    def forward(self, x, e, edge_index):\n",
    "        start, end = edge_index\n",
    "        # Aggregate edge-weighted incoming/outgoing features\n",
    "        mi = scatter_add(x[start], end, dim=0, dim_size=x.shape[0])\n",
    "        mi2 = scatter_add(scatter_add(x[start], end, dim=0, dim_size=x.shape[0])[start], end, dim=0, dim_size=x.shape[0])\n",
    "        mo = scatter_add(x[end], start, dim=0, dim_size=x.shape[0])\n",
    "        mo2 = scatter_add(scatter_add(x[end], start, dim=0, dim_size=x.shape[0])[end], start, dim=0, dim_size=x.shape[0])\n",
    "        node_inputs = torch.cat([mi, mi2, mo, mo2, x], dim=1)\n",
    "        return self.network(node_inputs)\n",
    "\n",
    "class Edge_Track_Net(nn.Module):\n",
    "    \"\"\"\n",
    "    Segment classification graph neural network model.\n",
    "    Consists of an input network, an edge network, and a node network.\n",
    "    \"\"\"\n",
    "    def __init__(self, input_dim=3, hidden_dim=8, n_graph_iters=3,\n",
    "                 output_dim=3, hidden_activation=nn.ReLU, layer_norm=True):\n",
    "        super(Edge_Track_Net, self).__init__()\n",
    "        self.n_graph_iters = n_graph_iters\n",
    "        # Setup the input network\n",
    "        self.input_network = make_mlp(input_dim, [hidden_dim],\n",
    "                                      hidden_activation=nn.ReLU,\n",
    "                                      layer_norm=False)\n",
    "        # Setup the edge network\n",
    "        self.edge_network = EdgeNetwork(input_dim+hidden_dim, hidden_dim,\n",
    "                                        hidden_activation, layer_norm=layer_norm)\n",
    "        # Setup the node layers\n",
    "        self.node_network = TwoHopAttNetwork(input_dim+hidden_dim, hidden_dim, hidden_dim,\n",
    "                                        hidden_activation=nn.ReLU, layer_norm=False)\n",
    "        \n",
    "#         self.output_network = NodeNetwork(input_dim+hidden_dim, hidden_dim, output_dim,\n",
    "#                                         layer_norm=False)\n",
    "        self.output_network = make_mlp(input_dim+hidden_dim, [hidden_dim, hidden_dim, output_dim],\n",
    "                                       hidden_activation=nn.ReLU,\n",
    "                                      output_activation=None,\n",
    "                                      layer_norm=False)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        \"\"\"Apply forward pass of the model\"\"\"\n",
    "        # Apply input network to get hidden representation\n",
    "        x = self.input_network(inputs.x)\n",
    "        # Shortcut connect the inputs onto the hidden representation\n",
    "        x = torch.cat([x, inputs.x], dim=-1)\n",
    "        # Loop over iterations of edge and node networks\n",
    "        for i in range(self.n_graph_iters):\n",
    "            # Apply edge network\n",
    "            e = torch.sigmoid(self.edge_network(x, inputs.edge_index))\n",
    "            # Apply node network\n",
    "            x = self.node_network(x, e, inputs.edge_index)\n",
    "            # Shortcut connect the inputs onto the hidden representation\n",
    "            x = torch.cat([x, inputs.x], dim=-1)\n",
    "        # Apply final edge network\n",
    "        e = self.edge_network(x, inputs.edge_index)\n",
    "        return e, self.output_network(x)\n",
    "    \n",
    "class Edge_Track_Truth_Net(nn.Module):\n",
    "    \"\"\"\n",
    "    Segment classification graph neural network model.\n",
    "    Consists of an input network, an edge network, and a node network.\n",
    "    \"\"\"\n",
    "    def __init__(self, input_dim=3, hidden_dim=8, n_graph_iters=3,\n",
    "                 output_dim=3, hidden_activation=nn.ReLU, layer_norm=True):\n",
    "        super(Edge_Track_Truth_Net, self).__init__()\n",
    "        self.n_graph_iters = n_graph_iters\n",
    "        # Setup the input network\n",
    "        self.input_network = make_mlp(input_dim, [hidden_dim],\n",
    "                                      hidden_activation=nn.ReLU,\n",
    "                                      layer_norm=False)\n",
    "        # Setup the node layers\n",
    "        self.node_network = TwoHopAttNetwork(input_dim+hidden_dim, hidden_dim, hidden_dim,\n",
    "                                        hidden_activation=nn.ReLU, layer_norm=False)\n",
    "        \n",
    "#         self.output_network = NodeNetwork(input_dim+hidden_dim, hidden_dim, output_dim,\n",
    "#                                         layer_norm=False)\n",
    "        self.output_network = make_mlp(input_dim+hidden_dim, [hidden_dim, hidden_dim, hidden_dim, output_dim],\n",
    "                                       hidden_activation=nn.ReLU,\n",
    "                                      output_activation=None,\n",
    "                                      layer_norm=False)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        \"\"\"Apply forward pass of the model\"\"\"\n",
    "        # Apply input network to get hidden representation\n",
    "        x = self.input_network(inputs.x)\n",
    "        # Shortcut connect the inputs onto the hidden representation\n",
    "        x = torch.cat([x, inputs.x], dim=-1)\n",
    "        # Loop over iterations of edge and node networks\n",
    "        for i in range(self.n_graph_iters):\n",
    "            # Apply edge network\n",
    "            e = inputs.y_edges\n",
    "            # Apply node network\n",
    "            x = self.node_network(x, e, inputs.edge_index)\n",
    "            # Shortcut connect the inputs onto the hidden representation\n",
    "            x = torch.cat([x, inputs.x], dim=-1)\n",
    "        # Apply final edge network\n",
    "        return self.output_network(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = cut_full_dataset[:int(0.5 * len(cut_full_dataset))]\n",
    "test_dataset = cut_full_dataset[int(0.9 * len(cut_full_dataset)):]\n",
    "train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)\n",
    "test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using  cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(\"Using \", device)\n",
    "# model = Edge_Class_Net( input_dim=2, hidden_dim=64, n_graph_iters=4).to(device)\n",
    "m_configs = {'input_dim': 3, 'hidden_dim': 16, 'n_graph_iters': 3, 'output_dim': 1}\n",
    "model = Edge_Track_Truth_Net(**m_configs).to(device)\n",
    "# data = dataset[0].to(device)\n",
    "learning_rate=0.0015\n",
    "# optimizer = torch.optim.SGD([\n",
    "#                                 {'params': model.input_network.parameters()},\n",
    "#                                 {'params': model.edge_network.parameters()},\n",
    "#                                 {'params': model.node_network.parameters()},\n",
    "#                                 {'params': model.output_network.parameters(), 'lr': learning_rate*10}], lr=learning_rate, momentum=0.9)\n",
    "# optimizer = torch.optim.Adam([\n",
    "#                                 {'params': model.input_network.parameters()},\n",
    "#                                 {'params': model.edge_network.parameters()},\n",
    "#                                 {'params': model.node_network.parameters(), 'lr': learning_rate*10},\n",
    "#                                 {'params': model.output_network.parameters(), 'lr': learning_rate*10}], lr=learning_rate, weight_decay=1e-4)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-4)\n",
    "s_configs = {'step_size': 20, 'gamma': 0.8}\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# exp = OfflineExperiment(api_key='JeB7KWbGOfnDj3EAhVrUwqPSe', project_name=\"node-track-regression\", offline_directory=\"comet\")\n",
    "exp = OfflineExperiment(project_name=\"node-track-regression\", offline_directory=\"comet\")\n",
    "exp.log_parameters({**m_configs, **s_configs, 'edges': 'true'})\n",
    "# if exp.alive is False:\n",
    "#     raise Exception(\"Something went wrong\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "COMET INFO: ----------------------------\n",
      "COMET INFO: Comet.ml Experiment Summary:\n",
      "COMET INFO:   Data:\n",
      "COMET INFO:     url: [OfflineExperiment will get URL after upload]\n",
      "COMET INFO:   Metrics [count] (min, max):\n",
      "COMET INFO:     sys.gpu.0.free_memory    : (15137701888.0, 15137701888.0)\n",
      "COMET INFO:     sys.gpu.0.gpu_utilization: (0.0, 0.0)\n",
      "COMET INFO:     sys.gpu.0.total_memory   : (16945512448.0, 16945512448.0)\n",
      "COMET INFO:     sys.gpu.0.used_memory    : (1807810560.0, 1807810560.0)\n",
      "COMET INFO:   Other [count]:\n",
      "COMET INFO:     offline_experiment: True\n",
      "COMET INFO:   Uploads:\n",
      "COMET INFO:     git-patch: 1\n",
      "COMET INFO: ----------------------------\n",
      "COMET INFO: Uploading stats to Comet before program termination (may take several seconds)\n",
      "COMET INFO: Starting saving the offline archive\n",
      "COMET INFO: To upload this offline experiment, run:\n",
      "    comet upload comet/a69089611e0b483d853b8fabdf19f5f8.zip\n"
     ]
    }
   ],
   "source": [
    "exp.end()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "COMET ERROR: Run will not be logged \n",
      "For more details, please refer to: https://www.comet.ml/docs/python-sdk/warnings-errors\n",
      "Traceback (most recent call last):\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/connection.py\", line 157, in _new_conn\n",
      "    (self._dns_host, self.port), self.timeout, **extra_kw\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/util/connection.py\", line 84, in create_connection\n",
      "    raise err\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/util/connection.py\", line 74, in create_connection\n",
      "    sock.connect(sa)\n",
      "TimeoutError: [Errno 110] Connection timed out\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 672, in urlopen\n",
      "    chunked=chunked,\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 376, in _make_request\n",
      "    self._validate_conn(conn)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 994, in _validate_conn\n",
      "    conn.connect()\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/connection.py\", line 334, in connect\n",
      "    conn = self._new_conn()\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/connection.py\", line 169, in _new_conn\n",
      "    self, \"Failed to establish a new connection: %s\" % e\n",
      "urllib3.exceptions.NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x2aab5006a278>: Failed to establish a new connection: [Errno 110] Connection timed out\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/requests/adapters.py\", line 449, in send\n",
      "    timeout=timeout\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 720, in urlopen\n",
      "    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/urllib3/util/retry.py\", line 436, in increment\n",
      "    raise MaxRetryError(_pool, url, error or ResponseError(cause))\n",
      "urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='www.comet.ml', port=443): Max retries exceeded with url: /clientlib/isAlive/ver (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x2aab5006a278>: Failed to establish a new connection: [Errno 110] Connection timed out',))\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/comet_ml/experiment.py\", line 563, in _start\n",
      "    self.alive = self._setup_streamer()\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/comet_ml/__init__.py\", line 237, in _setup_streamer\n",
      "    self.rest_api_client = get_rest_api_client(\"v2\", comet_api_key=self.api_key)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/comet_ml/connection.py\", line 1342, in get_rest_api_client\n",
      "    comet_api_key=comet_api_key,\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/comet_ml/connection.py\", line 1460, in __init__\n",
      "    check_api_backend_version(server_url, minimal_backend_version)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/comet_ml/connection.py\", line 213, in check_api_backend_version\n",
      "    response = requests.get(version_url)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/requests/api.py\", line 75, in get\n",
      "    return request('get', url, params=params, **kwargs)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/requests/api.py\", line 60, in request\n",
      "    return session.request(method=method, url=url, **kwargs)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/requests/sessions.py\", line 533, in request\n",
      "    resp = self.send(prep, **send_kwargs)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/requests/sessions.py\", line 646, in send\n",
      "    r = adapter.send(request, **kwargs)\n",
      "  File \"/global/homes/d/danieltm/.local/cori/pytorchv1.2.0-gpu/lib/python3.6/site-packages/requests/adapters.py\", line 516, in send\n",
      "    raise ConnectionError(e, request=request)\n",
      "requests.exceptions.ConnectionError: HTTPSConnectionPool(host='www.comet.ml', port=443): Max retries exceeded with url: /clientlib/isAlive/ver (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x2aab5006a278>: Failed to establish a new connection: [Errno 110] Connection timed out',))\n"
     ]
    }
   ],
   "source": [
    "exp = Experiment(api_key='JeB7KWbGOfnDj3EAhVrUwqPSe', project_name=\"node-track-regression\")\n",
    "exp.log_parameters({**m_configs, **s_configs, 'edges': 'true'})\n",
    "# if exp.alive is False:\n",
    "#     raise Exception(\"Something went wrong\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyper_parameters = {**m_configs, **s_configs, 'edges': 'true'}\n",
    "exp.log_parameters(hyper_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2aab7fcdaf60>]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA54AAAD4CAYAAACEyjk9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3ib1dk/8O/xtuMVJ7bjxHGcvUMSkrBp2askUGgLtJT2bUvpD1oofd9CN1AKlDJaCmWvsiGshOxJ9nAS7733tmXLsq11fn9IsmVbe8v+fq4rF9ajZ9yyjK37Oefct5BSgoiIiIiIiMhbQvwdABEREREREY1vTDyJiIiIiIjIq5h4EhERERERkVcx8SQiIiIiIiKvYuJJREREREREXhXmy4tNnTpVZmZm+vKSREQ0jp06dapdSpns7ziCGf82ExGRJ1n72+zTxDMzMxNZWVm+vCQREY1jQogaf8cQ7Pi3mYiIPMna32ZOtSUiIiIiIiKvYuJJREREREREXsXEk4iIiIiIiLyKiScRERERERF5FRNPIiIiIiIi8iomnkRERERERORVTDyJiIiIiIjIq5h4jjObcxqhUGn8HQYREREREQUgKSU+OlmLfrXOp9dl4jmOVLf34ZcfnMG9H53xdyhERERERBSAcuoVeODTPHycVefT69pNPIUQUUKIE0KIHCFEgRDiYeP2t4QQVUKIbOO/ld4Pd9grByqQ36AYerw5pxGFjT3oG9Tipa8roNfLEftvz2/CobJ2bM1rQrtycMRzhY09+Lq0DQBQ1d6HngEN9HqJr3IboRt1noo2JUpbeoeOa+zuH3quvLUXVe19FuPV6yXeOVaDQa3hzoJWp4dOLyGlxD93l6K1Z2DMMSXNvXhoUwGOlLfj01P1AAx3KO798AxOVHXiaEUH3jlaPbT/gPHcTd3D5ypoVKCxux+tPQMjvl8A0K1SY+7vt+JIefuIOF87WOmROyB1nSo8tKkA/9hRjLbeQZQ09+LzM/VDz3+Z3YDTtV1uX4ccd6qmC5c9vR8qtdbfoRARERGRH9R0GPKVg2Xtdvb0rDAH9hkEcKmUUimECAdwSAixzfjc/0kpN3ovPOse21oMAKh+4joAwC8/MIzy3XHeLLx9tAYzEqNx/VnTh/a/693TQ1+vSE/ApnsuHHp87XMHh851yVP7MS8lFndePAe/3ZiLv1w/iB9fMHto38ue/npoX/PjAODyZw6MeAwAbxyqwqwpMehWafCnL/LR1juI+69YgHl/2IYV6Ql4eP1S/HN3GU5Wd+K9n5474jX+8I3jaOkZxFtHqgEAN52dDuWgFl9mN2JPUSuUg4bk4fbzMq1+n6577hAAICo8BAMa/YjY8hoU0Okl/rO/AufPmwoA+CqvCY9uKUKTYgB/+tYSAMCjXxXi6mXTsCYzyep1LLn7/dPIrTcku8VNvdhT3AoAuHFVOgDg3g+zx3y/AEMCf+urx7HllxciJT7KqWuSbY9tLUJFWx8KG3ucfj+JiIiIKPjVdKgAAMcqO6DR6REe6ptJsHavIg2Uxofhxn/SxiF+1WtMxga1eqv7mI9SWlLeqkRbr2FUtLV30Oa+9jzyVSF+8nYWegcM6y4VKvXQc7n1iqER1QGNId76LhU6+9RjT+Qm0/nt6TeOhJniBYDXDlXh5peOOn1NvRz+MbH1foz25uFqtPUOYkdhi9PXJCIiIiIi60yJp3JQi5y6bp9d16H0VggRKoTIBtAKYJeU8rjxqb8JIXKFEM8KISKtHHunECJLCJHV1tbmobDtM0+cXDF6iu2RinafFO258O/7sObRXXb3M412AoBGp4faQmLX5mbS7CydXmJTTiOkMeHMb+gZes58Sm2HcnDEdGcpA/Y+BhERERHRuFLXqcLC1DgI4dvptg4lnlJKnZRyJYB0AOuEEMsA/A7AIgBrASQBeMDKsa9IKddIKdckJyd7KGz7Ht5caPW5dqXlEcW73x+ejvvMrtKhrw+VteO2V4/jrEd2Wjxu9MLcuk7VmH10dnKrUzVdQ4mi3sk87BtP7sOCP24bs33t33Y7dyI3vX6oEr/64Aw+O92A8lbliOdUZmtGz350N9Y8Ohzb7N9tHbGvab2tlBKfnqrnOlAiIiIiIhsGtTocKW93aECnprMPy9MTsGJGAg6VB1jiaSKl7AawH8DVUsom4zTcQQBvAljnhfh8aktuk8XtzRYK/5j74ETtiMfFzb1j9nlsa5Hd6xc399jdx5JGhe34zD2zqxTX/OugxeeOVLTjQKnhh8/a98KWlh5D4tylUqNb5fp04fouw1ToP39ZgN98koNv/+eIy+ciIiIiIhrvvsppwm2vHR+qDWPNgEaHlp5BzEqKwYXzpyK7rhs9bs4UdZQjVW2ThRCJxq+jAVwOoFgIkWbcJgDcACDfm4EGu9FTd71hW16z3X2e21OGoibLCe5trx7HljxDwtmn1uFIhW8rXRERERERkfNKjF03/ralCKdqrM8WrDXOzMyYEoML5yVDp5c4VtHhkxgdGfFMA7BPCJEL4CQMazy/AvCeECIPQB6AqQAe9V6YBAC//zzPZlWnf+0p8+j1evrZciNQFTb2oLXX8ZFuIiIiIhq/yluVmDUlBtMTo3HP+6fRobRc66XWWFgoIykGq2clIjo8FId9NN3Wkaq2uVLKVVLKFVLKZVLKR4zbL5VSLjdu+4FZ5VvykveP19rfiSaEa587iIuf3OfvMIiIiIjIQ7blNeH6fx9CQaPC6WMr2pRYNiMB//n+anT0qXHfR9kWZ1zWGEc8Z02ZhMiwUJwzJwkHAyXxDDYCwt8hOEX4MdzPzzT47+LkNkdb5BARERFR4NLpJZ7cXoxfvHcaeQ0K/OqDM+g3K8xpz4BGh7pOFeYlx2LZjAQ8sn4pDpa14/m95WP2re3oQ1xkGCbHhAMALpw3FZVtfXbbTXrCuEs8yXGfnbadeN717ikcr/TNnG/yvEc2F2L5Qzv8HQYRERHRhKUc1OKojTWUCpUG//PWSfxnfwVuXZeBN3+0FhVtfXh0i/UOHaNVd/RBL4G5KbEAgO+tnYmrl07Dawcrx4x61nSqMDMpBsI4+nXRfEPXkUM+aKvCxNMOT7eYFP4c4nTBL94bbjEzukXKaGzH6R1tvYN4fm+ZzfLYDd392JzTOGLbG4er0DvAdbpERERE/vL6wSrc+uoxi8U9NTo9bnrpCI5UtOOxG5fj8W8vxyWLUnDnxXPw3vFa7CywXzgUGP6MPi/ZkHgKIXDVslT0DmpR1jqy20ZtpwqzpsQMPV6QGovkuEifTLdl4kmQNksWDavrGtuflLzv/o+z8dTOUpyp67a6z40vHMYvPzjjw6iIyFuEEFcLIUqEEOVCiActPH+XECJPCJEthDgkhFhi3J4phOg3bs8WQrzk++iJiMicqXCPpVot2/KbUd6qxL9uWYXbzskY2v6/Vy7E0unxeODTXLTaaesIGBJPIYA5yZOGtq3OmAwAIyrc6vQS9Z39yDBLPIUQuHDeVBwub4fey104mHh6wEQc6StvVWJQ6/jcc3KdyjjH39Yvg9Zey5XLiCi4CCFCAbwA4BoASwDcakoszbxvLO63EsCTAJ4xe65CSrnS+O8u30RNRESWqNRanKnrQliIwOdnGtA3OHIm2ttHqjFrSgyuXjptxPaIsBD865ZV6Nfo8NtPc+1ep6KtD+mToxEVHjq0LSMpBlNjI0Ykns09A1Dr9JiVNGnE8RcvmIpp8VFo7/Pu50kmnkEmEJLcDuUgLn/ma/zxc7ZuddWgVoef/TcL5aOmPxDRhLcOQLmUslJKqQbwIYAN5jtIKc3na00CHJy2QkREPpVV3QWNTuL/XTIPykHtiGVR+Q0KnKrpwu3nzkJIyNilePNSYvHzi+dif0kbuvrUNq9T3qocmmZrIoTA6ozJOG2WeNZ09AEwJKXmblg5A1vvvQgpcVFOv0ZnMPEMIIGQVDrCtG7wRHWnnyMZNqBxfPT1/z7JwWsHK70YjX1naruxq7AFv2fyTkQjzQBQZ/a43rhtBCHE3UKIChhGPH9l9tRsIcQZIcTXQoiLrF1ECHGnECJLCJHV1tbmqdiJiMjMkYoOhIcK3PWNOViYGof3TwxPt33rSDWiw0PxnTUzrR5/zuwkAEBeg/X2Knq9RGWbEnNHJZ4AcPasyajuUKHd2NPT1MPTfI0n4LsaNOMu8Qyy2j3kAQfL2rDoT9uR5WAi/Mmpejy6pcjLURERucTSX7ExtyWllC9IKecCeADAH42bmwBkSClXAbgfwPtCiHhLF5FSviKlXCOlXJOcnOyh0ImIyNzRinasmjkZMRFhuO2cDOTWK5BXr0CHchCbchrx7dUzkBAdbvX4ZekJAIDceut1Phq6+zGo1WNeiuXEEzAMeACGwkJhIQJpCd4d2bRm3CWegcC53JeZsjOq2/vQMKrP0CHjou2T1V2WDiEiCib1AMxvf6cDaLSyL2CYinsDAEgpB6WUHcavTwGoALDAS3ESEZENin4N8hoUOHfuFADAjatnIDo8FO+fqMGHJ+ug1upxx/mZNs8RHxWOOVMnIbfe+ojnUEVbC4nnshkJCA8VQ+s8azpVSJ8cjbBQ/6SATDzJYxytjuuObz61Hxc8sdfr1yHyhKMVHfjhGyfG9NAisuEkgPlCiNlCiAgAtwDYZL6DEGK+2cPrAJQZtycbixNBCDEHwHwA/l1XQEQ0QZ2o6oReAucbE8/4qHBcf1YavsxuxDtHa3D+3ClYkBpn9zzL0xNsJp4VbYbE09JU26jwUCybkTC0zrO2w9DD01+YeI5D/p5uLIQIikoXJc29yLPxP7IvSSlR18l2NePN3e+fxoHSNnSrbBcFcEdrzwBUavZrHS+klFoA9wDYAaAIwMdSygIhxCNCiPXG3e4RQhQIIbJhmFJ7h3H7xQByhRA5ADYCuEtKGTiL8YmIJpCjFR2IDAvBqozEoW3fP2cWVGodmnsG7I52mqxIT0Rzz4DVtirlrUpMmRSByZMiLD5/dsZk5NR3Q63Vo6ajb8z6Tl9i4knjWlefGpkPbrHYO+mqfx7A9c8f8kNUw07XdEGr0+OtI9W46Ml9yLexeHw8u+nFI/jmP/b5O4ygtO6xPVjy5x1sbzSOSCm3SikXSCnnSin/Ztz2ZynlJuPX90oplxpbplwipSwwbv/UuP0sKeVqKeVmf74OIqKJorZDBUW/ZsS2IxXtWJuZhMiw4RYnK9ITsHxGAtInR+PyxakOnXvF0DpPy58RK6wUFjI5e9ZkDGr1OFrZgZ4B7ZhWKr7ExNMOR6aPemp0z6GRymAYSvSBgkYFznp4J9rs9K+s6zKMIn5wYmziGQi0eomnd5XiRJVhUKLWwqinDJZyx244VdOF6g6O+Lpjd2Grv0MgIiKacNRaPTa8cAg3/ucwFCpD8tmhHERxcy/OM06zNRFC4NUfrsEHPzsXoRZaqFiydHo8QgSQa2VworxVibkW1nearDYWGPr8dD0AcKotBZdASINeO1gFRb8GB8uCvw1AabNjvTw9OYXaG+/h16VtyHxwC0pb2JvUH/w9xZ6IiGgiOlzRji6VBpVtffjFe6eg0elxrNIwoHD+qMQTAKYlRDmV/MVEhGF+ShzyLFS27VAOokulsVhYyCQ1Pgrpk6Oxo6AFwNhWKr7ExJNcFmyfcz8/U4+eAY39HV2k00t8dLI2oAvJePM925bXBABDldNomKn3LQHtykGvrnklIiLytNKWXtzyylF09o39+7UtrwmxkWF47MblOFLRgT9/mY/DFe2IjQzD8hkJHrn+CmOBodGz4Cra+gAAc5NtT589e9Zk9Bt73mdwxNNzAiEZ4siD99+Hz880OLV/SXMvfv1RDv734xwvRQS8c7QaD3yah3eOVnvtGu742X+zkMWk0C+u/OcBSCmRU9c9IaZO27Lm0d1Y+cguf4dBRETksL3FrThW2Tlm6ZZGp8fOwhZcvjgFt52Tgf/3zbn44EQdNmbV45zZSR5rW7IiPQEdfWo0KkYWGLLVSsXc6gzDdNupsZGYFBnmkZhcMe4Sz2AzXpJUtVaPNw9XAwiM5N+kqt1wJ0it0wMAWmysCT1T24W5v9+K1l7LVcNMBjQ6vHawcszIZqdxXn93v/1RVX98j3YVtji03zvHapBdZ71RMTlPrdVjc24TNrxwGJtybLVktO6hTQX425ZCD0dGRERE9pS1GBK894+PnNl2vLIT3SoNrl6WBgD43ysX4ppl06DW6ces73THinRDZdzcUZ/PKtqUiA4PxfSEaJvHn21c55mRZHs/b2PiGUCCcSCksLEHAHC6dngkzV/JdEGjAh3K4cRSo9Pj3g+zHT7+jcPV0OkljlZ04L3jNdhXbLlYy3N7yvDoliJ8Zlyk7QmB9N7/6Yt83PDCYX+HMe5UGvtsmabFOOutI9V49WCVJ0MiIiIiB5S39iImIhQN3f3Ya/b5cGt+E2IiQvHNhckAgJAQgWe+uxL/d9VC3LQ63WPXX5QWh/BQMabAUHmrEnOSJyHETqGiRdPiMCkiFJlT/VfRFmDiSW6o7lDh2ucOYrOLIzie9lVuE657brg9ylvGEVhX/OHzfPz4rZMWnzOt1zPNlfckX+Xs//PmSRyr7PDR1YiIiIiCk5QSZa1K3LQ6HdPio/Dfo9UADLU9dhY045JFKYgKH26ZEh0RirsvmWe1r6YrIsNCsWhaPHJHFRgqb1XanWYLAGGhIXj9R2vx68sXeCwmV9hNPIUQUUKIE0KIHGPD6oeN22cLIY4LIcqEEB8JITz33XXC41uLoDcb8t5pNp2wzFhd82jF2A/YpqmEtXZaOHT3WZ426WoBGUdHAw+WtaGlZ+y0UEfauwDAAxtznQnLLWXG+eUmrhRSUQ5qPdKHsNmsuW7voHNxeCuBzq3vhtY41dcaR99XT+kd1OKe90/79JrkXYE0xZ2IiCjQ1HWqcNnT+1Hf5Vz7tkbFAFRqHRalxeG2czJwsKwdlW1KnKzuRLtSjWuWTfNSxCMtH1VgqLy1Fw3d/TZ7eJo7d84Uv7ZSARwb8RwEcKmU8iwAKwFcLYQ4F8DfATwrpZwPoAvAT7wXpnUvH6gcMexs3rz1imcPIKeuG7e+emzMcTe8cBjKQS0uttO0/qOsOovbX9xfbvWYn7ydhRf3V1h93nxaqjW3v37C4na1nQTGxFrczrJUCMVecZRndpU6fZ1lf9mBm148gi+cLBoU6Iqbe7D++cP4x44Sh/YPtDW/fYNavHm4asIXxCEiIqLA99SOEvz+8zyLzx2t7EBFWx+Km5xr+2ZqEzc/JQ63rJuJ8FCB947XYnt+MyLDQnDJwhS343bEihkJ6B3QoqZDhTO1Xbj5paOYGhuJ9WdN98n1PcFu4ikNTENa4cZ/EsClADYat78N4AavROgAWx+KN9hYq7bsLzscOv/nZ8au5Ss2771o4fpP7ihGl4WSy03dA9ia2+TQdS1Z97c9Lh/rKd5KQfIbevD20WqnjnE6UfNxAtVmLGZUYFwLG2we3VKEhzcXYvbvtvo7FCIiIiKbvsxpwNa8Jou5gakCrMKBIpAjjjMWFpqfEouUuChcvSwNn2TVYWteE765MNlnVWJNBYZe3F+B2149joTocHz6i/P8vm7TGQ6t8RRChAohsgG0AtgFoAJAt5TSNJexHsAMK8feKYTIEkJktbW1eSJmnztc7vxaOCmBVX8d2zJge0GzJ0KiccZWPuzPsUZFP/stBqvXD1Uhq7rT32EQERF5jEqtxR8+z0Njd/+Y57r61Kjr7Ee3SjN049+caQmeI90HRhzX2oupsZFDazZvP3cWega0aO0dxLXL01x4Fa6ZnxqLyLAQfJRVhznJk7DxrvMxa0rwJJ2Ag4mnlFInpVwJIB3AOgCLLe1m5dhXpJRrpJRrkpOTXY90nAqwmZXkAe4Nqtr6iQjsnxbOxnXQqG/UF2cahtr+uGP06P9fvyrEzS8dHbrGT6wUyyIiIgoWn2TV473jtfgie+zSrDyzpXclLWOn05a5OOJZ1qrEfLMCPmszJ2PRtDhEhIbg0kW+mWYLAOGhIbhm2TRctigFH955LpLjIn12bU9xamxYStkthNgP4FwAiUKIMOOoZzqAwChtSn4xOiWaiDlIYKeFvuPv74NOL/H6oUrcfm4moiNC7e6v0emhHNB6tPqciflUH2HlO3PfR9mIDg9F0V+vHvNc74AGMRFhCLVTJt2e+z5yvK0QERFRINLrJd4+Ug0AOFU9tl6KecXXkuZeXDR/eMBLpdaivsswStrjROIppUR5ixI3rh6e2CmEwOPfXo7aThXiosKdfRlu+ectq3x6PU9zpKptshAi0fh1NIDLARQB2AfgZuNudwD40ltBknNGrD91wKBGj09Pea4nJdknnFicmmPWLLixux9qrWMFpiaqTTkNeGxrMZ7d7ViRq99uzMWqv+4aUR3bm45VjZ3+aqk1j1qrx/KHduIvm/J9ERYREVFA+7qsDZXtfUiNj0RWTdeYv9u59QrMnjoJU2MjhwoCmVS0Ds8s6lY5voyouWcAvYPaESOeALAqYzI2rLS4ypBscGSqbRqAfUKIXAAnAeySUn4F4AEA9wshygFMAfC698Ikb8qq6cJvPsnxdxhBqXdAg+++fBQ1HcO/0AoaFTaOcE5LzwBMv1f71Tqc/8RePPip71rleMLWvCa8csB6lWdPU6kNSZyjbX2+NE7X8dUo/QkLiaclpgrWn592tNKz98eaCxoVaLCwroaIiMjb3jpcjZS4SNx72QIo+jUobxvZzi+vQYEV6QlYOC0WJS0jnytrNSSikyJCnZpqW2Y8z7yUODejJ8Cxqra5UspVUsoVUsplUspHjNsrpZTrpJTzpJTfkVKOXcU7jk3EqaT+dOlT+/12bZ1eWmkrA+wpasWJqs4RLWR2F7WiSWH9w/m832/F/R+PnPpo7edJadaLdMA4KravpNWJ6P3v/713Go9tLfZ3GB7VN6jFWxOwzcx1zx3CBU/s9XcYREQ0wZS3KvF1aRt+cO4snD93CgDgpFkBvdbeATQpBrB8RgIWpMahrKV3xIhoWasSYSECy2YkOJd4GteFLkh1rFcm2eZQcSHyjUDr4RhIKj1QfAVw/oZBz4AGc3+/Ff8x68vqyPs0erStu189NBKq1cuhdQZDcUnHz+2pVKddqcamHC7NdmWG7d+2FuGhzYVBdxOAiIgoGP33aDUiQkNw2zkZmDUlBlNjI0es88yrN3zGWpGeiIWpcVCpdSNm6JS1KIem4TpT1ba8tRdJkyIwJTb4CvkEIiaefmAtuQiWwZNAitNawRZP6VAa1gF8klXn1nnyG3pw3XOH7O7nyKtxZlG8Pb/64IzHzjWRmNaH9Ku53paIiMibFP0abDxVj+vPmo6psZEQQmDNrMk4WTM84plbr0CIAJZOj8eCaYZpsSVmNU/KW3sxPzUW8dHhTn2OKmtRYl4KRzs9hYmnBwRQHuY3zhTL8SSdjeEqb0T0ZXYD/nu0xgtnNqju6MPrh6q8dn5fML0jin4N3jvuve9VsOGMBiIiIud9klUHlVqHH1+QObRtTeZk1HX2o1kxAMCwvnNeSiwmRYYNFQIytVQZ0OhQ26nCvJQ4JMaEQ9GvcWipjJQSpS29YwoLkeucaqdCnuevhC0YmX65mPvNJzn4+Tfm+CyGP3w+XGE0p67b4zcdTGshb103EzERwfW/5+gf5d9uzMGOghb/BOMCwx8h/v9IRETkbxqdHofK2vFldgN2FLRgbeZkLJuRMPT82swkAEBWTSeuW56G3PpufHOhoadmXFQ4ZiRGD1W2rWzrg14C81Ni0dDdD41Ool+js/s5q613ED0DWixIZWEhTwmuT7bkVw1d/Vj3t934+80rvHYNWx/7z318j9eua41Kbb0y6mdnHK026hnBdo+is8/xcuVEREQUPPR6idvfOI4bV6Xj5rPTPXruNw9X4bk9ZehSaZAQHY4NK6fj7kvmjdhnyfR4RIeHIqu6C6szJqNdqcaK9OHEdEFq7NBUW1NF2wWpcUNFG7tVGruJp6mwEEc8PYeJJzns09P1aO0dxG83Blc7D3f87yeG19rW61zRZufXwVo+YHfh8IhhIK2ttcQb8VW192FWUgxCQmxn3adqujA3eRISYyI8H4QDTlR1YmZSNNISov1yfSD4bkwQEVHwyq7vxuHyDjR1D+Cm1TM8MoNPSoknthXj5QOVuGj+VNxxXiYuXpCMiLCxKwPDQ0OwcmYismo6cW69ocrtcrMR0QXT4nC4vAManR7lrUqEhghkTo1BhbEFi6Jfg+mJtv9mlxlHTOexoq3HjIs1nn6ZrhrgSYA3OZKEBepnYGeTo6KmHgCAytjKxNce3zbchsQUuq2f954BDbKqHesT6S2eeu+Lm3twyVP78eLXtnuASilx04tHcPvrJzx0Zed99+WjuPSpr/12fSBw/58jIqLxZ0dBMwBD1wFH+1PbotXp8duNuXj5QCVuP3cW3vrxOly+JNVi0mmyNnMyCht7cLSiHWEhAovT4oeeW5gaB7VOj5qOPpS1KDFrSgwiw0KRGB0OAA61VClrVSIhOhzJrGjrMeMi8SQ/C/ShuCDgqW/hXe+cws0vHUXfoPUpwsGiwdhy5lRNl509DfIaFN4MZ4Qm43pjaXYHqt9PNyeIiIh8SUqJncZ1l3GRYfjopHuV/we1Otz17ml8cqoe910+H49sWIpQOzOdAGBNZhL0Evj0dAMWTotDVHjo0HOmdZklzUqUtQ4XCIo3Jp7dKgcSzxYl5qfEsh6LBzHx9AP+AAcfKQ3FhFw5znMx2D9ZvjH50up4M8CbztQ6/7NAREQ0HpS3KlHV3of1K2dgw6rp2JLX5NAIojWbc5qwu6gFf7l+Ce67fIHDn5NXZSQiRADKQe2I9Z0AMC8lFiECyG9UoLpDhfkphkQ0wZh42mupIqVEaWsv5rOwkEcx8SSH2fo1MBFy6Z+/c8qh5M/bJsC32kOce6+8/c56+33blt/s8L5V7X14YGMutDr2ISUiIufsNNafuGJxKm5Zm4FBrR6bsl0vuHiyqhOJMeG447xMp46LiwrHommG6bUr0hNHPBcVHorMKZOwo6AZOr3EfOM6zcQYx6ba1sjEFOYAACAASURBVHf1o1ulwUKu7/SocZF46v2QDGjMPrAFQC7iNdLK1/bYaK8ZkHYWOP6hfSIJxp9tMUFT88+dqLJ874dn8FFWHQoae7wYERERjUc7CpqxcmYipiVEYdmMBCydHo8P3Zhue6rWUJnWXiFBS9ZmTgYwsrCQyYLUOFS29QEwjIACQGxkGEJDBLr7bVfe/yq3CQBw2eJUp2Mi68ZF4vnZ6XqfX3OnWbXRJkW/y+f5/qvHPBGOTzgzvfD7r7n2uk77YQqjVqfHne+c8vl1zQVhfmfXRE0ATZSDWnQ5sIYk2L15uAqZD27BoJZrXImIxrvG7n7k1itw1dJpQ9tuWTsTBY09yKt3vtZCt0qN8lYlzp412aV4vrt2Jm4+Ox2Lpo2dErvAuE0IYG5yrPFrgfioMLsjnptzGrEqIxEzk2JciossGxeJZ4fSv/0C2528/isHKoe+7lMPf1gbT8mHJojWGI4enbU1yufoq5JB+m6ael6ZnHSjQq4/ZiL0DWrdWmfiSRueP+TvEHzi33vLAQDKgeAvaBVIhBBXCyFKhBDlQogHLTx/lxAiTwiRLYQ4JIRYYvbc74zHlQghrvJt5EQ0nu0yDrxcuXR4JHD9yhmIDAvBhydrnT7f6VpDAUFXE8+l0xPw1HfOQljo2JRmoXF9ZkZSzIjCQ4kxEVD0W/+bVd6qRGFTD65fMd2lmMi6cZF4Elliad2pIwnhrqIWu/s4ej1Hki9Le7y433oLEW+up/3szMjZA87eVDE34IcRsHMe24O/by+2v6MPVBin9xA5SwgRCuAFANcAWALgVvPE0uh9KeVyKeVKAE8CeMZ47BIAtwBYCuBqAP8xno+IyG07CpoxLyV2aAQRMBTsuW55GjZlN6Kzb+znhq4+NZ7eWYKvchvHPHeqpgthIQJnjVqj6QkLpxliNFW0NYmPDke3yvrnm69yGyEEcN2KNI/HNNEx8Qwg+0tafXat332W67NrBZt2B/qUOuqdozV298mtHzu92DSK5Avb85vwsZul0L3BlVFzpQfbyHx0shaZD27BgI02KaPvK5im3R8ub/dYHDQhrQNQLqWslFKqAXwIYIP5DlJK8wW6kzB8D2sDgA+llINSyioA5cbzERG5pVulxvGqTly5ZOy6xx+cNwt9ai0u/PtePLy5APVdKqjUWrywrxwXP7kP/95bjkc2F0I3appZVnUXlk6PR3SE5++PzZoyCXFRYVg2av1nQnS41aq2UkpszmnEObOTkBof5fGYJrowfwdAw948XO2za31wwrOJxsRezWf99Td021//265U+7Va7l3vngZgWCfhKYfL25Hf4F7hmrve9e2629FvwbO7ygAAXSo10hKiHTpHdm030pZH4197yjwdHk0sMwCY/5KuB3DO6J2EEHcDuB9ABIBLzY41X2Rfb9w2+tg7AdwJABkZGR4JmojGtz1FrdDp5Yj1nSarMyZjy68uwqsHKvHO0Rr892gNEqLD0dmnxuWLU7ByZiKe2lmK45UdOH/eVACGQp059d24dZ13fgeFh4Zgx30XI2lSxIjtidHhqO2wPCupqKkXFW19+J8LZ3slpomOI55kkyMNdoOJSu369M/mngG3jrfF02lnjoVRVE+y1IbDfBrz9187bvN4KSWe2FbsUGJuMqjVDd0pvfv900FXiXgitBwij7H00zLm14SU8gUp5VwADwD4o5PHviKlXCOlXJOcnOxWsEQ0/qnUWrx1pBrT4qMsVpAFgMVp8Xjmeytx4LeX4H8uyMTazMn45K7z8Noda/HTi+ZgUkQoNuUMT7ctbOzBgEbv8vpOR0xPjB6xvhMwjHh2Wxnx3JzbiNAQgWuWcZqtNzDxpAml0sK6u2AtBGTLD984MfT1scoOZD64xWPn3p7fjHl/2IbSll6LzzuSYBU09uClrytwz/unHb7uwj9ux23GKtBbcpv8XomYyIvqAZhPQUgHMHZx1LAPAdzg4rFERDaptXrc9e5pFDQq8ND6pXbbnkxPjMYfrluCl29fg7WZSQAMfTWvXDoN2/KbodYabl6fqnGvsJCrTFNt9aOm/Zqm2V44b+qYUVLyDCaeNO50KAdtLhr3BW/MnHX1lO8dd77KnC07Cw0jjbkulE03MRVd0jq5jvN4letVdsm7xt/tG786CWC+EGK2ECIChmJBm8x3EELMN3t4HQDT/O5NAG4RQkQKIWYDmA/gBIiIXKDTS9z/cTYOlLbh8W8vx9XLxk6zddT6s6ZD0a/BgdI2AIb+nTMSox1ezuIpCdHh0EtAqR5ZFyK7rhv1Xf24/ixWs/UWu4mnEGKmEGKfEKJICFEghLjXuP0hIUSDsZR7thDiWu+Hay1Gf12ZAtHZj+7Gykd2uXy8J36ejld24MvsBvdPZEGg98f043JV8pBmxQB+8Npxh1rTBPZPY3CSUmoB3ANgB4AiAB9LKQuEEI8IIdYbd7vH+Dc5G4Z1nncYjy0A8DGAQgDbAdwtpWSTVSJympQSf9mUj69ym/C7axbhe2vdW4t54fypmBwTji9zGiGlxKnqLqz28WgnACTEhAMAFKOWk23OaUJEaMiIVjHkWY4UF9IC+I2U8rQQIg7AKSGE6VP9s1LKp7wXnmP4QZcCzUObCwHAYkNjV1n6OW/s7sf0RN/eKTT30tcVaOrux8MblgV8QuwpdZ0qj50rp867a3Fd9cK+chwqb8eX2Q344XmZ/g5nQpJSbgWwddS2P5t9fa+NY/8G4G/ei46IJoLPzzTg3WO1+Pk35uDn35jr9vnCQ0Nw7fI0fHa6ARVtSjT3DGCNPxLPaGPi2a8ZsS7hUHkbzp83BfFR4T6PaaKwO+IppWySUp42ft0Lw93XMRXyiMYjd0c/hTeG481OecMLhz1/fic8sa0YbzvQMsbcqZpOrH/ev3FbIiFR16lCv50CUre9Nlww1No9L0ff9SMVHQ7tZ6ulizdJCbx5uAodSudbDDV29yO/wfXp2ERE5F+fn2lA5pQYPHj1Io+dc/1Z09Gv0eGJbSUAfL++ExiZeJro9RI1HSosSPXcgAGN5dQaTyFEJoBVAEwlK+8RQuQKId4QQlj8yRFC3CmEyBJCZLW1tbkVLAWuYJ7ufPd71gvcODOa7s6In6uj9q0e7DnqK1tyPVuNttdK705HvqeVbcoRjy96ch9++t+ThuNHpZXKQS1uf/046jodr8TrKYv+tN3lY7ssNPN2VHFzDx7eXIj7Psp2+tjzn9iLb/37kMvXJiIi/+lWqXG0ogNXL0vz6E30tZlJSEuIwu6iFsREhHp0ZpijEo1Tbc07N7T0DmBQq8esKTE+j2cicTjxFELEAvgUwH3GxtUvApgLYCWAJgBPWzqOJdvHP3/PdC5sdK9f5O6i1hGPe/otJzLe5Mj30JHf+z0DY2MP4nsCDlnx0E6Xj7W0hvFw+chRSNMNhe35zThY1u7ytbwy+u2An7vRD1WtNfxkOrLWk4iIxo9dhS3Q6iWucaOYkCUhIWKoeM/KmYkIC/V9nVNLI57V7YZlNJlTJvk8nonEoXdbCBEOQ9L5npTyMwCQUrZIKXVSSj2AVwGs816YIzUrBnx1KYe19gReTBOFtWTA1RHIwibXEllftWVx9lWp1J5LpMtaevHZae8UTbJm9Ov15OuZCBq6rI/QWvuJPVTueoJNRETBb3t+M2YkRmNFuuWene5Yb0w8/THNFrCceNZ0GNrtccTTu+wWFxKG2/SvAyiSUj5jtj1NStlkfHgjgHzvhDjWuY/vGfE4LwDWEf30v1n+DoH8yNoHeOnByleutogZPaLrjiv/eWDo6y25rrUGNB/4c2UQ8OmdpS5d1x3vHqvB6doun1/XW2x923sGNKhqH9vv1h4WeSMiGh96BzQ4WNaO28+b5ZXZOkunx+Nft6zEhfOmevzcjogOD0VEaAi6+4c/V1V3qBARGuLz1i4TjSNVbS8AcDuAPGPZdgD4PYBbhRArYfjMXQ3g516J0AEdSv/2bASATjfWUdH48NrBKq+e/4FP87x6fkeYJxf7Skau2fbViO/2fPfWiA5odJASiI4IdfiYP37hs/tqfqcxNvZ2VDCv7yYimujqu1SIjw4fUcl1b3Er1Dq9x6fZmgghsGGl/+qUCiEQHx2OnlEjnjOTohEawj9q3mQ38ZRSHoLlG+RbLWwjmrCUVorcuGpfiedGKn3JE4nIzgLryWVDt3vFfS55aj+aFAOofuK6EdtdGbGz9lLd/R609AwgxonE2JyiX4MiF6eLj+armwlEROR7g1odNjx/GEmTIvD53RcgNtKQFmzLa0ZKXCRWZ/hnKqwvJESHjVzj2aHi+k4f8P2KXhp33C3uQ5an5H5dYrkK9KCTI1KBxpGczJ0iPvY0BeAacXMlzb0457E9eP2QayPoP337JG555Zj9HZ2gduBn7rPT9R69JhERedfeolZ09KlR1qrEbzfmQEoJlVqL/aWtuHrZNISM49G/xJiIocRTSomajj7MYuLpdeMi8eRdef/aU9yKNw9X+zuMoLCv2PIopqVE660j1Q6d8/bXj9vfibzCG795Ht9WBAD45+4yl44vbur1ZDiGczbbP+fj24o9fl0iIvKejafqkRofiQevWYStec145UAl9pe0YUCjx9VemmYbKBKiw4faqbQpB6FS65A5lYWFvM2RNZ7kgIleWMPVSrDeFIg3JH781kmPn9Obo4PeECxrAv31/3Rrj/d6s5a39mJeCptjExFNJINaHSLDRi7faOsdxP7SNvzsojn4+cVzkNegwN+3F2NBahySJkVgXWaSn6L1jYTocJS2GG6q1nQYWqlwxNP7xsWIpyPTwIi8yVou5chI0URRa/zF7is59Qr8e49ro4bmgiVRdsQ/dpT4OwQiIvKRAY0Ov/88D8sf2oms6s4Rz32Z3QCdXuLms2dACIEnb1qBeSmxKG7uxVVLU/3SX9OXEqLDh6baVhsruWeylYrXjYufKn0ADGy5W/CEaLyr7x6beFrK6d44VIWDZZbXtzqjqKkHT+/yfesVT+tT65zaf/SvQ0+29LFsHGXmRETjRG2HCje/dATvH69FZFgIHvg0FwMaw98TKSU2nqrHWTMTh2bBTIoMw8u3r8G6zCR8/5xZ/gzdJxKiw9E7oIVOL1HToUJYiMCMRLZS8bZxkXgSkf/tLGwBAGjt3Amy1xPska8KUe3j0VFPEz5KxrydVPrqdRARkefsKmzBdf8+iNoOFV774Ro8f9tqVLT14YV95QCAgsYeFDf34uaz00ccN3vqJHx813lYNiPBH2H7VEK0oX1MT78G1R19SJ8cPe5HeQMB13gSkUeYFun39LveVqZdOXZ9457iVgxqnRv187ejlR0+uc7yh3Z69fyurJMuaFRgw/OHvRANERHZU9epwl3vnsLitDi8+P2zMTPJMH3026tn4MX9Fbh2eRo2nqpHRGgI1q+Y7udo/ceUeCr6NajpUHF9p48wtSePCMRxEV+O1vgq0fCmfrUOL39dafX5bpXa6zGseXS3xe3eqNQ6Hni6d6wnvHus1u6oNxERece7x2sAAK/+cM1Q0gkAf7puCRKiw/HbjbnYlNOIK5akIiEm3F9h+l2i8bV3G0c8ub7TN5h4EnlAixcrkfpKXoPC5vOeSnL8cZOioFGBi57cC4VKY39nJzgz1dXRIkXupmy2LuPvqbPeX29KRDRxDWh0+PhkHa5YnIq0hJHrFSdPisDDG5Yir0GBzj71mGm2E41pxLO6vQ+9A1pkcMTTJ5h4ElHQuu/DMw7t96/dZajr7B8XI9P2jE7t7K2p9aXXDlb5OwQionFrS24TulQa/PA8y8WBrluehquWpmJGYjQumj/Vx9EFFlPimVPfDYAVbX2FazyJyKOsrQvcWdCC+SlxePmA9em8zvoiu9Gt482TstGDcaaHgZO2jeTK1Gd/56AnR5XzJyIiz3nnWA3mJE/CeXOnWHxeCIF/37oaA1rdhC+kY5pmnFNnSDy5xtM3JvZPHXlMIE6g23i6zt8hTBjb85vs7nO4vB2FTT0jN/o7E7JBBnjmaakQk7M8MfU1gN9CIqIJI69egey6btx+7iybM10iwkIQHzVx13aamEY8Cxp7IAQwM4mtVHyBiSd5RGVbn79DGKOuk71VfaW42bvFf4Iluenu9+waUq8Jlm8oERE55J1j1YgOD8VNE3ztpqMiw0IRFR6CQa0e0xOiERkW6u+QJgQmnkTkNvMips4Movk+/fHu2LwuAKq5jh7FNH/sSL454k65/18OERHZoVBp8GV2I25YNYOjmU5IjI4AAGRO5fpOX2HiSUQOsTV1Z0AT2H023Rng61cH9msjIqKJ7ZNTdRjU6nH7uZaLCpFlpum2XN/pO0w8iQgAZ1+aW/rn7UNfX/LUfo+c01etTFypYlvW0ouvct0r1ERERP6xOacRK2cmYsn0eH+HElRMiScr2voOE08impCaFQMjpqGaV+PtMxvl9Mbs2bx6BTIf3ILG7pHrkI9UtKNodAEmDxvQ6NA3qifrFc8ewD3vn8GaR3ePnKrLmxFERAFNr5coaenF2bMm+zuUoGOqbMsRT99h4klEAIDvvHTU3yFY5c5oYX6DwuL2cx/f4/I5XdFvNh35/RO1AID9pa0j9tlf0ubVGAQELnv6a5S3Ki0+74lKuaPpA2DdKxHReFXXpcKARo8FqbH+DiXoDI94MvH0FbuJpxBiphBinxCiSAhRIIS417g9SQixSwhRZvwvb7UQTVCOpIWW0g9fTO/91r8PDRX9ca57iGcTpn/vLbP5fHW7ZypD22yRIoCGbserPX92usHuPqPfwtHv6ZY8+612iIjINSXGqvILUuP8HEnwSTQmnhlJnGrrK2EO7KMF8Bsp5WkhRByAU0KIXQB+BGCPlPIJIcSDAB4E8ID3QiUif7KZIwb4lMzdRa32d/Iye0mv82ss3f+m6/QSv/4o2+3z2KJSa+3vRERELikzzmCZz8TTabesy8C8lFhER7CViq/YHfGUUjZJKU8bv+4FUARgBoANAN427vY2gBu8FSQRBb8Az019LDCmnxY39+LzM/ZHNcm3hBBXCyFKhBDlxhu7o5+/XwhRKITIFULsEULMMntOJ4TINv7b5NvIicjXSlt6MSMxGrGRjowlkbl5KbG4ZV2Gv8OYUJz6KRVCZAJYBeA4gFQpZRNgSE6FEClWjrkTwJ0AkJHBN5dovPvHjhJ/hxA0fFHp1pUqt+Q/QohQAC8AuAJAPYCTQohNUspCs93OAFgjpVQJIX4B4EkA3zM+1y+lXOnToInIb0qae7m+k4KGw8WFhBCxAD4FcJ+U0uGyi1LKV6SUa6SUa5KTk12JkYj8QBdARWE8lTvZekWuvFzn1oz6h801n17w/vHaUdf36eXHg3UAyqWUlVJKNYAPYZhhNERKuU9KqTI+PAYg3ccxElEA0Or0qGzr4/pOChoOJZ5CiHAYks73pJSfGTe3CCHSjM+nAfD/Iioi8pj/7Ct3eF9XR+7O1HZjwwuHXTrW0w6UereirD3Oj0w6n9F5e+xzwKxyL7lsBoA6s8f1xm3W/ATANrPHUUKILCHEMSEEl8AQjWM1nSqodXqu76SgYXeqrTB8GnodQJGU8hmzpzYBuAPAE8b/fumVCInIL0qttNywxJ0RyZy6btcP9iCNTj/icbtS7bVrWRoF9MUUZW9MuzU/JRNPj7D0Jlm8yyCE+AGANQC+YbY5Q0rZKISYA2CvECJPSllh4VgugyEKcqXGirYLmXhSkHBkxPMCALcDuNSsYMG1MCScVwghymBYi/KEF+MkIj+zlbM42k4l0FYb+nv940RYfvngZ3mobHP8JgahHsBMs8fpAMaUPBZCXA7gDwDWSymHGrBKKRuN/60EsB+GugxjcBkMUfArbVFCCEORHKJg4EhV20NSSiGlXCGlXGn8t1VK2SGlvExKOd/4305fBExE/tHYPeDW8eWtSuQ3KjwUjWfsLGh263h/JI7vHKvB8r/sgCfSeEvDaNUdKgtb3bPdze/zBHMSwHwhxGwhRASAW2CYYTRECLEKwMswJJ2tZtsnCyEijV9PheHGsXlRIiIaR0pbe5GRFMN2IBQ0WHuZiBxy04tH3D7Hk9v9W/F29BTX/+wfMwPRbSerHb8HV+NCkvenL/KtPjeodX+qqyPvs5QSrb2DSI2Pcvi8XX1q7CxscSe0CUFKqRVC3ANgB4BQAG9IKQuEEI8AyJJSbgLwDwCxAD4xjtrXSinXA1gM4GUhhB6GG8tPjKqGS0TjSGlzL+ancJotBQ8mnkTktmCZMtqlsr5u09Xqq2UtvSMeP7Ax17UTeYBGN/ZFNCqGR6o99Ta9erASj20tdnj/J7eXjLnpoNbqERHmcGH1CUVKuRXA1lHb/mz29eVWjjsCYLl3oyOiQKDW6lHV3ocrlqT6OxQih/GvPhEFvPdP1NrfyQF/2VTgkfOYSAlc8ewBl44LZgfL2i1uv+zprx0+R7DcrCAiCkTVHX3Q6iUWTuOIJwUPJp5EZJGv+z/aMro3pKsCqTepP3g72evo814lYCIiGlZirGjLqbYUTJh4EpHbXO3jSZ4lpXShuycREQWbspZehAhgTvIkf4dC5DAmnkTktvEwbXJnYfBXXq3r7PfaufsGtV47NxEROae0RYnMqZMQFc6KthQ8mHgSkdvGQd6JrXnOJ54Wxxf9+M2w167GndBO13a5cTQREXlSaUsvFnCaLQUZJp5EREGosk3p9DE7CtjOhIgo2A1odKju6MMCFhaiIMPEk4jc9tzecn+HMOHc+c6psRul7VHNfo37fT49ZTyMkhMR+UNFmxJ6CSxIjfV3KEROYeJJRBaxSI19owv/anR6/wRixpn37Y1DVV6Lg4iIvMNU0XZBKkc8KbiE+TsAIqJgNTrxfHhzoUMVBgOl9mxVe59LxwVQpx0ioglB0a/BzoJmbM5twuHydiTGhCNzCivaUnBh4klE5KLG7rFVZP05hfS94zVQqb0znZYtc4iI/KO8VYkNzx9Cn1qHmUnR+PnFc3Dz2emICOPERQouTDyJiFz09K5Sf4cwQk697aq2REQUfD7JqsOgVo9Pf3EeVmdMhhgPPcxoQuKtEiIiIiKasAJhfb41er3EppxGXLwgGWfPSmLSSUGNiScRkY9xjST44YmIAkJWdSeW/nkH8hsCc8ZIVk0XmhQD2LByur9DIXIbE08iIrLLPE8MlOJIRETukFLiyR0lUOv0KGrq8fj5W3sHoBzUunWOL7MbEBUegssXp3ooKiL/YeJJRJYxt3AJR/KIiILD4fIOnKjqBAA0KQY8fv5bXj6Gq549gNKW3jHP9Q5oUNBoe5RVo9Nja14TrlgyDZMiWZaFgh8TTyIiH/vsTIO/QyAimtCklHh6VwnSEqIwOSYcTYqxVcrd0a4cRGV7Hxq6+3HTi0dwpLx96Lpbcptw2dNf4/p/H0J9l8rqOQ6Vt6NLpcH6szjNlsYHJp5ERD6m0wf3cPLh8g63zyG50JWI/Gh/SRvO1HbjnkvnYWZSDBq6PTviaVoz+sx3z0JaQhR++MYJvHawEj9+6yTufv80YiJCoZcYGnG1ZFN2I+KjwnDxgqkejY3IX+wmnkKIN4QQrUKIfLNtDwkhGoQQ2cZ/13o3TCIi8idbH46IiIKJlBLP7CpF+uRofOfsmUhLiEKThb7M7ihoNKwZvXxJKj6563ysm52ER7cU4WRVJ/70rSXY+etvIC4qDCerLf9u7VfrsLOgGdcuT0NkWKhHYyPyF0cmjL8F4HkA/x21/Vkp5VMej4iIKIiN1xWeB8ra/B0CEZFH7CxsQV6DAk/evAIRYSFIS4h2eCaHQqVBfHSY3fX8efUKzJ46CfFR4QCAt368Dl+cacBFC6YiLSEaALBm1mSrN/X2FreiT63jNFsaV+yOeEopDwDgrW4iIiIiCmpSSvxrdxlmT52Eb6+aAQCYnhgF5aAWPQMam8cqB7U4/4k9eHZ3md3r5DcqsHR6/NDjiLAQfHftzKGkEwDWzk5CRVsfOpSDY47/MrsBKXGROGfOFEdfGlHAc2eN5z1CiFzjVNzJ1nYSQtwphMgSQmS1tfGOORFRMDpT2+3vEIiIHLK3uAUX/n0vulXqMc9VtClR2NSDH1+QibBQw8dgUzLYZGedZ3FTD/rUOrz0dQXqOq0XBerqU6O+qx/LZiTYPN+6zCQAwMnqrhHbFf0a7C9pw7dWTEdoyHidR0MTkauJ54sA5gJYCaAJwNPWdpRSviKlXCOlXJOcnOzi5YjI19irkYiIgtHGU/Wo7+rHzoKWMc/tLDRsu2LJcF/M6YlRAIBGO5VtTb0+pZR4fFuR1f1M6zuX20k8l6cnICIsZMw6zx35zVDr9NiwktNsaXxxKfGUUrZIKXVSSj2AVwGs82xYRERERETOUWv1OFBqaF2yJa9pzPO7CluwfEbCiCmvjo54Fjb1Ij4qDL+8dD625jXjWKXldaF5xoq25lNtLYkMC8XKmYnIGpV4bsppxKwpMViRbjtxJQo2LiWeQog0s4c3Asi3ti8R0URip94EERF50fGqDigHtVg0LQ6Hy9uhUA2v22ztHUB2XfeI0U4ASImLRGiIsNvLs7i5B4vT4nHnxXMwIzEaD28utNgeK79BgZlJ0UiMibAb77rMJOQ39qBvUDsU45GKdmw4a7rdAkZEwcaRdiofADgKYKEQol4I8RMATwoh8oQQuQAuAfBrL8dJRERERGTTnqJWRIaF4KH1S6HVS+wqahnxnJQYk3iGhYYgNS4SDTZaquj1EiXNvVicFo+o8FA8eM0iFDX14OOsujH75jcqsGy6Y6OVa2cnQaeXQ+vot+Q2QS+B9ZxmS+OQI1Vtb5VSpkkpw6WU6VLK16WUt0spl0spV0gp10spx85lICIiIiLyESkldhe14MJ5U3HO7CTMSIzGVrPptrsKW5A+ORqLpsWNOTYtMdrmVNuaThVUah2WKRRfjQAAIABJREFUpBmmz35rRRrWZk7GUztKRlTDVfRrUNOhsltYyGR1RiJCBHDCON32y+xGLEmLx7yUsTESBTt3qtoSEdEopS1Kf4dARDQhlbYoUd/Vj8sWp0IIgWuWTcPBsjb0DGigUmtxqLwdVyxJtTiFNS0hyuZUW1NhocXGxFMIgT99awk6+tR481D10H4FjYb1nY4mnnFR4VicFo+TVZ2o6ehDdl03Rztp3GLiSURERERBb7dxWu1li1MAANeuSINGJ7G7sAUHStuh1urHTLM1mZ4YjSbFAKS0XNG9qKkHIQKYnxo7tG1FeiKuXJKK1w5VDq0lzTcWFlpmp7CQubWZSThT14VPTzcAAK4/i4knjU9MPInIIit/e4nIy4QQVwshSoQQ5UKIBy08f78QotDYS3uPEGKW2XN3CCHKjP/u8G3kRJ5zsroTt79+HA9vLsDe4pah4ju27CkyVKxNjTe0R1mZnoi0hChszWvGrsIWxEeFYa2xd+ZoaQlRGNTq0dk3tvcnYEg85yTHIio8dMT2X1+xAL0DWrx2qBIAkN/Qg+kJUZgSG+nwa103OwkDGj1eP1iJtZmTMSMx2v5BREEozN8BEBERkYEQIhTACwCuAFAP4KQQYpOUstBstzMA1kgpVUKIXwB4EsD3hBBJAP4CYA0ACeCU8diR3emJAtypmi786I0TiAoPxYmqTrx5uBphIQKL0uKQEheF5NhIJMdF4oolqThrZiIAoEM5iDN13bj3svlD5wkJEbhmWRrePV6D6PBQXLooBeGhlsdchlqqKAYsJo1FTb1YPWvymO2L0+Jx3fI0vHGoCv9zwWzkNyiw1MFptiamZLhPrcP6lTOcOpYomHDEk4iIKHCsA1AupayUUqoBfAhgg/kOUsp9UkqV8eExAOnGr68CsEtK2WlMNncBuNpHcRN5RE5dN370xgkkx0Vi670XIecvV+K9n56Dn140B0mTItGsGMC+kla8+HUFbnrxCF4/VAUpJfaVtEFK4PLFI6fSXrt8GtRaPRT9GlyxZJrV605PNIySNlqobKvo16Chux+L0ywX/Ln38vlQaXR4ZlcpKtv7sNzJxDM5LhKzp05CaIjAtcusx0gU7DjiSUQWsX0YkV/MAGDen6EewDk29v8JgG02jrU4fCKEuBPAnQCQkZHhaqxEHpXfoMDtrx9H4qRwvP+zc4emzF4wbyoumDd1xL6Kfg1+83EO/vpVIU7XdKFPrcW0+CgsHbW2cnXGZKTGR6KrT4NvLEy2eu3pxumtlhLP4lGFhUZbkBqH9WdNxzvHagAAy2Y4vr7T5EfnZ6K5x/JoK9F4wcSTiIgocFi65WNxxbUQ4gcwTKv9hrPHSilfAfAKAKxZs4YruscJKaXFiq2BrqajD/89WoMPT9QiMSYCH/zs3KFE0JqE6HC8cvvZePlAJf6xoxh6Cdx2TsaY1x8SInD/FQvQ0jOI2EjrH3unTIpARFgImhRjW6qYKtousZJ4AsCvLpuPzTmN0EvHK9qau+P8TKePIQo2TDyJiMjnmOlYVQ9gptnjdACNo3cSQlwO4A8AviGlHDQ79pujjt3vlSgp4HxxpgH/2FGCT39xPqYlRPk7HIcUNCrw7K5S7CluRagQuHZ5Gv7vqoVInxzj0PEhIQK/+OZcrJyZiL9vL8atay2P3n/PynZzQgikJUSh0WLi2YukSRFIibM+Gjk3ORbfW5uB41UdSIkLju8/ka8x8SQii7bmNfs7BKKJ6CSA+UKI2QAaANwC4DbzHYQQqwC8DOBqKWWr2VM7ADwmhDBVQLkSwO+8HzIFguy6bjR09+P/Nubg7R+vQ0hIYI98Silx17unoBzQ4peXzMP3z501NLXWWefNnYIv7r7A7ZjSEqLQZGGqbVFzDxanxdkdTX70hmXQ6PRux0E0XrG4EBERUYCQUmoB3ANDElkE4GMpZYEQ4hEhxHrjbv8AEAvgEyFEthBik/HYTgB/hSF5PQngEeM2mgBaegYQGiJwsKwdbx2p9nc4dmXVdKGusx9/+tYS3H/lQpeTTk+anhA9ZqqtVqdHSXMvFk+zv24zNESMabdCRMM44klERBRApJRbAWwdte3PZl9fbuPYNwC84b3oKFA19wzgnNlJiA4PxRPbi3HBvKlYOM1yFdZA8NnpBkSHh+KqpYFTxTUtMQrNPQPQ6SVCjSPG1R0qDGr1WGRjfScROYYjnkRERERBrrVnENPio/D3m1cgPioM932UjUGtzt9hWTSg0WFLbiOuXjYNk2wU/PG1tIRo6PQSbb2DQ9uKhiraBm4STxQsmHgSERERBTG9XqKlZwCpCVGYGhuJv9+0AkVNPXh+b7m/Q7NoX3Erega0uGGVxW4/fjPUy1MxvM6zqKkHYSEC81Ji/RUW0bjBxJOIiIgoiHWq1NDqJaYZ10letjgV6zKTcLi83ePXalL041+7y6DTu16b+rMzDUiOi8QFc6d4MDL3pSWM7eWZXdeNeSmxiAzj2k0idzHxJCIiIgpizcaCOKnxw+0+psZFoGdA6/FrbcltwrO7S3G6tsul47v61Nhf0ooNZ01HWGhgfQw19Q5t6jZ8P7fnN+NIRQeuXZ7mz7CIxo3A+j+eiIiIiJzS2mtKPIcrw8ZHhaOnX+OFaxnWPx4sbXPp+K9yG6HRSdy4OrCm2QJAfFQYJkWEolHRjw7lIP7weR6WTo/HL74519+hEY0LTDyJiIiIglizwpAMmieecVFh6PXCiGdLjyHJPejiNN7PzjRgYWoclgRglVghBNISo9HUPYD/396dR8dRnXkf/z7a15Zla/EibzJeMMY7OxgMY2xIXhzO4LyQhDDZzDCQM+9kJZOZZEImmcwcMksmyQAhZJsAYSAJPsGEMMFACAQj4wUbG6+SJS/arF1qrff9o0uyJKvllrq1tPr3OaePuqqrq+6tKlX30/fWc7/8q700+Dv41w8uJ3GctcyKRCv9J4mIiIhEsfJ6P2aQm3m2q60vJZGW9k7aOroivi2A3aW11A2xRfVYVRM7j9dy68oZmFlEyxUp07JSePlgBb/dd5rP3LhgXA9JIxJtFHiKiIiIRLHyej9T0pP7tMz5UhMBaPBHtrtt97AtXQ7eODK0Vs+HXj5CnMHG5dMjWqZImp6Vir+9i1Wzs/nUNYVjXRyRCeW8gaeZPWZmFWa2t9e8yWb2opkd8v5mj2wxRURERGQg5fV+pmYl95nnSw2MjxnpBEMVDa2sW5xPelI8fzgUeuD57K4T/KKolLuvndeTPXY8WjA1k4zkBL69aRnxceOzVVYkWoXS4vljYEO/efcDv3fOzQd+702LiIiIyCg7Xd9KfmZKn3m+lECLZyQTDDW2dtDY2sGM7FSumDcl5MDzaGUjf/vLd1g9O5vPrlsQsfKMhI9dOYfXv3Q9c3LSx7ooIhPOeQNP59yrwJl+szcCP/Ge/wT4QITLJSIiIiIhqKj3k5/VL/D0utrWR7CrbUX92WFbrpmfy/EzzRyvbh70Pf72Tu59fCeJCXF8544V424Ilf7i4qwnaBeRyBruf3++c+4UgPc3L9iCZrbZzIrMrKiycnipt0VERETkXK0dnVQ3tQVt8YxkZtvyei97bmYKV8/PAeAPhwf/bvePz73L/lP1/OsHl/WMkykisWnEf3Zyzj3inFvtnFudm5s70psTERERiRmV3ria/e/xzEzx7vGMYFfb7vFC83wpFOakMz0rhT8cDN7d9tc7T/DffzrO5jWFXL8oP2LlEJHoNNzAs9zMpgF4fysiVyQREZnonBvrEohMDN3Dm+T5RqOrbau3rWTMjGvm5/L6kSo6Os8dsmXviTq++MweLp07mc+vXxixMohI9Bpu4LkFuMt7fhfwbGSKIyIiIiKhOl3ntXj2CzzTk+KJM6hviWRXWz+pifFkJgdaU6+en0O9v4M9J+r6LFfd2MrdP9vBlPQkvv/hlX2GeRGR2BXKcCpPAG8AC82szMw+AXwLWGdmh4B13rSIiIiIjKLynoQ/fQNPM8OXmhjRFs/yhlbyvdZOgKsuyMEMXuuV3ba9s4v7Ht9JZWMrD925ipyM5GCrE5EYk3C+BZxzdwR56YYIlyUkJ2pbxmKzIiIiIuNOeb2fpPg4stPOzcTqS0mM6D2e5fX+Pl16J6cnsWR6Fj949SivHa5iqi+FupZ23jhazbc3LWNpwaSIbVtEol/U9X1473T9WBdBREREZFwIBINnWyF786UmUB/BrLYV9f5zWla/uGERaxcFBjfYXVbLjpIa7lt7AX++qiBi2xWRieG8LZ4iIiIiMj6drvefc39nt8zkRBoi1NXWOUdFQyv5mX27zl49P6dnaBURkcFEXYunMiGKiIiIBFTUt57TCtnNl5oQseRCja0dNLd1kufTPZsiMjxRF3iKiIiIxKIdJTU9Y2lCoBXy9ADdX7v5UiKXXKjcG0ol2LZERM5HgaeIiIjIONfW0cVHHn2Tr/9mf8+87lbI/CCtkL7UyCUXqugeLzRTgaeIDI8CTxEREZFx7sDpelraO3lpfzn+9k7g7FAqU7OCt3g2tXXS0dkV9vbLG7qHbVFXWxEZHgWeIiIiIuPcrtJaAJraOnn1YCVwtvtrsFZIX2ogh2RDBDLbVnRvS11tRWSYoi7wVHIhERERiTW7jteSk5HEpLREnt97GoDTdedv8QQicp9neX0r6UnxZCRrQAQRGR5dPURERETGuV1ltSyfmc3k9ESef+c0rR2d5+3+mpkSuRbP8obgSYxEREIRdS2eIiIiIrGkrrmdo5VNLJ+ZxU0XT6OhtYM/Hq6ivM5PZkoCaUkDtyP4Ur0WzwgkGKqo92soFREJiwJPERERkXFsd1ng/s7lM7O5al4OmSkJbH3nNOWDjOEJke9qqxZPEQmHAk8REZFxxMw2mNl7ZnbYzO4f4PU1Zva2mXWY2W39Xus0s13eY8volXr8q/e381RRKS4Kk0XsKq3FDJbOzCIpIY51i/N58d1yymqbmTpY4OklF6pvCa+rrXOOCnW1FZEwRV3gGX0fFyIiIqExs3jge8BNwGLgDjNb3G+x48BfAI8PsIoW59xy73HLiBY2yvxyRxlfeHoPhysaR3W7W985xRef3hPWOnaX1jIvN6OnBfPmJdOoa2ln74n6Qbu/9nS1DbPFs97fgb+9i7xMdbUVkeGLusBTRESin9PPiMFcChx2zh11zrUBTwIbey/gnCt2zu0Bwh+cMYYcrWoC4Ejl6Aaez+05xVM7SmnrGN7hcs6xq7SW5TMn9cy7en5OT3bZwVo8M5ISMAv/Hs8Kb7xQDaUiIuGIusDTxroAIiIiI2cGUNprusybF6oUMysysz+Z2QeCLWRmm73liiorK4db1qhyrCfwbBrV7RZXN+EcnKxtGdb7y2paqG5q6xN4piTGc8OFeQCDdn+NizMykxOoDzOrbfd4oflq8RSRMERd4KnfyEVEZAIb6PfVoXz0zXLOrQY+BPy7mc0baCHn3CPOudXOudW5ubnDKWfUOTYGLZ7OOYq97ZbVDC/w3FnanVhoUp/5Ny2ZBsC0IGN4dstMSQy7q215ffewLWrxFJHhi7rAc7S7yIiIiIyiMmBmr+kC4GSob3bOnfT+HgVeBlZEsnDRyt/eyQmvxXE0WzyrGttoausEoLSmeVjr2F1aS3JCHAunZvaZf+PifL5zxwrWLsob9P2+1MSwkwtVNARaPDWcioiEI+oCz84utXmKiMiE9RYw38zmmlkScDsQUnZaM8s2s2TveQ5wFfDuiJU0ihw/04xzMDk9iaOVjaOW2bak+myQWzbMwHNXaS0Xz8giMb7vV7a4OOOWZdPPmd+fLyUhIi2eg40XKiISiqgLPEVERCYq51wHcB/wArAfeMo5t8/MHjCzWwDM7BIzKwM2AQ+b2T7v7RcCRWa2G9gGfMs5p8ATOOq1cq5dmEeDv4PKxtZR2W53996k+DhKz5y/q+2eslo++NAbvH28BoD2zi72nqg7p5vtUARaPMNMLqShVEQkAsL66crMioEGoBPo8O4rERERkWFyzm0Ftvab95Vez98i0AW3//teBy4e8QJGoe4A8M8uzOOZt8s4WtlEXubIB1Il1c3ExxnLZ04KqcXzie3H2V58hv/78Bv83fsWs3JWNq0dXSyfFUbgmZJIQwSSC2koFREJVyT6TKx1zlVFYD0iIiIiEXesqpGcjGSWei2HRyobubxwyshvt7qJguxU5uSk8fJ7g2cPds6x7UAl18zPISk+jq9u2cfsKWnAuYmFhsKXmhB2i2d5vZ9L5kwOax0iIupqKyIiIhNacVUzhTnpTPOlkJoY39P1dqSVVDcxZ0o6M7PTqGhoxd/eGXTZ98obOF3v5/1Lp/GDj67m8+sXUnqmmZyMZGZMSh12GXwpiTS2ddA1zBwZzjkqGlqVWEhEwhZui6cDfmdmDnjYOfdI/wXMbDOwGWDWrFlhbk5ERERkaI5WNXHDojzi4oy5OemjkiHfOUdJVTOrZmVTMDkQOJ6obWFebsaAy287EGgRvW5hoJz3rr2AK+ZNob2jC7Phj2KemZKAc9DQ2kFWauKQ31/X0k5bRxf5o9A1WUQmtnBbPK9yzq0EbgLuNbM1/ReI9FhhYVx7RUREJMbU+9upamxlbm46APPyMkalxbO6qY2G1g5mT0mnIDvQZbb0TPD7PLe9V8Hiab4+SXxWzsrmsjC7BPu8YHO43W0PnG4A6On2KyIyXGEFnr3GC6sAfgVcGolCiYiIiERCsZdYaM6UQOBZmJNOaU3zoN1eI6F7KJW5OYGutgBlNQNntq33t7OjpIa1i8L/gb4/X0pizzaGY0dJIMPuqtnZESuTiMSmYQeeZpZuZpndz4Ebgb2RKlgwozT0loiIiEwA3RltC3u1eDoXyDg7stsNrH/2lDTyMpNJio8LGni+dqiKzi7HdQvzIl4OX2rgrqr6luFlti0qPsMFeRlMSkuKZLFEJAaF0+KZD7zmjRe2HXjOOffbyBRLREREJHxHK5swg1mTA62OhTmBAHSk7/MsqW4iPs4oyE4jLs6YkZ1KaZAhVbYdqMCXksCKMLLXBhNOi2dXl2NHSQ2r1dopIhEw7ORCzrmjwLIIlkVEREQkoo5VNTFjUiopifHA2ZbPoyMceHZvNykh8Bt/QXbqgC2ezjlePljJmgW5JMRHfrCBrDDu8Txc2Ui9v0PdbEUkIjScioiIiExYxdVNzPVaOQHSkhKYnpXCkRFOMFRS3dwnIU9BdiplAyQX2neynsqGVtaOQDdbCGS1BWjwD72rbVFx4P7O1RrDU0QiQIGniIiITEjOOY5V9g08oTuz7ci1eDrnzgl4C7LTqG5qo7mtbwD4ysHAMCprFkQ+sRBARrJ3j+cwutoWlZxhSnoSc5TRVkQiINxxPEfds7tOjHURREQkTEoUJ6OhqjEwpEn/wLMwJ51n3j6Bc65njEx/eyfHzzRTXu/ndJ2fxtYObr9kFqlJ8UPe7pmmNhr8gaFUuhVkB8byLKtpYUF+Zs/8bQcqWFqQRW5m8nCqeF4J8XFkJCcMK7nQjpIaVs3ODmscURGRblEXeMbp4iciIiIh6M5oO1CLZ2NrB5UNreT5Ujhd5+fW7/+RU3X+Pss5Bx+/eu6Qt1vsZcydm9O7q233kCrNPYFnTVMbbx+v4b7r5w95G0PhS0kYcotnZUMrJdXNfPiyWSNUKhGJNVEXeMbHKfAUERGR8ztWFehOW5iT0Wd+9/Thykay0hK55+c7qGtp58FNy5g1OY18XzL3Pb6Tp3eUDS/w9ALe3i2eMyefbfHstnXvKbocrL8of8jbGApfauKQkwvtKDkDwKrZur9TRCIj6gLPBAWeIiIiEoKjVU0kxgeGMultXl53Ztsmtr5zip3Ha/n+h1dy88XTepbZtLqArzy7j30n67hoetaQtltS3UScwczssy2euRnJJCfEUdorwdCzO08yPy+DxdN8w6leyHwpiUNu8SwqriEpIY4lM0a2bCISO6IuuZDuMxAREZFQFFc1MWty2jm9pab6UkhLiuex147x3386zt1rCvsEnQC3LJtOUnwcT+8oG/p2q5uZkX12KBUIfH+Z0WtIlRO1LWwvPsPG5dNH/LuNL3Xo93gWldSwrCCL5ISh3+MqIjKQqAs81eIpIiIycZyqa+GNI9U0tp4/MGrwt7PtvQqqGltDWvexqibm9utmC4EgsDA3naNVTVxROIXPr194zjKT0pJYtzifZ3edpK2jK6TtdSuubmLOlPRz5s/MTqO0JtDiuWXXSQA2Lp8xpHUPR2ZKIg2tobd4+ts72XeyTt1sRSSioq6rbZwCTxERkah2uKKR3+49xe/eLWdPWR0QyOFw8YwsLi+cwspZk7hwmo+C7FTMjNIzzfz49WJ+8VZpT4C6rCCLaxfmsXhaJpWNbVTU+ymv9xMfF0duZjJ5mckUVzdzXZDxMZcWTKKmqZ3//NAKEuIH/h3+tlUFPPfOKV46UMGGJVNDqptzjmNVTXxggICyIDuV3WW1QCBL/8pZk5g5eeSHKvGlDK3Fc3dpLe2djtWzs0ewVCISa6Iu8IxXV1sREZGo9crBSj72o+10OVg+cxJf2LCQRVMzebukljePVfPD147yUGdgvJ3MlATmTEln38k64sy4+eJpfGDFdN49Wc+29yr57kuH6PKG5okzyMlIpss5qpvaeobsCXb/5AO3XER7pxt0uJRr5ueQl5nM0zvKQg48a5rbvaFUzg0oZ05Oo7a5nR0lZzhwuoEHNl4U0jrD5UtNpMHfTleXC+kH/KKSGgBWKfAUkQiKvsBTLZ4iIiJRqcHfzv3P7KEwN4Off/Iy8n0pPa9dvyiQ2bWlrZP9p+vZf6qeA6caOFzRyOY187jrytlMy0rtWfa+6+dT09TGidoW8jKTmZKR3PMdoaOzi+qmNhr87edktO2WEB/H+W5fTIiP49aVM3j0D8eobGjtGWuzo7Or5/X+ujPpDtTVtnssz+++dJj4OON9/e4rHSm+lES6HDS1dZCZknje5YuKzzAvN53s9KRRKJ2IxIqoCzzV4CkiIhKdvrn1AOX1fp6558o+QWdvqUnxrJyVzcpZ529ty05PGjA4SoiPI9+XEnQbQ7FpVQEPv3KUZ3ed4IYL83n8zRL+Z0cZ83Iz+MXmy88JPh97rZjkhDiWFpybCbd7LM9t71Vy3cJcpmQkh12+UPhSA1/36v3nDzz/991yXj5YySeuGvowMiIig4m65ELKaisiIhJ9XjtUxRPbj/PJawpZEUJQOV5ckJfJ8pmT+PbvDrL2wZf50R+LWTQ1kx0lNfzgD8f6LPvqwUqee+cU9669gLwBgt6ZvYZ1Gege0JHi84LN843lebC8gb9+cidLpmfx2RvPTbgkIhKOqAs8L9Z4UiIiIlGlsbWDLz6zh8KcdD6zbsFYF2fI/vLaQgqyU/nsugW8fv/1PPGpy7lpyVT+7cWDHCpvAKC1o5OvbtnHnClpbF5TOOB6JqcnkZoYT2piPOsW549a+X2p5w88a5ra+ORPikhNSuCRj64a9N5XEZHhiLrA88p5OWNdBBEREQlRR2cXX9uyj5N1LfzLbUtJSYy+gGbDkmm8+Jlr+fQN88nzpWBmPLBxCenJ8Xzu6T10dHbxyCtHOVbVxAMblwSto5mxbGYWt66cQXry6N3tlJkS2FaDf+DMtu2dXdz7+NucrvPz8J2reu6lFRGJpKi7x3NBfuZYF0FERMLkb++MygBEhuat4jP8/a/3cuB0A3dfW8jqORNnXMjczGQe2LiETz+xk6//5l2efKuUmy+eypoFuYO+74lPXd6TcXe09HS19fdt8XTO8crBSr6/7Qjbi8/w4KZlymQrIiMm6gJPxyhfrUVEJOLaOrrGuggygqobW/nm1gM883YZMyal8tBHVrH+otHrWjpa3r90Gs/tOcVP3ighLSmev3//4vO+x8xGPVFid1fbF/adxt/eRZY3vMqPXy/mwOkG8n3JfOPWJdy2qmB0CyYiMSXqAs9U/UIuIhL1QhlLUKLTsaomPvLom1Q0+Pmr6+Zx3/UXkJYUdV83QmJmfP0DSzhc2cjHrpozbruoZqUmMjcnnRf2lfPCvvKe+QvyM3hw0zJuWTadpISou/tKRKJMWJ8EZrYB+A8gHnjUOfetiJRqEKGMPyUiIuNbvDKUT0h7T9Rx12PbccAz91zJ0oJJY12kEZebmcyLf7NmXGfdj48ztn3uOvztndQ2t1Pb0kZHp+Oi6b5xXW4RmViG/fOWmcUD3wNuAhYDd5jZ+fuYRMBTd1/BZ9Yt4OaLpwLwiavnkhhvXL8oj8+vX8jfve9CrpnfNwnRxuXT+d/PrOG+tRfwoctm8dBHVnLn5bNZNDWTRVMzyc1MZsNFU7n/pkXcu3Zez/v+844VPLhpGQBXFE7BDO6/aVHPINDTslL4/PqzKcd/+vFL2f7lG7jRy1a3fObZD92k+DhmT0nrmZ6fl8G1C3K557p5zJzc91fSzJQEVs3OZu3CXKb2Ssn+8J2reHLz5T3TwTLndfv1vVcN+nqkJSXE8fWNFwHwpZsWAfDV/xM4Lf725kU8sPEirr4gcGwumZPNn68s6JkGWDRV9/CKxIKsVP2IONG8caSa2x/5EymJ8Tz9l1fERNDZLVqCt5TEeKZmpbBoqo8lM7KiptwiMjGYG+Yd7mZ2BfAPzrn13vSXAJxz/xTsPatXr3ZFRUXD2l6seP1wFRcXZA2rZbe2uY0ntpdy64oZVDe1snhaaL9k/uXPdjA7J42HXznKrq+sY1JaEg3+do5WNvGNrfvZfuwM+762nvTkBE7VtfDoH45RUt3E/+6vOGddB76+IaIJQ57ZUcZn/2f3OfPvXlPIw68ePe/7l8zwMdWXMmBZr5w3hdePVPeZNzk9iTNNbcMvMIEv1HXnGSut29qFuSTGx/G7d8vPv3CETEpLpLY5tPKJjJTib70vIusxsx3OudURWdnC53uzAAAL+UlEQVQ4cb7eRGa2Bvh3YClwu3Pu6V6v3QX8nTf5j865n5xve5H4bP79/nLu+fnbzJ6cxk8/cem47XIqIiIjL9hncziB523ABufcJ73pO4HLnHP39VtuM7AZYNasWatKSkqGtT0Zvyrq/UzJSKa1o3NC3cfT0taJw/WpU3NbB51dLuQfBrq6HA2tHWSlJlLvb6e9o4spGckjVeQ+nHOYGc45yutbyfcln/NDRGVDK5kpCSQnxHGsqonC3AwAjlY2UlRSw6ZVBT3vOVbVRKO/g2mTUjhwqoG6lnYWT/fR3NbB8++c5udvlvDk5ito6+iipb2Tk7UtpCbF809b9/Pk5ivIy0zmp28U8/M3j/PQnat4bs8pPnrFbJ4qKuWbWw/wyJ2rOFnbwsGKRm5YlEeDv4O2ji6KSs7wVFEZ6xbn8/6l0/jPlw7z+Kcu4+M/fovCnAw2rynk3VP1vHKwEn9bJ+svmsrDrx7hU9cU8rM/lfCFDYvISE7g7ZIaXj5YwQdXz+R4dTPXX5jH3/96L3tP1jM/L4MPrp7JV7fs47K5k7nryjnUNrfz3ZcOcbLOz7c3LaOjq4udx2t5fu9p1i7MZcWsbL66ZR/P3HMFf/5fb7Bi1iR2Hq/t2bc/+otLWLsoj7aOLrbsPkmDv52Vs7LZ+L0/AvDdD63gvsd3cuuKGewqrSUpPo6SM01cvyiPqb5UjlY18k5ZHbetKmDZzEn87a/ewTn6/Khx6ZzJ5GelcM+18/jO7w/x232ne17zpSTwjVsv5tNP7ORzNy7gwd8d5NI5k9lefKZnmfddPI1XD1WeM8RCUkIcj911CR/54ZtAoJteZ1fwz4qVsyaxq7SWC/IyuGnJNP7j94eCLnvpnMncunIGd1w6K+gyQzHRAk+vN9FBYB1QBrwF3OGce7fXMnMAH/A5YEt34Glmk4EiYDXggB3AKudczWDbjETgeeB0Pd96/gD/9sHlZKcnhbUuERGJbiMReG4C1vcLPC91zn062HvU4ikiIpE0AQPPkHsTmdmPgd/0CjzvAK5zzt3tTT8MvOyce2KwbeqzWUREIinYZ3M4KczKgJm9pguAk2GsT0REJNbNAEp7TZd58yL6XjPbbGZFZlZUWVk5rIKKiIgMRTiB51vAfDOba2ZJwO3AlsgUS0REJCYNdGN+qF2TQn6vc+4R59xq59zq3NzckAsnIiIyXMMOPJ1zHcB9wAvAfuAp59y+SBVMREQkBoXTm0g9kUREZNwKKxOMc24rsDVCZREREYl1Pb2JgBMEehN9KMT3vgB808yyvekbgS9FvogiIiJDF05XWxEREYmgYL2JzOwBM7sFwMwuMbMyYBPwsJnt8957Bvg6geD1LeABb56IiMiYmzhjX4iIiEwAA/Umcs59pdfztwh0ox3ovY8Bj41oAUVERIZBLZ4iIiIiIiIyohR4ioiIiIiIyIgy50LN0h6BjZlVAiURWFUOUBWB9USbWKx3LNYZYrPesVhniM16R7LOs51zGg8kDPpsHhPaV6HRfgqd9lVotJ9CF86+GvCzeVQDz0gxsyLn3OqxLsdoi8V6x2KdITbrHYt1htisdyzWORbouIZO+yo02k+h074KjfZT6EZiX6mrrYiIiIiIiIwoBZ4iIiIiIiIyoqI18HxkrAswRmKx3rFYZ4jNesdinSE26x2LdY4FOq6h074KjfZT6LSvQqP9FLqI76uovMdTREREREREoke0tniKiIiIiIhIlFDgKSIiIiIiIiMq6gJPM9tgZu+Z2WEzu3+syxMuMys2s3fMbJeZFXnzJpvZi2Z2yPub7c03M/uOV/c9Zray13ru8pY/ZGZ3jVV9gjGzx8yswsz29poXsXqa2SpvPx723mujW8NzBanzP5jZCe947zKzm3u99iWv/O+Z2fpe8wc8581srpm96e2LX5hZ0ujVbmBmNtPMtpnZfjPbZ2Z/7c2f6Mc6WL0n7PE2sxQz225mu706f22wcppZsjd92Ht9Tq91DWlfyPijYzWwoV4TBcws3sx2mtlvvOlxde0bD8xskpk9bWYHvHPrCp1TAzOzv/H+9/aa2RPeZ5fOKSL33XxInHNR8wDigSNAIZAE7AYWj3W5wqxTMZDTb96/APd7z+8H/tl7fjPwPGDA5cCb3vzJwFHvb7b3PHus69avTmuAlcDekagnsB24wnvP88BN47TO/wB8boBlF3vnczIw1zvP4wc754GngNu95w8B94yDOk8DVnrPM4GDXt0m+rEOVu8Je7y9/Z/hPU8E3vSO4YDlBP4KeMh7fjvwi+HuCz3G10PHatB9M6Rroh4O4DPA48BvvOlxde0bDw/gJ8AnvedJwCSdUwPupxnAMSDVm34K+AudUz37J+zv5kN9RFuL56XAYefcUedcG/AksHGMyzQSNhK4qOD9/UCv+T91AX8CJpnZNGA98KJz7oxzrgZ4Edgw2oUejHPuVeBMv9kRqaf3ms8594YL/Hf8tNe6xkyQOgezEXjSOdfqnDsGHCZwvg94znutfNcDT3vv773/xoxz7pRz7m3veQOwn8CFf6If62D1Dibqj7d3zBq9yUTv4Qhezt7nwNPADV69hrQvRrhaMjw6VkEM45oY08ysAHgf8Kg3Pe6ufWPNzHwEAoYfAjjn2pxzteicCiYBSDWzBCANOIXOKSBi382HJNoCzxlAaa/pMgb/chcNHPA7M9thZpu9efnOuVMQ+NAC8rz5weofrfslUvWc4T3vP3+8us/rpvBYr64wQ63zFKDWOdfRb/644XWlXEGgJSxmjnW/esMEPt5el7hdQAWBHweOELycPXXzXq8jUK+Jdl2LRTpWIQjxmhjr/h34AtDlTY/La98YKwQqgR95XZIfNbN0dE6dwzl3AngQOE4g4KwDdqBzajBD/b42JNEWeA50L1e0jwdzlXNuJXATcK+ZrRlk2WD1n2j7Zaj1jKb6/xcwD1hO4CL4bW/+hKqzmWUAzwD/zzlXP9iiA8ybSPWe0MfbOdfpnFsOFBBo9bpwoMW8vxOizjIgHavzGMI1MWaZ2fuBCufcjt6zB1g01s+tBALdI//LObcCaCLQJVL68X7s3UjgNo7pQDqB79v9xfo5FYqI/C9GW+BZBszsNV0AnByjskSEc+6k97cC+BWBL2/l3c3X3t8Kb/Fg9Y/W/RKpepZ5z/vPH3ecc+Xel/Uu4AcEjjcMvc5VBLo5JPSbP+bMLJHAF6yfO+d+6c2e8Md6oHrHwvEG8Lp5vUzgvo9g5eypm/d6FoEuPhPtuhaLdKwGMcRrYiy7CrjFzIoJdNe+nkAL6Li99o2RMqDMOdfdq+ZpAoGozqlz/RlwzDlX6ZxrB34JXInOqcEM9fvakERb4PkWMN/LRpVEIEHFljEu07CZWbqZZXY/B24E9hKoU3cWz7uAZ73nW4CPepmlLgfqvGbwF4AbzSzb+3XnRm/eeBeRenqvNZjZ5d79IB/tta5xpV9/+FsJHG8I1Pl2C2T+nAvMJ5BEZ8Bz3ru/cRtwm/f+3vtvzHj7/4fAfufcv/Z6aUIf62D1nsjH28xyzWyS9zyVwAf8foKXs/c5cBvwklevIe2Lka+ZDIOOVRDDuCbGLOfcl5xzBc65OQTOoZeccx9mnF37xppz7jRQamYLvVk3AO+ic2ogx4HLzSzN+1/s3lc6p4Ib6ve1oXHjIKvSUB4EsiodJHAv0ZfHujxh1qWQQPa/3cC+7voQuKfh98Ah7+9kb74B3/Pq/g6wute6Pk4gKcdh4GNjXbcB6voEga6G7QR+NflEJOsJrCbwpf4I8F3Axmmdf+bVaY/3Tzyt1/Jf9sr/Hr0ytQY7573zZ7u3L/4HSB4Hdb6aQNeLPcAu73FzDBzrYPWesMcbWArs9Oq2F/jKYOUEUrzpw97rhcPdF3qMv4eOVdD9MqRroh49++06zma1HVfXvvHwIHD7RpF3Xv2aQPZ3nVMD76uvAQe8z6mfEcigrnPKRe67+VAe5q1MREREREREZEREW1dbERERERERiTIKPEVERERERGREKfAUERERERGREaXAU0REREREREaUAk8REREREREZUQo8RUREREREZEQp8BQREREREZER9f8B7GkipOp2gjAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axs = plt.subplots(1,2)\n",
    "fig.set_size_inches(16,4)\n",
    "axs[0].plot(np.arange(len(loss_v_node)), loss_v_node)\n",
    "# axs[1].plot(np.arange(len(loss_v_edge)), loss_v_edge)\n",
    "axs[1].plot(np.arange(len(acc_v_node)), acc_v_node)\n",
    "# axs[3].plot(np.arange(len(acc_v_edge)), acc_v_edge)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  1 , loss:  1.4867796897888184 , node accuracy:  10.348211161853333 %, lr:  [0.0015]\n",
      "Epoch:  2 , loss:  0.6890468597412109 , node accuracy:  11.420300263816676 %, lr:  [0.0015]\n",
      "Epoch:  3 , loss:  26.742918014526367 , node accuracy:  11.963833091554985 %, lr:  [0.0015]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA54AAAD4CAYAAACEyjk9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdeXxU1d348c+XsIusomAAUUQRXEDRahXrXqwbtNqiz+NPn9pqF1uXtk/p4lK1rVXbalu0xaqPtba4lCAKgoogiIogCfsWWTMhJGEJCSHrnN8fmcBkMsu9M/fOvTPzffvyRTJz77nfu8zkfO859xwxxqCUUkoppZRSSrmlg9cBKKWUUkoppZTKbpp4KqWUUkoppZRylSaeSimllFJKKaVcpYmnUkoppZRSSilXaeKplFJKKaWUUspVHdO5saOOOsoMHTo0nZtUSimVxT777LNKY0x/r+PIZPq3WSmllJNi/W1Oa+I5dOhQli1bls5NKqWUymIiss3rGDKd/m1WSinlpFh/m7WrrVJKKaWUUkopV2niqZRSSnlARMaLyAYRKRaRyVHev1BElotIk4hcH/b6aBH5WETWiMhKEflG2HvHi8gSEdkkIq+ISOd07Y9SSikVjyaeSimlVJqJSB4wBbgSGAncKCIjIxbbDtwK/Cvi9Vrg/xljRgHjgSdFpHfovd8BfzTGDAf2Are5swdKKaWUPZp4KqWUUul3DlBsjNlsjGkApgHXhS9gjNlqjFkJBCNe32iM2RT6uRQoB/qLiACXAK+HFn0RmODubiillFLWaOKplFJKpV8+sCPs95LQa7aIyDlAZ+BzoB+wzxjTlKhMEbldRJaJyLKKigq7m1VKKaVs08RTKaWUSj+J8pqxVYDIQOAl4H+MMUE7ZRpjphpjxhpjxvbvr7PRKKWUcp8mnkoppVT6lQCDw34fBJRaXVlEegKzgF8aYz4JvVwJ9BaR1qnSbJWplFJKuSlh4ikiXUXkUxFZERpB71eh1/9PRLaISFHo/9Huh6usKKuqY966XV6HoWL4dMsedu2v8zoMpSxbsKGciup6r8PINkuB4aFRaDsDk4CZVlYMLV8A/MMY81rr68YYA8wHWkfAvQV4w9GolVJKZZXmoOHDTZU8+vZ6Wv6MuKdj4kWoBy4xxtSISCfgQxF5O/TeT4wxr8dZV3ngq08vprSqjq2PXuV1KCqKr//tY3p168SKB67wOhSlEjLGcOsLSzmh/xG8/6OLvA4naxhjmkTkTmAukAc8b4xZIyIPAcuMMTNF5GxaEsw+wDUi8qvQSLZfBy4E+onIraEibzXGFAE/BaaJyCNAIfBcevdMKaWU3xljWLezmhlFAd4oCrBrfz1HdunILV88joG9urm23YSJZ+gOak3o106h/91Nh1VKSqu0Nc3vqg42eh2CUrZsrjjgdQhZxxgzG5gd8dr9YT8vpaW7bOR6/wT+GaPMzbSMmKuUUkq1sbPqIDMKS5lRGGDDrmo65QkXnXw0E8fkc8mIo+naKc/V7Vtp8Wydb+wz4ERgijFmiYh8F/i1iNwPzAMmG2Pa9cUSkduB2wGGDBniWODZpLqukU55HVw/2UoB1DU267WmlFJKKZUD9tc1MmdVGQWFAT7Zshtj4Kzj+vDwhFO5+rSB9Dmic9pisZR4GmOagdGhCaoLRORU4GdAGS3DuE+lpXvPQ1HWnRp6n7Fjx2pLaRSnPfgOA3p25ZOfX+p1KCrLfbZtD1975mP+8c1zuPCk7BrJcuOuaq7440IKvvdFxgzp43U4WcPlxz2UUkop5bCGpiALN1ZQUBTgvbW7qG8KcvxRR3D3pScxYcyxHNfvCE/ispR4tjLG7BORBcB4Y8wToZfrReQF4MdOB5dLynSwGZUGn27ZC8DizyuzLvGcv74cgLdXl2niqZRSSqmcYoyhcMc+ZhQGeHNFKXtrG+l7RGcmnT2YiWcO4oxBvRCJNutW+iRMPEWkP9AYSjq7AZcBvxORgcaYndKyBxOA1S7HqpRSSimllFIqZGvlAQoKA8woCrBtdy1dOnbg8pHH8NUz8xk3vD+d8vwze6aVFs+BwIuh5zw7AK8aY94SkfdDSakARcB3XIxT+VAwaJi9eidfOXUgHTp4ewfFj+asLuOik/vr85TKVz7cVMnZx/ehS0dr16X2tFVKKaX8Zc+BBt5aWUpBYYDC7fsQgfNO6MedF5/I+FMHcGTXTl6HGJWVUW1XAmOivH6JKxGpjDFt6Q5+XrCKhyc0cvO5x3kdjq8Ubt/Ld/75GTeeM4TffvU0r8NRCoDVgSr++7kl/L/zjuOh6071OhyllFJKWVTX2Mx763YxozDAgg0VNAUNIwYcyc+uHMG1o491dRoUp9h6xlOpcOXVLc+l6sTy7e2vawKgZG+tx5Eoddje2gZAp0ZRSimlMkEwaPhky25mFAZ4e1UZ1fVNHNOzC7ddcDwTxuRzysCeXodoiyaeSimlYjI6rK1SSimVVhvKqikoDPBGUYCdVXX06NKR8acOYOKYfM49oR95GfqImyaeSimllFJKKeWhXfvrmFlUyvTCAOt27ievg/Clk/rz86+cwmWnHEO3zpk/Zogmnkq5wPetRD4PTymllFIq29XUNzFndRkzCgMs/rwSY2D04N786tpRXH36QPr16OJ1iI7SxDOLGWNcna/H77mVUip1+jFXSimlnNPYHOTDTZUUFAZ4Z20ZdY1BhvTtzg8uGc6E0cdyQv8eXofoGk08lXKB1xP0JuTz8JRSSimlsoUxhpUlVRQUBnhzRSm7DzTQu3snrj9rEBPH5HPmkD7+rzs6QBNPlbQc+HwkTbvaKqWUUkrlth17aplRGKCgKMDmigN07tiBy045mgmj87no5KPp3LGD1yGmlSaeWcwYTQ6VUqnx+z0UpZRSyk/21TYwa9VOCpYHWLZtLwBfOL4vt487gStPG0ivbp08jtA7mngqlYv0hoRSSimllCPqm5qZv76c6csDzN9QTmOzYfjRPfjf8Sdz3eh88nt38zpEX9DEU6lcpK1YSnlORMYDTwF5wN+NMY9GvH8h8CRwOjDJGPN62HtzgHOBD40xV4e9finwONABqAFuNcYUu70vSimVa4JBw9Kte5hRFGDWyp3sr2ui/5FduOW8oUwYk8+oY3vmxHObdmjimcU0t/COftGobGH0m8QVIpIHTAEuB0qApSIy0xizNmyx7cCtwI+jFPE40B24I+L1Z4DrjDHrROR7wC9DZSillHJAcXk1BYUBZhSWEth3kO6d8xg/agATxuTzxWH96JiXW89t2qGJp1Iu8P3gQpoXK+W1c4BiY8xmABGZBlwHHEo8jTFbQ+8FI1c2xswTkYuilGuAnqGfewGljkatlFI5qLy6jjdX7GRGYYBVgSo6CIwb3p+ffPlkrhh1DN07a0plhR4llTS/51YqDj13SnktH9gR9nsJ8AUHyv0WMFtEDgL7aemO246I3A7cDjBkyBAHNquUUtmltqGJd9bsoqAwwKJNFQQNnJbfi/uuHsk1Zwzk6CO7eh1ixtHEM4u1tLpp05ZSKnl6g8k10b6cnTja9wBfMcYsEZGfAH+gJRltuyFjpgJTAcaOHatnWSmlgKbmIIs/382MwgBz15RR29BMfu9ufPeiYUwck8+JRx/pdYgZTRNPlTR9jDGD6blTymslwOCw3weRYrdYEekPnGGMWRJ66RVgTiplKqVUtjPGsKZ0PwWFAWauKKWiup4ju3bkutHHMnHMIMYe14cOHbTi5ARNPFXStCUkg2XhucvCXVLZbSkwXESOBwLAJOCmFMvcC/QSkZOMMRtpGbhoXYplKqVUVirZW8sbRaXMKAywqbyGTnnCJSOOZuKYfC46+Wi6dsrzOsSso4lnFtOKuPd0dFvlJ3qzyD+MMU0icicwl5bpVJ43xqwRkYeAZcaYmSJyNlAA9AGuEZFfGWNGAYjIImAE0ENESoDbjDFzReTbwH9CAxLtBb7pwe4ppZQvVR1s5O1VOykoDLBkyx4Azh7ah19PPJWrThtI7+6dPY4wu2niqZKmOVVivh/dNovo5agyjTFmNjA74rX7w35eSksX3GjrjovxegEtyapSSimgoSnIgg3lFBQGmLe+nIamICccdQQ/uvwkJozJZ3Df7l6HmDM08VRJ05wq8+jNgtym518ppVQuMMawfPtepi8PMGvVTvbVNtLviM7cdM4QvnpmPqfl99JeaR5ImHiKSFdgIdAltPzrxpgHQs+lTAP6AsuBm40xDW4Gq+zRxNB7+qWm/CSZ7wT9HlFKKZUpNlfUMKMwwIyiUrbvqaVrpw5cMXIAE8/M54ITj6JTXgevQ8xpVlo864FLjDE1ItIJ+FBE3gbuBf5ojJkmIn8FbgOecTFWy34zex079tTyzH+f5XUoWU1zKhXL3gMN3PbiUv5805nk9+7mdTiu211Tzz2vruDJb4ym7xH6fIhSSimVLpU19by1opSColJW7NhHB4HzTzyKuy4dzpdPHUCPLtrB0y8SngnT8pBaTejXTqH/DXAJh0fgexF4EJ8knlMXbvY6BF85UN/E1IWb+cElJ9LRwTs92hLincC+gwzo2ZU8nw7vXVAYYPn2fTy7cDMPXjvK63Bc938fbWXhxgr++ck2fnjpcK/Dicmtm0XBoGFTeQ0nD9D5zZRSSrnvYEMz767bxYzCAB9srKA5aBg5sCe/+MopXDv6WI7p2dXrEFUUlm4BiEge8BlwIjAF+BzYZ4xpCi1SAuS7EqFKmgmNa/v7dzby/OIt5PfpxtfHDk6wlvK7sqo6zn/0fe648AR+9pVTvA5HZZCkutpaGB/72UWb+e3b6yn43hcZM6RPEpEppZRS8TUHDZ9s3s305QHmrN7JgYZmBvbqyrfHncDEMfl68zMDWEo8jTHNwGgR6U3LaHnRartRaycicjtwO8CQIUOSDFOl4mBjy/2Bxuagx5EoJ1TW1AOwaFMlP/M4FqUAVgaqACjZe1ATT6WUUo5at3M/BYUB3igKsGt/PUd26chVpw9kwph8zj2+Hx182vtLtWer07MxZp+ILADOBXqLSMdQq+cgoDTGOlOBqQBjx47VzplZSD/uSmUGt5/L1i94pZRSTthZdZA3ikqZURhgfVk1HTsIF53cn/uvHsSlpxxN1055XoeokmBlVNv+QGMo6ewGXAb8DpgPXE/LyLa3AG+4GaiyL13PYGplU6nMoKPaKqWU8qvqukbeXl3GjMIAH2/ejTEwZkhvHr5uFFedfqwO3pcFrLR4DgReDD3n2QF41RjzloisBaaJyCNAIfCci3EqB4i2TSqllFJKKZ9obA6ycGMFBYUB3l27i/qmIMf1685dlw5nwuh8hh51hNchKgdZGdV2JTAmyuubgXPcCEplFk1n29NGIuVHOgWSUkoprxljKNqxjxmFAd5cuZM9Bxro070T3zh7MBPG5DNmcG+dBz1L6cQ2KmWaZCmVGdzuNqvVBKWUUrFs232AGYWlzCgKsKXyAF06duCykccwcXQ+F57Un84dnZvyT/mTJp5KKaVispOr6k0opZRS4fYcaGDWytJD83uLwLnH9+O7Fw1j/KkD6Nm1k9chqjTSxFOlTFs5lMoM2nNJKaWU2+oam5m3rpyCwgALNpTTFDScfMyRTL5yBNeecSzH9u7mdYjKI5p4ZrHWbnVud6/TVo72/F6/13OWm3SEWqWUUm4IBg1LtuxhRmGA2at2Ul3fxDE9u/DNC45nwuh8Thl4pD63qTTxVMoNWr9X2cJotqqUUiqGjbuqmb48wMyiAKVVdRzROY/xpw5k4ph8zhvWj7wOmmyqwzTxVCnTr5TMo+fMWZmSm+nNZqWUUqkq31/HG0Utz22u3bmfvA7ChcOP4qdXjuCKkQPo1jnP6xCVT2nimcVMRLubVjpVqwzJk5TD3EqQ9aslOSIyHngKyAP+box5NOL9C4EngdOBScaY18PemwOcC3xojLk67HUBHgFuAJqBZ4wxf3J7X5RS2e1AfRNzVpcxoyjA4uJKggbOGNSLB64ZyTVnHMtRPbp4HaLKAJp4KqVUirL5po6VXFVvZNgnInnAFOByoARYKiIzjTFrwxbbDtwK/DhKEY8D3YE7Il6/FRgMjDDGBEXkaIdDV0rliKbmIIuKK5lRGOCdNbs42NjMoD7d+P7FJzJhTD7D+vfwOkSVYTTxVCqN/PK8XBbnSSqObE6QM9A5QLExZjOAiEwDrgMOJZ7GmK2h94KRKxtj5onIRVHK/S5wkzEmGFqu3PHIlVJZyxjDqkAVBYUB3lxRSmVNA726deKrZ+YzcUw+Zx3XRwcJUknTxDOL+STHyWl+/WrWSyM36XeCr+QDO8J+LwG+4EC5w4BviMhEoAL4oTFmU+RCInI7cDvAkCFDHNisUiqT7dhTyxtFAQoKA3xecYDOeR249JSjmTAmn4tO7k+XjvrcpkqdJp45QCub3tFDnxuy+TOWzfvmsWj3pZw42l2AOmPMWBH5KvA8MK7dhoyZCkwFGDt2rJ5lpXJQVW0js1btpKCwhKVb9wJwzvF9+da4E/jKqQPp1b2TxxGqbJOTiefOqoMM7JW5k9eW7K2lrrGZE48+0tZ6brW+aY3FOr9U4t24FowxvLN2F5eMaPtI2bMLN3PB8KM4ZWBPF7aaXQ7UN9EprwOdO3ZwpXwne0cdqG9iz4EGBvft7tuWfZ8roeVZzFaDgFKHyv1P6OcC4AUHylRKZYn6pmbmry+noDDA/PUVNDQHGdb/CH7y5ZO59oxjGdy3u9chqizmTu3Gx+at28V5v32f99fv8jqUqHbsqaU5GD87ueB387nsDwvTFJEz6puamfj0YpZv3+t1KGnlZoU8mEIWm2jN+99YzZT5xbbKfH99OXe89Bl/fr/ter+evY6r/rTIZoSZxamEbtQDc7n5uSXOFBZFqjc+9h5ooKm55XHDSVM/Ydxj822XMWf1TiZN/ZjqusbUgsl8S4HhInK8iHQGJgEzHSh3BnBJ6OcvARsdKFMplcGCQcOnW/bws+mrOPuR9/jOP5fz2bZ9/Pe5x/HWDy7gvXu/xPcvPlGTTuW6nGvxXFFSBcDKkiouGXGMx9G0Fdh3kHGPzeeOL53Az648JeXy0tW4ZqXOvbGshsLt+7j/jdW89YN2vb5UEtaXVbtW9j8+3gbA9y8+0fI6uw80ABDYe5De3dp2z0lwL8URXjYmO9mSvWTLHucKc0LYvo15+F1uOGsQj99wBqsCVUkVt7Oqjk8270l4gy3bGWOaROROYC4t06k8b4xZIyIPAcuMMTNF5GxaWi37ANeIyK+MMaMARGQRMALoISIlwG3GmLnAo8DLInIPUAN8K/17p5Tyg+LyGmYUBphRFKBk70G6dcrjy6OOYeKZgzh/WD865uVc+5PyWM4lnn5WWV0PwMef7/Y0jmDQsL+ukd7dO1ta3k710S9dTb3il93XrpG5yYmW2bdW7uTxG86I+p6VUZtz/TsgnDFmNjA74rX7w35eSksX3GjrRr2DZ4zZB1zlYJhKqQxSUV3PmytKKSgMsCpQRQeBC4b350dXnMQVIwdwRBet+ivv6NWn2nlq3iaemreJpb+4jP5H6oTAqYhXx/7q04vZfaCBD35ycdriaZWNdX9NphNzIuk72NjMr2etTbxgAqJnTCmlHFHb0MS7a3cxfXmAD4sraQ4aTs3vyS+vOoVrzziWo3t29TpEpQBNPLNaa+uDsZlmzF1TBrTcNbOSeGr1MTnLt+/zOgRH6XWQnaJ9fzy7aEvUZa3M7ZaNNz2UUirdmoOGxcWVzCgMMGdNGbUNzeT37sYdF57AxDH5DD/G3gCUSqWDJp4+pF3RskdkNdxKV0SVeTJlLm1fxemnWJRSKgMYY1hTup8ZhQFmriilvLqeI7t25NozjmXimHzOHtqXDh30y1X5lyaeSuUQ/XPkjky5n5ApcSqllDossO8gbxQFmFEYYOOuGjrlCRedfDRfHZPPxSOOpmunPK9DVMoSTTx9yKlWicg6ppetHb5qaclh6cg77HbtVv5mJ1m1NriQXh9KKZVI1cFG5qzeSUFhgE82t4x2ftZxfXhkwqlcddpA+hxhbQBIpfwkYeIpIoOBfwADgCAw1RjzlIg8CHwbqAgt+vPQCH1KqRi0yq285KcbQH6KRSml/KChKciCDeXMKArw3rpyGpqCHH/UEdx7+UlMGJ3PkH46z6bKbFZaPJuAHxljlovIkcBnIvJu6L0/GmOecC885QU354d02nVTFtO/R2f+fsvZXoeSEdJd1/dT69b6sv189emPeP9HFzGglzsj/Pk9l/LR6VBKKUXL38nl2/dSUBjgrZU72VfbSL8jOnPTOUOYMCafMwb1sjRwm1KZIGHiaYzZCewM/VwtIuuAfLcDU6nLhUrmih3ZNTKs29y8JPz+h/HFj7ZR29DMvPW7+K8vHOfKNrLxI+fWPvn7alFKKXdtqTxAQWHLc5vb99TSpWMHrhg1gIljjmXc8P50yuvgdYhKOc7WM54iMhQYAywBzgfuFJH/ByyjpVV0b5R1bgduBxgyZEiK4TogF7Ix5Vs5cfnlwj5mKD/cG8iJz4BSSkWxu6aet1a2PLdZtGMfIvDFYf34wSUnMv7UARzZtZPXISrlKsuJp4j0AP4D3G2M2S8izwAP01LNfBj4PfDNyPWMMVOBqQBjx47VKocFTlfMnB4cJBVa6fSWm3mHD3KaBPTic+vzl0xrt99byJVSygl1jc28u3YXMwoDfLCxgqagYcSAI/n5V0Zw7Rn5rj36oZQfWUo8RaQTLUnny8aY6QDGmF1h7z8LvOVKhE7zqLLT0BREhPR2nYioZIqF1KCxWSvnjohxGBubg+mNI0K6z66frqbWpMvK5yAT1Tc106Wj80PqOz1SrY56rJTKds1Bw5LNu5leGGDO6jJq6psY0LMrt407ngmj8zllYE+vQ1TKE1ZGtRXgOWCdMeYPYa8PDD3/CTARWO1OiNnh1Afn0r1zHkX3X+F1KHHVNzUf+lkriM770uMLvA7BdeFXTa61cHuV0r67dhff/scy3rzzAk4b1Cvmcn5qZPRRKEop5Yj1ZfspWB7gjaJSyvbX0aNLR648dQATx+TzhRP6kddBv/lUbrPS4nk+cDOwSkSKQq/9HLhRREbTUs/cCtzhSoRZoqEpSEOTt61dVtQ1+j/GTFZZU+/p9vVPnru8yrMXbCgHoKhkX9zEM9duBCillNt2Vh1kZlEpBYUB1pdV07GD8KWT+vOLq07hslOOoVtn53uiKJWprIxq+yHR66s6Z6dLnGqVSKbFMrzFM1GXRK3DZp7Wc7a4uBJjTE4+Z5eDu5wSpz/nmvwqpTJddV0jc1aXUVAY4OPNuzEGRg/uza+uHcXVpw+kX48uXoeolC/ZGtU219z7ahHTlwfY+uhVXoeSEjv1PLdbZXOmzunz5GZN6X5eWbqDSec4N9J0tIQu/OaH14luuhKem59bQs9unZhy05mOlz108ix+8uWT+f7FJya1vluHP7nBhVwIRCmlXNLYHGTRpgqmLw/w7tpd1DcFOa5fd354yXAmjMnn+KOO8DpEpXwv9xJPG7XP6csDLgbiT3bq5lpvjMOnGXb4Oduxtzat224KGjrlZf9Vs2hTJQBTbnKn/Mfnbkg68fRDa6MPQvANERkPPAXkAX83xjwa8f6FwJPA6cAkY8zrYe/NAc4FPjTGXB2l7D8D/2OM6eHiLiiVEz7dsod7Xy2iZO9BenfvxNfHDmbCmHzOHNI7J3sOKZWs3Es8M4BTlcNkyrE19Yr94pWPpGN0Vz8kOumUjdUP16ZgycqjZZ2I5AFTgMuBEmCpiMw0xqwNW2w7cCvw4yhFPA50J8r4CiIyFujtdMxK5ZrG5iBPvbeJpxcUM7hvd/5281lcfPLRdO6YxhkKlMoiuZd45vKdqRzedZU+4VNr+DHvdPNj4Mf99ZtcuxkRxzlAsTFmM4CITAOuAw4lnsaYraH32j0DYYyZJyIXRb4eSmgfB26iZcR5pVQStlQe4O5phawoqeKGswbxwLWj6NEl96rNSjlJb9moCNZrhW5U4KsONvK1Zz5i++70dgPNNq8t28HqQJXXYThyjdQ1NnP7P5axpfJASuXo9ECH+eH+mx9i8Fg+sCPs95LQa6m6E5gZNt1ZVCJyu4gsE5FlFRUVDmxWqexgjOHVpTu46k+L2FJ5gCk3ncnjN5yhSadSDtDEM4u5Xc12o/y5a8r4bNte/vT+JhdKzx0/eX0lV//5w7jLpKPi78Q1smTLHt5Zu4v73/D/VMGe51IWmxPtdanXhN0l0S6XlA62iBwL3AD8OdGyxpipxpixxpix/fv3T2WzSmWNfbUNfO/l5fzvf1Zy+qBezLn7Qq46faDXYSmVNfT2jWojvEKqFc7U5VKrTtRRbXPsEsqx3U2Kfq8cUgIMDvt9EFCaYpljgBOB4tCAJ91FpNgYk9xoVErlkI+KK7n31RVU1tQz+coRfHvcCeR1yKE/4kqlgSaeKml2vo5bkxKTY5lIju0uEDv58suxyOqbARZ3LquPQeZYCgwXkeOBADCJlucyk2aMmQUMaP1dRGo06VQqvoamIL9/ZwNTF23m+H5HUPC98zltUC+vw1IqK+Ve4umX2m8cTlUKW5O8ZEeqTTTqpP+PpIonPbmHf66SDPjop42tY+HwcdPz0MIY0yQidwJzaZlO5XljzBoReQhYZoyZKSJnAwVAH+AaEfmVMWYUgIgsAkYAPUSkBLjNGDPXm71RKjMVl9dw17RC1pTu56YvDOGXV51C9865VzVWKl300+VDWjHLHtqydJh2sVTh9LMBxpjZwOyI1+4P+3kpLV1wo607zkL5OoenUlEYY3h5yXYembWWbp3ymHrzWVwxakDiFZVSKdHEU7WhSW8Ocbjm39pC3uY54TReT364dF2dqsXBg6lJn1IqV+2uqeen/1nJe+vKGTf8KJ644QyO6dnV67CUygmaeGaxyGqq1jWVl/xyUyNRF/JU+GQXE0q2+72T3DwPSikVzQcbK/jxayuoqm3kl1edwjfPP54OOoCQUmmTe4lnht7qb2gKEjSGrp3yXN2One6QtgYX0kqmclmiKyxTksJsl2sDjCmlvFfX2Mxjczbw/OItDFKeDLUAACAASURBVD+6By/+zzmMPLan12EplXN0Hs8MMf6phYy4b05S6yb7bF2i9dyuPr70yTaGTp5FQ1PQ5S3lJqdvBTg9ncq3XlzGX2zM5+qHdMbdrrbOleXk/bdkE8kMvQeolMowG8qqmTBlMc8v3sIt5x3Hmz+4QJNOpTyiiWeG2FxxwPY6ydQH/dQY8ft3NgBwoL7J40iUF95bt4sn3tnodRhZyVZXWx99JyillFXGGP5v8Rau+cuHVNbU88KtZ/Or6051veeYUiq23OtqqxyjDRaZLR0tTuGt5m4nMAm72rZuP0MvXL/mf3bPqyaySim3lVfX8ZPXVvLBxgouPrk/j11/Bv2P7OJ1WErlPE08fcTtREAc3oDWH1U0fr8uXO0O62LZTvJDN1cfhKCUykLz1u3if19fSU19Ew9dN4qbzz3O8fqPUio5uZd4+vh2u9OhJfNsZ3gMOiBQ8nTOyvb0mKTGyUF57I1q6+x506tAKeWGgw3N/Gb2Ol76ZBsjBhzJv28/l5OOOdLrsJRSYXIv8VRKAem5seDj+zyuyMVbNYbk9ltbIJRSTllTWsVd04ooLq/hWxccz0/Gn0yXjvosp1J+k3BwIREZLCLzRWSdiKwRkbtCr/cVkXdFZFPo3z7uh+sADyo7S7fuSfs2k5VM64aVQ5ordcynFxRz83NLtLU4JPxq8joJtXtt/27Oev7f85+6FI19Th4+Lz+PXl8HSqnsEQwanl24mQlTFrP/YCMv3XYOv7x6pCadSvmUlRbPJuBHxpjlInIk8JmIvAvcCswzxjwqIpOBycBP3Qs1c72/vtybDaepgqcVycMem9MyEu83zz/e40gSy5WbAZGstrQ9s+BzlyPxjpOf2aSnU3EuBKVUDiqrquNHrxWxuHg3l488ht997XT6HtHZ67CUUnEkTDyNMTuBnaGfq0VkHZAPXAdcFFrsRWABmnj6k06doNKsNbcLv548v7Q8DyA1Xn029TtBKeU3c1aXMXn6Suobg/z2q6cx6ezB2n1fqQxg6xlPERkKjAGWAMeEklKMMTtF5OgY69wO3A4wZMiQVGJVaWa1a6J+17eXywPpGGM0WfGKxQPv5GfW7qnO5c+GUio1B+qbePittUxbuoPT8nvx5KTRDOvfw+uwlFIWWU48RaQH8B/gbmPMfqt3lowxU4GpAGPHjtUaRxxOJ3DpOtiaZGSO8GvM6fsF0b4T2s7jqRdKKpxM2PxwKvSGlVLKjhU79nH3K0Vs3X2A7140jHsuO4nOHRMOVaKU8hFLn1gR6URL0vmyMWZ66OVdIjIw9P5AwKMHGW3yQ43LI8nU8+oam/nRqysor65zJIZkDn/unjHnGGNoDmpLJOizhXYlumSiXVNlVXU89OZamoPt39RrUCllR3PQMGV+MV975iPqGpv517fO5afjR2jSqVQGsjKqrQDPAeuMMX8Ie2smcEvo51uAN5wPT6VbZKXwrZU7+c/yEh59e33MdbTlwv9+/85Ghv18Ng1NQde3FSuv8Drf8Hr7qXIyYXP7M/uT11fw/OItLNm8O04M+sWhlIovsO8gNz77CY/P3cCXRw1gzl0Xct6wfl6HpZRKkpXbRecDNwOXiEhR6P+vAI8Cl4vIJuDy0O/+5+PKjtMtAa3lJVus1SlB3G7B8O8Zi81v06n8c8k2AGobmw+95vRHIWpxmZ7tZSlHR7Vt05265d/Wls5om9FL4jARGS8iG0SkODQ6fOT7F4rIchFpEpHrI96bIyL7ROStiNdfDpW5WkSeD/VYUirjvLmilPFPLmRNoIrHrz+dv9w0hl7d9XJWKpMlTDyNMR8aY8QYc7oxZnTo/9nGmN3GmEuNMcND/2bOZJUO+vuizRk1T2cibg384eN83xXJHse1pft5ekGxw9Ec5mU3x3jbNsbw1HubKNlbm76AlCWpPJur3WpjE5E8YApwJTASuFFERkYstp2Wqcv+FaWIx2m5KRzpZWAEcBrQDfiWQyErlRbVdY3c+2oRP/h3IcP692DWD8dxw1gdtVapbGBrVFvV3iOz1gGw9dGrPI7EO1b+FmgF1Jqv/GkRAN+76ETXt5WOP+JWT/vW3bX88b2NzFlTxtt3jXM1pqyuu1jcOUdHtQ07yZbK1S+DVucAxcaYzQAiMo2WacrWti5gjNkaeq9dH3ljzDwRuSjK67NbfxaRT4FBTgeulFs+27aXe14pomRvLT+85ER+cOlwOuXps5xKZQv9NCfhzRWl/OX9TV6HkVBkq5vdBNGN1k+dSsEb+2obgTQlXSZ8Hk9rE3kGQ8vVh3UFdjysJBOeYNAwdeHn1NQ3xS47Dde1o91j0/QxjHW9ZXXyb10+sCPs95LQa44IdbG9GZgT4/3bRWSZiCyrqKhwarNKJaWpOchT723i63/7mOag4ZU7zuPeK07WpFOpLKMtnkn4wb8LAbjzkuGOlpvNlbFcbeRw45RmwrHMhBitenfdLn4zez1bd9fym4mneR1O2qVyLrPpOnBBtK8HJ4/Y08BCY8yiaG/qVGfKL3bsqeXuV4r4bNteJow+locmnErPrvosp1LZKPcST60JxaVHJ7t5+oxnhl5ddaFW2Oq62C2e6RhMysnj59ZNLivXV2ZeBa4oAQaH/T4IKHWiYBF5AOgP3OFEeUq5wRjDjKIA981YgwBPfmM0E8Y41uivlPKh3Es8c4ifcuxUKs0+2g0VR7Rkxo/nLpt7Fljl9XeDngIAlgLDReR4IABMAm5KtVAR+RbwZeBSY4z78ycplYSqg43cN2M1M1eUMva4PvzxG6MZ3Le712EppVymiadqI5URLFV7bhzNVJL48KQr3QmY15dWpl/ZXh+/WKLFFe/a8ut+pJsxpklE7gTmAnnA88aYNSLyELDMGDNTRM4GCoA+wDUi8itjzCgAEVlEy+i1PUSkBLjNGDMX+CuwDfg4NIDYdGPMQ2nfQaVi+HTLHu55pYiy/XX86PKT+O5Fw+ioz3IqlRM08cwBfkgmU3tOzPv4080Yk9FDxx8eXMjbOKJxo1tspnUjdvvSSnTeM/nadlJoBNrZEa/dH/bzUmKMSmuMiTr8szFG/64rX2oMDSD09IJiBvftzuvfOY8xQ/p4HZZSKo1y7w9UDlV4kqkKh6+TjufWsl2yR9CY2JeqH5O5cAYTNUafh+17Th4/R0fItRlZpiXpSqnUbak8wN3TCllRUsUNZw3igWtH0aNL7lVBlcp1+qlXvpeJ1VS/J4eQfTcWEh1yN8+JE8fSry37VsKKvEmS6P5edl15SqlYjDG8tqyEB99cQ8cOwpSbzuSq0wd6HZZSyiOaeOYQu/N4Ku9ky2kI3w+/JFZudHpIzzye/hzV1hh73xs+uQyUUi7bV9vAz6av4u3VZZx7Ql/+8PXRHNu7m9dhKaU8lHuJZxK1nkx93i7ViqoblelkQsrEimqqMbecu+jXnFOHw+lL2usW1Mz7hLaVzuvc689UBn6dKqVs+Ki4kntfXUFlTT2TrxzBt8edQF4H/eArlet0GDEfqm1oYtbKnR5t3Z0aaUqDC2VN+5/3op2H4vIatlQecGUb4Tc/nDiLi4sraWyOPkNEtl8lXu1fos+fQRNJpVSLhqYgv529jv96bgndO+dR8L3z+c6XhmnSqZQCNPG0JN2tA59XHOD7/1pO0Y596d2w8o14l5zTXVYv+8MHXPzEAkfLTFa8PSvcvo//+vsSnpi7wfGyc42XiaKeB6WyU3F5DROfXszfFm7mxnOG8NYPL+C0Qb28Dksp5SO519U2CalWlJKt4x2ob0ppu635iZ343UqyU2q1zMCaaiaEnI7co+0znqmVVRP6PHxeEb111sv76Y4MLpTofSvHz+JBdnRU2yQK87pbtlLKOcYYXl6ynUdmraVbpzym3nwWV4wa4HVYSikfyr3E04Nb/cnW8ZxOAu1W9tpsPxMyqSzi9TN4yQifu/PQx8zmfli7QqMX6uUhy+bu4E53k8/Ea1spFd3umnp++p+VvLeunHHDj+KJG87gmJ5dvQ5LKeVTuZd4JiHeQC/KGruVzTbPCTobSlq4ObBT5Eixdga+Ci/XzXsw0efxTHxMMvFcOyXhNePgwfH8mUyvt6+UStkHGyv48WsrqKpt5L6rR/I/XxxKB32WUykVhyaePuZF5TBm3Vb/lmQMv0xb4p7oF2OiS9TN45JpXUcd7Wrb5ue2BUc7LtncOqxULqhrbOaxORt4fvEWTjqmB//45jmcMrCn12EppTJA7iWeyUyn4kIYUbeTxfWxVHYtE49LqiHH2+e2o8a2v0ER93i5eCyjpV52k4x0pG9+nRop4TOeHiVsKX12Y6ztzzOglEpkQ1k1d00rZH1ZNbd+cSiTrxxB1055XoellMoQuZd4JsGrxMeL7WZikqesCz+9aW+lc+za0os0VU7m3uHfGZnW8quUssYYw4sfbeU3b6+nZ9eOvHDr2Vw84mivw1JKZZiEiaeIPA9cDZQbY04NvfYg8G2gIrTYz40xs90KMle4VZ1ONpn0euTLQ+tqohFTtCPjh6MVPtBQLnDiGk34iKcDx9IXp8MXQSilrCqvruMnr63kg40VXDLiaB67/nSO6tHF67CUUhnISovn/wF/Af4R8fofjTFPOB6RD6VaqfSqDSCZiqrbzwfmSp0z1cMYf317hYef03Q8/2kwMQYXsrKuFcl9onLl2nNaMoMeJWr59GlvZ6VUhHnrdvG/r6+kpr6Jh68bxX+fe5xvH1dQSvlfwsTTGLNQRIa6H4p/pZxEOBNG2vmlpTFXWs3CWT320UZcjpcotOkWmaN1Bzd2Ox1dTDPhY9B63VodlVkp5U8HG5r5zex1vPTJNk4Z2JNpk0Yz/JgjvQ5LKZXhOqSw7p0islJEnheRPrEWEpHbRWSZiCyrqKiItZjC/dYoK4mGa919XSrXv9zb41SmpkmXaF1trcSRyjyeXnKkq20a9suN75hk4tZnQVuIyHgR2SAixSIyOcr7F4rIchFpEpHrI96bIyL7ROStiNePF5ElIrJJRF4Rkc5u74fKLmtKq7jmLx/y0ifb+Pa445nx/S9q0qmUckSyieczwDBgNLAT+H2sBY0xU40xY40xY/v375/k5hyUQ808qVZkE9ZRIxaYuaKUm59bktI2o27G8RL9L5X8IHJVE+c9J0X7aNm9BnPxXFtlKWl04fstmXMSLbF8ddkOHp+7Pgem+7FGRPKAKcCVwEjgRhEZGbHYduBW4F9RingcuDnK67+j5VGY4cBe4DanYlbZLRg0PLtwMxOmLGb/wUZeuu0cfnHVSLp01FFrlVLOSCrxNMbsMsY0G2OCwLPAOc6G5aJkplNJUz3J7mYmTf2Y15btcDaGFPb1h/8uZNGmSsfLVbHZPazpOA+xtuFci16SyZXPr0E3z01DU5DvvfwZn1cccLxsq1P//O/rK5ky/3Mgp+7/xXMOUGyM2WyMaQCmAdeFL2CM2WqMWQkEI1c2xswDqsNfk5aH7y4BXg+99CIwwYXYVZYpq6rj5ueX8OvZ67j45KOZc/eFjBvug8YCpVRWSWo6FREZaIzZGfp1IrDauZDcEQyarKvsfLJ5D59s3sMNYwd7E0ASBzSZ1o5MbCFx87lgmw3Rnhw/P56y1sQ3274HrCjasY/Zq8qYvarM1e1Y6ULrx2vDI/lA+J3DEuALKZbZD9hnjGkKKzM/2oIicjtwO8CQIUNS3KzKZHNWlzF5+krqG4P89qunMenswTqAkFLKFVamU/k3cBFwlIiUAA8AF4nIaFrqwFuBO1yM0REn/Hw2N54zhP497D/ukq5BdpyukCU1qm3YviZc3dYGUphOxaOK6rMLN2Mw3H7hsLRv22qyaGWxWIuko2Jh9xlPi6U6VZBjsnlwITuD2kZ+V8a6xLRKC0Q/DKmeZstlGmOmAlMBxo4d678PlXLdgfomHn5rLdOW7uC0/F48OWk0w/r38DospVQWszKq7Y1RXn7OhVhc9+9Pt/PDS060vV6qFWavK1nxwj9Q30THPNFnOKL49ex1AEklnm7W4mJ2ZTWGg43N5HWIfcW5eRMl2oi5drfm9WclWX4ZAdpvtHUzrhIgvLvKIKA0xTIrgd4i0jHU6ulEmSoLrdixj7tfKWLr7gN896Jh3HPZSXTumMp4k0oplZh+y/hYOiqzox6Yy1V/+jB8o4e8t3YXmytqDv1+zZ8/ZOwj7x1ewEaLWa5WQJNtVTRARXU9Tc3tHu1qY/L0lYx95F0A/rZwMyPvn0tFdf2h95du3RPz2LuZ5CU7j2d9U5D9dY0Jlkot8kwdUTXaMd1ZdZB563Yd+v2+GavZUFbdfsEoKqrrGTp5Fks273YqREty9KsgmqXA8NAotJ2BScDMVAo0LV0l5gOtI+DeAryRUpQqqzQHDVPmF/O1Zz6irrGZf33rXH46foQmnUqptNBvGgtiVZScf3bO2fKsllZcXhP19b/ML+bpBZ8f+n1VoIrKmsNJTc5mkxakemjqGps5+9fv8YuC+I9PT18eoLKmAYA3V7Q0bJRV1R16/4a/fty2+3Q6BhdKYd3AvoOc/uA7rmzB7z0Xkonvmj8v5rYXl7V5bfn2vZbWXbZ1DwAvLN6aKLL47xoT89jE7Gqrz48RapG8E5gLrANeNcasEZGHRORaABE5O/SIyw3A30RkTev6IrIIeA24VERKROTLobd+CtwrIsW0PPOZkT2UlPMC+w5y47Of8PjcDXx51ADm3HUh5w3r53VYSqkcktTgQpnGROsDmOz6bV63uL7tLbbwomXG7bwkmfJzMb+tb2xp6Xx79U5+d/3pbd7z62BL4VFFn8fTn3FbZSV6N/OpaD0g2twIirdunGOfru7X8V7LVcaY2cDsiNfuD/t5KS3dZaOtOy7G65vJpJHmVVq8uaKUnxesIhg0PHHDGXztzHy9AaSUSrscSTy9jsCadqOR+r1Tmp2uti6G4WfJJltOXrMxu9pqncNxqc2/mt5PiVPnP7l5PpVS6VBd18gDM9cwfXmA0YN789Sk0RzX7wivw1JK5aicSDyDKdbiY3a1tbi+V5Ws1qTHTvITc9For6cpo/d9Ah5FqjHHWz92128L5bp4zqIXba2br72wUnzGM1OzHo9GvXZ8tO0M/DwrlYk+27aXe14pomRvLT+8dDg/uOREOuXpE1ZKKe/kSOIZ9ktS80jGet2bZzIzUaa0Ojst1a5MTk9Y06Y7rM1YLMdh3D7f3rciu8Gr+FLt0h8t7oSXfaYm/0plgKbmIFPmf86f3t/EwF5defWO8xg7tK/XYSmlVG4knnqH3bqYx8pGRdEY4+izI35PGKLxIubWGyGRhz5dsaT2OUs9SKslZOL1BJl1YypTj7FSmW7HnlrufqWIz7btZeKYfH513Sh6du3kdVhKKQXkSuKZaiUoVotnisW6LV3xRbaYGBMt+fH70fKX1lb6qLPBp/IcYZpOQ7TBheLJlsvDD914nQ4h0amxe8MhW861Un5ijGFGUYD7ZqxBgKcmjea60fleh6WUUm3kROKZ+jOeqY1qa3k7PqiQJRND5PHx9yQz6eFlzPEGqQr/2ekRDRPN3enc9R09bqt7k+5RXC2v62HZTkt0afkgP1cqa1QdbOS+GauZuaKUs4f24Q9fH83gvt29DksppdrJkcTT2+0nu3k/JKJOyaJdSQsnp7/IhOvI5h4lVUaqCacmSzFkwPWlVLb6dMse7nmliLL9dfz4ipP47kUnktdBv62UUv6UE8ObpdrN8xcFqxn7yLvty3W4xuX4YEWRLV9WRj1NYjvtu9r6+7jkMrtT2s5bt8t62Qnf98d5TPZyal1ta+UBhk6eRdGOfe2WSaUROdF1nsxIx5a2m2BtNz5+On+gUqlpbA7yxNwNTJr6MR3zhNe/cx53XjJck06llK/lXotnEhWeWat2Rn3daoUs2T8DmVI3s9LVNtdyx1ST5WTWjjn6cqyfLWzk4bfWcukpx9iNJGwb1vbEiesjXR+XBRvKAShYXsLowb3TtNX4kjl+bszj6fWUPkrlgi2VB7h7WiErSqr4+thB3H/NKHp0yYnqnFIqw2XFN1V5dR3VdU0M698j6vttKjo+rvS4FVmscitr6tsv68DxcX7eP3sONjRTU99E/yO7OBtIGsU9DzYPSLoq+glb7OLukp0Yo2dMlke1tbGlxFt1TmrPePr3ey2aTLmpppSfGGN4bVkJD765hk55HXj6v87kK6cN9DospZSyLCu62p7z63lc+vsPYr7v9TOeyUo9X4hsiWz7+9hH3kt1AzG26lzSBPaPw9ee+Yizf+3OvqVLMtesleSt7T0Yfw6yY6H05NaKsdqslTsZOnkWpfsO2tqqn1r2fXw/rZ0MClUp39hX28D3Xl7O//5nJWcM6s2cu8dp0qmUyjhZ0eKZiFsV7Eyq7Flla5diNFtEH900fQdr7c79adtWLKleGw42eMZcPh1nJFY333bLpbGrbeT3weuf7QBgfdl+ju3dLfF2XGquS3QM3D5+sbebSkt2dNrgqZR1HxVXcu+rK9h9oJ7JV47g2+NO0Gc5lVIZKStaPBPJlBZPK5XHfbUNcd9/a2VpSuXbkrbMO7ntvLxkGx9srHA4lra2Vh7gxqmfcKC+qc3rqSbayUwBFPMZzzavxy7374s2295mrO3ZncczE0Sr5tU1NvPb2euobWhuWSZNdcHH565nVUnVod+dvLHzwcYKvvbMRzQn8cWpXWiVck5DU5Dfzl7Hfz23hO5d8pj+3fP5zpeGadKplMpY2uKZSrkOVPbsxvbeuvK479/5r8KwspMKyR0exPKLgtUAbH30Kte28djc9Xy8eTcLNlRw1emHuz252eIZc50Ut/HIrHV8a9wJ9jdseXtxRmVNw/Xhxn2Xlz7ext8WOpOwJ97e4T2YMv9znl7weZtYnPLDfxdSdbCR/Qcb6XNE58RxtRlMqu170arHvvpeUsqnistruGtaIWtK93PTF4bwy6tOoXvnnKiyKaWyWE58i7nV4un8IDpuBerAsjbKcPy45GBFNe7UGSn0tbW7rp1updG7WFtcN4Oe/As/JI3BoHMF2x40KulVE5TbUpoTrdYxu9pq06hSURljeHnJdh6ZtZZunfKYevNZXDFqgNdhKaWUIzKyq20waNhaecDy8m0qtRYqPIs2VVAT0XUyG7iVwDU1G178aOvh7USpbjq96abmIK9/VuJwqc7xpMUzxkqxWqTSPYpy5OvF5TUUFPrnHIYfm/Cu0yvDurR6Ke6zlKkkhxHrtv4aOT+vU9vOpJsMbhOR8SKyQUSKRWRylPcvFJHlItIkItdHvHeLiGwK/X9L2Os3isgqEVkpInNE5Kh07ItK3e6aer79j2X8csZqzh7al7l3X6hJp1Iqq2Rki+ef3y/mj+9t5L17L+TEo49MuHybFs8EtaTSfQe5+blPuXxk4rkLnag+tQknosB563Zx4Un9ky87qXVirBWnDvrsos3UNx1u+UnHdCpTF23msTkbnN2Qg1I9BPGe8Uw42IvFbTjdBT3qDYc427jsDy0jUU8cMyitrdpWtjV5+qpDPz/34ZaYy1lNzqxIJSFLbt0EsdvYNbstmNreCSKSB0wBLgdKgKUiMtMYszZsse3ArcCPI9btCzwAjKXlI/+ZiMwEqoGngJHGmEoReQy4E3jQ3b1RqfpgYwU/fm0FVbWN3Hf1SP7ni0PpoM9yKqWyTMIWTxF5XkTKRWR12Gt9ReTd0J3Wd0Wkj7thtrV06x4AdlbVWVo+aKOvbesgIZ+X1yRc1u35EV/8eBurA/5obYknPOmE9EwzUVkdf5Alr6V6baTyjGfk9d62ldP9DC9W0ux1l+lY24+WNFnpUWEw/hlMJ9GIuCl0t09ltN2UY8lu5wDFxpjNxpgGYBpwXfgCxpitxpiVQGSf7i8D7xpj9hhj9gLvAuNpyekFOEJaLuyeQOwR55Tn6hqbeejNtdzy/Kf06d6JN+48n9suOF6TTqVUVrLS1fb/aPmDFm4yMM8YMxyYF/rdtzK5olNd51yXXyuHwYljFXcQmaTKSz4Wr7TGnGzVIZlRbQ9tO957bk67ESo7mdFQ06H1ukw2+T7cBfWwyPPrZJfXZFlPhmN1zY77dsoMOvptSD6wI+z3ktBrSa9rjGkEvgusoiXhHAk8l3qoyg0byqqZMGUxzy/ewq1fHMrMOy/glIE9vQ5LKaVckzDxNMYsBPZEvHwd8GLo5xeBCQ7HZYnVilpSFU0LFSNHuto6XF6bsh0dcSTV1Z3dO79XXFOfTiVO2QmKjkxaYy3e0OzgwDhxtp84kijPGYa94FTPgmYb5Vi9vtJ5HTo5t2u4yH2wm6AbY2ydI7d7imSQqIP+prKuiHSiJfEcAxwLrAR+FrUAkdtFZJmILKuocHfaKdWWMYb/W7yFa/7yIZU19bxw69k8eO0ounbK8zo0pZRyVbKDCx1jjNkJEPr36FgLuvHHzW5lz14DjJ0KlL04vGalwudIi6fF5WobmhLOS9pSXupB7att4GCoG7WTrM2dmVTJya8SL4kLe93KM7J2Pmqtm2lqTv18ufHZav0esFJ2tGUij0Vjk6Giur7tMh7dEAk/xzX1zQydPIuXPtmWYpmhfx3+TmwZCNjnd47SowQYHPb7IKx3i4217mgAY8znpuWieBX4YrQCjDFTjTFjjTFj+/dPfiwBZU95dR23vrCUB99cywUnHsWcuy/k4hExq1BKKZVVXB/V1g9/3FLpthiu6mBjm98dH5zFQnGeVddsbNjqYfnykwsZ/dC7bocDwOiH3uXqPy9KalutduypjR1PjICSvUIOV/rtl5lKa6kTwrvatnm+NG6LnbPds6OWk+pzt60/hE72K8t28Oyi2AMPWVHX2Ex1XWPb8mNu39ox2rW/5dn3FxYnF9uhZ4UtHi9jYg8uFO2Y66i2hywFhovI8SLSGZgEzLS47lzgChHpExpj4YrQawFgpIi0/rG9HFjnwZqPTAAAIABJREFUcNwqSfPW7eLKJxfxyebdPHzdKJ67ZSxH9ejidVhKKZU2ySaeu0RkIEDo33LnQrLOavXFqcr2pb9fQFnYgEZuPw8J7Stp9sYIMSzYUM4nm3fbWMcBUQqJtps79hy0Vly01qckMvDPK6xPwRPpjaIA4x6bz6JN0VvtY01JYUdx2IBWST0Le+jf5K+ZWGXaWbbJ5jOeG3dVO3L8EmlNpGJ95PbXNUZ/w0UTpizmtAffSbmc8H1qDHWh7tQhua/31uMUNDB9eYmtZ3atLOlSD++MY4xpomXE2bm0JIevGmPWiMhDInItgIicLSIlwA3A30RkTWjdPcDDtCSvS4GHQgMNlQK/AhaKyEpaWkB/k+59U20dbGjmvhmrue3FZRzdsytv/eACbj5vqM5nq5TKOclOpzITuAV4NPTvG45F5IJoyV0y0yBU1jRw7m/nHS43pajac6Ml4NYXloaVnx5xW2YyrX9yyIodLaMLbyirZtzwwy33seoNrZV3q1fZ3DVl3PHSZ4fXD9pbHw4f28g8Id2HPGhM1G3GCuOKPy7kje+f366MhCvalCiBuueVFUwcMyi0yWjfGc5bX1Z96OeEN6EsPuPZup95YaNixv9MRv/9tc928NicDUwYfWzcuOKJ2mJvfDQasMeMMbOB2RGv3R/281JautFGW/d54Pkor/8V+KuzkapkrSmt4q5pRRSX1/Dtccfz4y+fTJeO+iynUio3JUw8ReTfwEXAUaE7rw/QknC+KiK30TLP2A1uBpms+qZmunTMi9r1MFGSZ6Ve5EQX3lRLcLoC50RiaCfpSLY8v7Mb87qd+9v8Hr+7bKIEJV6XTPcOZut2w5/xtDxATbuy7G07GDQcSPAM76FnPCNed/IjlM5rVST69lpbnDvmJdqz+O/vrml5/rqipj7ucuHaXXtR4nPq0Qel/CwYNDz34RYem7uePt0789Jt57S5aamUUrkoYeJpjLkxxluXOhyL407+5Ry2PnqVrYqOnTpRrHKHTp7Ff31hCL+eeFrCMv61ZLv1DUYRr+rYLrw01fesbMZOt8ZMfCYs1Yjt7vPnFTVs3d3yDOqPX1tpuawn5iYeYMiu5uDhFi2rn6dVJfva/B4ec6JjEdh3kPMffT/m+//8ZBtdOnaIOp/vgg3l1NTbm7Io7mfOVkn21o33fnjC1xxK/DsmOQ9gqjcBEgkaHVpIZbeyqjp+9FoRi4t3c8XIY3j0a6fT94jOXoellFKeS7arbUZx7QZ7nHJfXrLdUuIZ/vxlMnE6/YyIa4cqYud+Nn1VSuX5/tkYmyczVnfHqMtGee3S339w6OfKyBYqE+Nn4C/ziy3FZ8XhZzyjP8QXb5/ue2ON5WUjzVu3K+77v5yxGoAzh/QOld1S+I49tW26omeaWL2RG0PHv2OSz3i2Ftb6CUt04y5er+hoNw20xVNlszmry5g8fSX1jUF++9XTmHT2YP//vVJKqTTJicTTrYpOWqpPCTbixz9nVrrrlu+vi/p61Mnpog0uZDOmVNltgUz12vDymg0/f8kc5/CGxWR3w+pouHZENnjWpnF6HSfWjfu5Cnsr2jOetuKIuEpsjhXVtqxo3e6N/+fhVcquA/VNPPzWWqYt3cFp+b14ctJohvXv4XVYSinlKxmdeFp9HrF1MTuVHSt3KJ1ODpIpLV6Y7R+3SrwFt+bxzNY2Dqfm8Yw8j05eWmk79qENNcYYttTWfJA2BsOxXGbrqLbJrX5I3M9cGq/08Gc8w7dr/RnP+Ey7H2ItZ2+ftcVTZZsVO/Zx9ytFbN19gO9eNIx7LjuJzh1dn61OKaUyTkZ+M7Ymhbe+sJTH5qxPuLytZzxtxKH1p+isHJeUj53PWkzaJ46pdbWNe83aPHZtWw+dufmwsmQfX3vmI+oa27ca2pl+IxY7raZWL4XmiIIyrdUt/jOeh39uam7taptki6eda7HdyvHLsl2eUj7WHDRMmV986LvwX986l5+OH6FJp1JKxZDRLZ4ATy/4POEy0ao5iaZTSXZU21QSjmRGlE1mWpj4Uq8URm0BybG6ZsqDCx1qyXKWU+Xd98YaVuzYx9qd+zlzSJ8274Unnm0GCbKx8XjJSbuE0WIGeejRUxdvjKTU1dZud+4YyXnzoRbP5Cq/rUUl84xnq9abg9HWbBlcKMOyfqUiBPYd5J5Xivh0yx6uOm0gv5l4Gr26d/I6LKWU8rWcuC0XbU5FJ7rE/fOT9iPSunEzP7DvYOw3bXT7S1dDw4Mz1yReyIZMbCCxG/NT8za1XT9eV1Ob1+4DM1dbXjZWN1krWuNyosXThWk8HWtpi5c0hW+hqtb6yM1WxG8EP/zme+vKgYgWTxsN6JE3wGz1AgktHXdKn0z8QCsV5s0VpYx/ciFrAlU8ccMZ/OWmMZp0KqWUBTmReLZWdIIGNpXXJFjWerl//aB9a6sbLVThU0VEVtocb++MtQM2dmz2qrLY5dsLx5eag4YFG8rbvGZh+kJbnKybz11zeNTXROX+smB1wmk7/r5oM1W1DTGXCe/SmuwgQfETl7a/W/0MRCaeyX52rCb+Zzz0jt2C26iorqdkb23MxcMbeqMdrg5hC0Tt9RHjALQu+8JHW4HEgwslul4iBU3mdXNWCqC6rpF7Xy3iB/8uZFj/Hsy+axzXnzVIR61VSimLciLxDK84vb06dlIEqbeE2m1VaTtfoYXlIyvdNv7eedVt85Wl25kfkajZ2077LXnZVe+vH3zOrS8s5f31uxypQEernM9cUQqk/1HWd9ZGfD4iAli+fS+PzFp3aM7QcIe6BztwoaXaBT2a1u8BVwfZcvCOwdm/fo8Lfjc/vHB7BSRITGNpXba15drO8T88kFvsrrbvr0/+u0Apr3y2bS9X/elDZhQG+OGlw3ntO+dxXL8jvA5LKaUySkY+42m3Mm7r2bJQT8NkEwo/9SJLJpaYq6SQAf30P4fn7HTq2cV032AOr1BvrTwAtLRIhSsurw5b3n5lPdwbRaW2lre8rQTvi0jc2BuaEnffDl8//G07N3VsPeNptcyIpjs3rqGUph5JYbvRn2MPfz/50q1M89K6rZK9cR4LCKPtQypTNDUHmTL/c/70/iYG9urKq3ecx9ihfb0OSymlMlKOtHjaSALS3OIZsXELi0R2F/SuCpfMlpM5vn5K5iOFx3bZHxYe+rm1tciLaXniSZQQp3I1mYh/U2Fi/ByN1QSy9TgfPgTJ7W28z1zkuXTyeca4z3gmTA7tr3NoOSstxKF//zK/OKltKOVHO/bU8o2pn/DH9zZy7RnHMvuucZp0KqVUCjKyxdMuW4lnmitKqW7PVldbK4mtnS551hdNqvx4InfbGJO252ysbKa15cvKjBZ2D4mb16hI/Hja73u0jCb6z06Napvs/jsw5lFCkZsIGkhxOs0kt2z93UThBZMfbyrmlvWZOOVnxhhmFAW4b8YaBHhq0miuG53vdVhKKZXxcqLFM+pw/zGqW6lW6sMrzEMnz2LjrmrLyWEyrYFpq77ZDC1WS4+j3X/D2B1J9UB9Ezc/t4TtUZ5VTEbkOT40krKlJDWNLZ4Jl2gb8P6DTfx53qZD3VQjd6fqYCPjHnufNaVVzk4BYyNhtdrqf6jFs3W9GKtV1zWydud+S2XG2karJhtZW0qdJSx0h01WwulUki5ZKf+pOtjIXdOKuOeVFZwy8Ehm3zVOk06llHJIziaesRyeesWZydcLCgMut1BZj9NalznrwcbbslMjmMbcdsTGm22W8d66XSzaVMkT72ywvW0rWhO1DhbOj5+6I4q0jaeypp7fv7uReaEBYSKvt8XFu9mx5yB/nne4i2XbZzyT27l49xEiD6nlrrYWb04s374v7vvxthd5LlNrLYwo2+Z74efKq0vMT9e2Uol8umUPX3lqEbNW7eTHV5zEtNvPY3Df7l6HpZRSWUO72obZsaeWXfvr0rItq9pN0xHxe9paPMM29EZRgLrG5riLxzoKrUlJm+TEJKrMJz6m8Vo8P69oP4WOnRbJ9vEkXqbZRvl2L5mUrrCErYfRJZrf04T+s7AJS/747sY2pbfZVopdbQ+Pvtu+IGOMpe7RsUSWaeeGSGoDACVq8kzqLcBCi6eJfd1o3qkyQWNzkKfe28TTC4oZ3Lc7r3/nPMYM6eN1WEoplXUyssXTbrIQreIUrZI37rH53P7SZ8mGFSo34ndbLX/RykulMpr0qnHdNa2ozUi10cSqrEZ7NVHlPNWutj+b3j7WQ6PUWig7GcEogwsV7Yjekhb3eUZnw0p4PbU849l+GQl7P+E2YmzCzvX4yrId1he2qP3AP+2XaUmikr8qIi/D5mbnzmDcwYUSrRvtnFoelMnacso+ERkvIhtEpFhEJkd5/0IRWS4iTSJyfcR7t4jIptD/t4S93llEporIRhFZLyJfS8e+ZKotlQe4/pmP+Mv8Yq4/axCzfjhOk06llHJJRiaedqWzu5ex2bUuYYUxssUzcgEbdeRfFKymoSlBy5VDxypWMhXt9fDXrG4+Mjmw+4xn5HyDVgl2Bxc6vHBxefuWV7C+z9V1jdQ2NDk6UmqkmM8+///2zjtOqups/N+HXbpUAUUQAQVRxIKIgg2xoRiJLT9LYonGaGJ/jSXRxBLRGJM3scSWYHxN7MaOgAo2CCJdOkvvnaVt3+f3x9zZvTtz78ydtjOzPN/PZz87c+495zzn3HPvnOc+z3mO7/khVm8rca3xrGvNTpX4azyjuealKXw+f0OdtMh9PL2KrVZNaZuVSFkTsnimeY2nxDkelPheHDFenPgcsthCICIFwDPAOcDhwGUicnjEaSuBq4FXI/K2B34HHA8MBH4nImFt6TfARlXt7ZT7ZabakM+oKm9+t4rhT37N8i17+NsV/Xn84qPYp+le4QhmGIaRFfYOxTMJu9HGncm53KbbQhnPSpPo/O0378a2VKaLRKxeyWwFEUnCiqfzP1MT4BpXW1eaX1VB3bP7PTCOUx6fkJJccZU4iX2OXxvmrt3B9j3lMetIdauiGhniXDNVZcLCTVz78tQ66ZH97O0JEf+einU8leBC8YjeSimBvEncd66KkyZd17yBMhAoUtWlqloOvA6McJ+gqstVdTYQOZDOBj5V1a2qug34FBjmHPsp8KiTv1pVN2eyEfnI9j3l/OLf07nrndkc1bUtY247mXP7dc62WIZhGA2evHy1l6iukMzcb9ueioTz7CqrpDzOWrjSiipEoGlhQaAyoy2eEZPPGLNwr0nfpCVbYtcXSKr4JOJqW+fcgAKkGlwo1SBSgct3Fd/I5zVPIore5l3lqQkWB9+1ejUW4oh01+cyx5quPseTJZ77etRY8LkFI/fx9H3Rk8Y1nukMLhSz3qSOB7s6iXplBDlmFk8AugBun/LVhCyYyebtIiJtne8Pi8gQYAlwk6puiMiPiFwPXA/QrVu3xCTPYyYVbeaON2exZXcZ95zTh5+d3JOCVBZ2G4ZhGIHZKyye9bVdxRG/G8vPPdaIuidZfe4fwwkjPwfgf96cxafzaucDnusfIyx58Sbd9UmsXg3LWVZZ5ZnuZuOOsoRremp83Y3qY1o8vQ7VuMLGqdorq0d5kcF3nv9yqVN+bQX+W/gkZ61NhnhVrS0uZc32khhn+HdYZa0vq6u+9LrdBmHUxGU1nxdv2FnzOTKqbSbWeEYWmVhwoVq8AmLFcrv3GkPxLcPB5KrP7X72MryuUPDVBt55C4GuwERV7Q/8F3jCqwBVfUFVB6jqgI4dOwasNn8pr6zm0dHzueIf39KiaQH/ufFEbjj1YFM6DcMw6pGUFE8RWS4i34vITBGZGj9HdvDTSVQ1YRfNeMyIsxUD1FpT35m+Ou65lXHkc/9k7iqrjFteGP99Nn39JBOiWpWPZ6/j0PvGxC1oyBNf1Hy+653ZiVXkEOs6ell+E4lqG0QxvPX1mZ7pdVxtfeqqz+At7r7wm29Fro10E6u/wtfAz73ys/kb+GzeBsoqq+KuNXaTit7z8ffrosoJF+fpaqvxX0bE6oPIMoMGF4ocY+7nyMjR830i8AYq2rN88L6lu9/zcVRaPMVz064yvl9TXCct3EWmssZkNXCg63tXYG2KebcAe4B3nfS3gP6piZn/FG3cyQV/m8jzXy3lsoHd+Ojmk+jXtU22xTIMw9jrSIer7Wm5soZEVX1cTz2sAQi/fncOr01ZyfLHhmdeuCRJRDF+6Ztl8U9ySIcRI/Zat5Ci4ZUO3hPSknLvLVqCyBrLpdE7WnCIINatt6fVfUGQiJU5SPCihC2eaZrN79+6GWuLo9cyxzQex1hjHLb6VlQpD344Lypv2ErdSKBN88YJy1tTZ+Q+ngEtlDsDvJip9n2GBCMqqm3Ai6XqPw5e+Gop157UIymX1prjcfI8MXYhx/dsn1TZl7/4bQy5vDNnysU9z/gO6CUiPYA1wKXA5QHzjgVGugIKnQXcq6oqIh8CQ4DxwOlA9M24l6Cq/Pvblfz+43k0b1zACz85lrP67p9tsQzDMPZa8nKNpx+q3kqB10RaUV6bsjLzMqGoBptkeU3S4gUnibnGM0Hv09gVJVZGwhZVYF1xLBfP2MTqp1iTbq/ue+yTBRSX1K7xje16Ght3+X7Xql4tngGW08a0HkcGu3J9rvSw7nmv6U1sDXWkBTVlxdspwC/Cclqj2ga8uKmsowzlj31CPKX16QlFPD0hubqN5FDVShG5iZASWQCMUtW5IvIQMFVVPxCR4whZL9sBPxCRB1W1r6puFZGHCSmvAA+p6lbn893AKyLyF2ATcE29NixH2LKrjLvfmc1n8zdycq8O/OmSo+jUulm2xTIMw9irSVXxVGCciCjwvKq+EHlCfQYw8Jsfxd8APTUrRyaJWuMZcdwtdaME1qpkcksOCCsX0YFwYtVaUuFj8QxQ3/WvTOP9X55IS49Q+N5uhmFX2+g+e+7LJTHrSrbr/C5Ppq+FG/dw8lOKItdCQm1/xZI0nRFcEyLJW9c7uFCQqLaxXvZEuNoGVDyrVeO8KNKYymW87VQmL40OKhY04myiazyDjOccfdzWO6o6GhgdkfZb1+fvCLnReuUdBYzySF8BnJJeSfOLLxZu5M63ZrOjpIL7zzucawZ3T+j30TAMw8gMqQYXOtEJYHAO8EsRifqxy0QAAz8l0W/CE2/ul+51npEEnWR5SRF3jad4f/ajJrKnnwwJdEWs6opLKvhi4aaEyk9FbynauIsPZ3kvj/LqwnBapifA7vb6KSyJxkVK18o5v3Efyz00VoCeCi+LZxpEjbsFTKLlRfyPPJjKmIha4xnU4qmR3xOz8noqnq6G/HPS8kB5gpYdC3eTzVpq1CelFVU8+OFcrn7pO9q3bMz7N53ItSf1MKXTMAwjR0hJ8VTVtc7/jYTcgQamQ6h4+G754JceZ/Zz4bOT2LY7tW0qMmW1ipq4RtVT2xuNImbM3m6O6l1MPDzOL40RIMbL2hmq179iP4UnqKx+CoNnnU5aKvORkvIqNuyIvd9rEGtRohYlLwUvOLV5/V5qeFk8a45FDkd3eXG2Eso1/FxtU3FrTdbVNq5XhkfZQWVatXWPZ3pZwABPiY7PIOebGmCkmwXrdzDi6Ym8NHE5Vw/uzgc3ncRhnVtnWyzDMAzDRdKutiLSEmikqjudz2cBD6VNsiTwDcgaZx40e3UxP3j6m5TqTofV1EvORMoNokTVBvfxsk4pm3bG29qklhe+Wup7bMyc9Z7psVqTah/6blfikVZj8UxiChxWcB/wCKATXU9t7f5RbRNrdyoure6qfF1tY64H9LfEvTczaEDOxIgU57vlW6msqqawIP57s9husdFp1RrfATWm1T5S8UzTC6lYLwNisXZ7CSc/7r148863ZgWrO2GLZ22GoO68hpEsqso/Jy3n0U8W0LpZIS9dfRyn9emUbbEMwzAMD1JZ47kf8K7jzlUIvKqqkXtn1Ct+k5wgE/vV25IPIBOqI6XsvsRd4+maV0daPL0IT2AXrY/eJ/BP4xbx9ISiqPRQRXGLroOfUhp70l73YFjWVK3J3gbP4BbPVCJwui+fr6U+weZVVCbfH+6cfgqs1xgIyxhtgI9vqUs3O0oreWp8Ebef2Tup/LUie7x8Ib6SF+to5DMoXRbP+OvUvdM3JvAiKUbpiZ0dwNU2V9fUG/nFxp2l/Oqt2Xy5aBND+3Ti8YuPpMM+TbMtlmEYhuFD0oqnqi4FjkqjLCnjNcmZtmIbc9bsiEpPdzj/b4qi1zOmg0h3yMhJcd19IiNcbX1cCdcVl0RZeN+dsTpuUJ10EEtRWbh+Z53vO0qD70sKiVkUa9d4BtjuxDXxnrV6O8s27w4sU1lFFYs27KT3fq1iRLX175Oyymo27ChlP1c0xoqULJ61dSVjYY5UrOIVMXt1/H1t49bp0T+LN4bGStHGnfxhzIKkyvVe+6tx25TItiax+njs3FqvgNB2Kv7lxlvr7XVUSNya7kWiw8Rd5eKNu6iqVgpsjZ2RZj6bt4G73pnN7rJKHh7Rlx+fcJC90DAMw8hxGtR2KlOWbWXpprqWvIueneR5brpdwN78brX3gTgTSjcrt0YrNFVxt1MJ/S8pr+Lp8Yvj1lGtsN1jK4vb3wjmdpcq4a7w6pP73psTM088/INOBStzR2kFzRsXxKzj/QTdScfN28C4eRuYeM9QX8V4/IKNvvnLK6s5fuTndfaa9dq2JCh1LZ6JlxOZJVlLXKqEXxxd+/JUNu9KbH12WJH1W/ob34rrf/y/S+pGj42leL40cZmrzNhUVcdZe+rlpq9Kqc/euG5StbZGcthvax1fnv1iCapwzzl96pxTHnB9qWFEUlJexSOj5/GvySs5rHNrnrz0aHrt1yrbYhmGYRgBaFCK55WjpmSt7nRYFkaOjrbcxI1q60zAZ63eHmhvxKTlTJMCEbR+9+Q+cHChBOoMKxduZfDIB8Zx1uH7BassQU58bLyvC9iDAdaKupWhihSC+ARxg4xFZF9mbQWfc92CuJf74ecRkIrFc31EsKmg4z20ttT/3HjWaa+8789cG+hFSbxnjNeLqkT4fP6GKMUzlb1xjb2XOWuKufX1GSzZtJufndyDO88+lKaFsV8WGoZhGLlD3imeO0or+DyGhShbxJq7peL9M29tXTfhyHrCZXtFqPQSKZ5LX6YJWveOkmg32x2lFbRu1tg3TyL9HFYIIhWXcfM2RJebJrfszbuSX2/nvr5Bo5F6keqlj1TWkg16kyrhK9IkQIChSBas30lpRZXnPVtWUR0owmxQKl3rlGO5Aca7L6qqE9/HMyiZvoaLN+5iUtHmjNZhNGyqq5W/f7OUP45dSLsWTXjl2oGc3Cs927MZhmEY9Ueq+3jWO3M91mvmAqlEGo3FR7PX1fnuN0UM6rqWtMUzTUtngtb+2fxaBVBR/rtkC0c+MI4vF/mvpW0kQrGHdabaw1oYVgjyZUVQictl8o9jFyZdTrKBmsJ9GDnM02Hpj8etr8+MSgsrcU0KE3+Evf7dKu57b46nIjfkiS/iW+MSaHNYqet93ydc8LeJMYqM7Uob3+KZPPVxDZ+th/XjRsNkfXEpPxn1LSNHL+C0Qzsx5rZTTOk0DMPIU/JO8dxTnljAmfrii4X+CtFWjz1Cn0hSeYhUHMKKU8YVTw+SsuQmWf3MVaEgNZFr6CLlGf7U19FVuuq89IXJAFQ56yTDG4unYzucTFJSEX+tXiYJ92FkL81aVVzvsrhpmoTiCaGgY5e/+K3nsSUboyM+h3lr6ioWRATBikX4BUdFlTJjZW2gpR2lFUxeurXme7zbcsQzE2O6xKYS+TnXx76x9zJmzjqG/fUrpq/YzqMX9uP5nxxL+5ZNsi2WYRiGkSR552q7O0CwjFyiWpXXpqyKSvfdtiQOkfPLsOWnvCq6X7z3KUxfYKVk5rrlVdUh605UWf6F7SippHFBqJ2VcdY3em2L4y552optoXKcyXZpRRW3vT6D/znrUN8yc2Evwhe/9t8zNRGS1U9ue2Mm89ftoP9B7eqkL9wQXAlLJ+F3Hk0bJ6d4btvjH5Aolh72q7dnJ1SPn1J35APjIuqse16FRwCp4hL/tZa5bvG0aKNGIuwuq+ShD+fxxtRV9OvShr9cejQHd9wn22IZhmEYKZJ/Fs+y9Fg8072dih8vfr0s/kkJ4DdJDGrxjOfS58fU5dsSz+TDc19GK1FLY2xRcsO/plHoWCY37CzztCCD/+TWq8/CCsEnc9bz3sy1nPrHCb7154JF6KWJy9NSTipK9PNfLU15T9V0Eb7U8dZ4+uk7LWJEL/a7xz6YlVhE41hlRaLUVR5//e73Uedc89J3CdcfhPoY37kybozcZ9aq7Zz31De8OW0VNw45mHduHGxKp2EYRgMh7xTPC/t3zbYIWeXp8XUtpeGJdXBX2+QsHCu37kk4jx8fzV7LrFV193csjeNKWugoGB/OWkv/hz/1PMfvVUJkc1+ZvKLG4hxWXGLNvb9ctKnBTJxTbUYO6OCAy+IZJ6KlX3vXFpd6HyC9FsCgW9akWmXQ/F77aU5fmfpeq/FoILePkUGqqpVnJhRx0bOTKK2o4tXrTuDuYX2SWsdtGIZh5CZ552qbrh+hXHCfTIadERbfWFFtt3hEUa2qVr5JIsJkI0mf0jF3bXSAqHgT07CrbSzW+gSFiVQa73ftFxpkjeDs1cX02b9h7BOXuuKZG/eNiKCqcV1tk5E3nW18+KN5rHK9tOl+z8dM+fXpUeeFPBGSrzdozkYC2ViskMwzx9h7WLO9hNvfmMmUZVsZ3q8zIy/oR5sW/hHMDcMwjPwk7xTPdPHypOXZFiGtlHusfbz+lWme5z4+JvHARo0LGqW0jUc8znvqm5jH/x7AZfnRT6L3QYXYk/KgE/Y3p64OeGZuk+oLl5tenZEmSVLj3RlreHfGGrq1bxHzvGRam06r7qadZVFRiL9fEx2QKdUqIz0I/Ajrtl8pUhXSAAAYG0lEQVQvNkXQyA0+nLWWX7/7PdXVyhOXHMVF/bvYmmDDMIwGyl7rw7ItxU3Rc4XwWtWgrrbJ4uWiV58sjhFpNB6VHoFawuxO05rhfCFHDJZpI54LeDJWxEy7Vb8yeUVUWrUqqzwCY6WbdLbs9D6d0lja3omIDBORhSJSJCL3eBw/RUSmi0iliFwccewqEVns/F3lkfcDEZkTmZ4r7Cyt4I43Z3LzazM4uOM+jL71ZC4+tqspnYZhGA2Yvdbi2VDYUVpBeWV1Rq2RAHvyLJqwm1j7Mu7a2xTPbAtQzyRzX2zYEe2ink68tl4a9Oj4jNYZJp2BhHrt14rPF2xMW3l7GyJSADwDnAmsBr4TkQ9UdZ7rtJXA1cCdEXnbA78DBhC6rac5ebc5xy8Ekn9bl2GmrdjGbW/MYM22Em45vRc3Dz2ExnEChRmGYRj5jz3p85zteyq46dXpGbd4NlTyWaFOhrkebp4NmbIk9j8db8pUIIKsuzZiMhAoUtWlqloOvA6McJ+gqstVdTYQ+YA/G/hUVbc6yuanwDAAEdkHuAP4faYbkCiVVdX85bNF/Oj5/6IKb/58EHec2duUTsMwjL2EvLR4dmnbnC7tmjNl2db4J2eQNs0bx9xbr74YN28Dh3dunW0xjDwg3/bBTZVMewLszXitVTUSogvg3uR5NXB8Cnm7OJ8fBv4EpC8UeRpYuWUPt70xg+krt3PBMV14cERfWjezAEKGYRh7E3mpeE68ZygQ2oLjsU8W8M8sBQrKlQifAPPWRUeKNYy9HVM8M8eu0r3LTT0DeJmMg/6oeOYVkaOBQ1T1dhHpHrMAkeuB6wG6desWsNrEUVXenbGG374/FwH+eunRjDi6S9x8hmEYRsMjr/1bmjUuYEdpeiyO/2/AgQnnqc6VTQ1zlAv72+SiIfKzk3tktf6iR84JfO6C9TszKMneze7yKo7s2ibbYuQzqwH3D09XYG2KeQcBx4rIcuAboLeIfOFVgKq+oKoDVHVAx44dExQ9GMUlFdzy+kzueHMWh3VuxehbTzal0zAMYy8mrxVPCLndpoNBB+8b6LwnLjmq5nO+uC326rQPHfZpGve8lk0K0lrvn390dFrLy3eG9+uc9jIHHNQuqXypXOtEAzI9csERSdcVyazfnUVhAuvB5ue4J8CB7dPz/MoGu8sqefHKAXXSgr5sspdSAHwH9BKRHiLSBLgU+CBg3rHAWSLSTkTaAWcBY1X1WVU9QFW7AycBi1R1SAZkj8u3S7dw7l+/ZvT367jzrN68fv0gDoyz/ZFhGIbRsMl7xfOmoYfw0Ii+dSbS79w4OOFy9t2niWd6ZACNi4/tWvP5qBTe9r9y7cCE89x/3uEsHXku8x46O6F8A3u0Z+p9Z8Q9r/f+rep8b944vYqom6sHd08q3xXHR7uEJTN579mxZVL1R3KJazz48ep1x/POjYN48rJj2Lel9zjzYvljw6PSIvuta7vmDA740sRN/xgK689P7Rkz77ri0oTq6tlhH99jiVrM2jSPvSZsWN/9PdO77xt8wpuuFzDxLMN3nNmbt29I/FmVKxzcsWWdbZYu6t810Mumn5/S015KAapaCdxESImcD7ypqnNF5CEROR9ARI4TkdXAJcDzIjLXybuV0FrO75y/h5y0rFNRVc0fxy7g0hcnU1ggvHPjYG4a2ivrW3IZhmEY2SfvFc+mhQVcOag7cx8axss/HcgfLz6SYw9qx33DD4uZr72jAJx4yL68dM1xdG1Xd2L681N6cmH/Lnx919CovMsfG843d5/Gv647no9vOcm3jl+dfajvse77+is+/7zmOE+l49TeHWjUSGjRpNBXcV3w8LCotEiP4MWPnEOXts2jFMv9WjXjupNCk+UzDuvE2NtOoZHAf34xmJMO6cA95/QBoFOrpky+9/SafN/cfRon9+pQ8/3/fjowql9O6NmeQT1rFaQHzu8LwPAja62Ad5zZO66CHJYB4Nkr+gNwVNe2vufv17rW0nv3sD4cul9IuX79Zyfw33ujr62bj26OvrY9O7bko5tP4sYhBwNEjZsw97rkHNijPcce1J6CRsKTlx3DkV3b0KtTtDLmTgtfmxYRStAD5/flmcv78+cfHcWLVw7gkQv6+U7owkrYuNtPqWOZ+t0PDuf5nxzLwz88glZNCxl72ym8c+Mgbhl6CC2aFHDHmb35w0X9osoLK/2XD+zGkEM70qJJAf+4agAtmxREeR4Mde3xeESX1rRsUuBp+WzVrHaZ+Z8uOYovfzUEgCGHxnb9u2tY7b31zd2nMea2k1n+2HBOPyxU7wM/OLzO+QdF3G+Rx92yHH5AbaCu47pHK+jPXN4f91aDJ/Rsz8XHduWZy/sz/f4za9I7tqode21bNGb+Q8PqjLlbTu/Ffq2bRZUflu3C/l14/5cnRh2PxfAjO9P3gLqBxj7/n1N58coBPDyib9T5Nw89JKHyAS48pgtv3RB6kdK4UegnpHGB8KcfhbxBHhrRlyGHduTdXwzmmcv717nHf3LCQdw9rI9nuXsjqjpaVXur6sGq+oiT9ltV/cD5/J2qdlXVlqq6r6r2deUdpaqHOH8veZS9XFXT524QgGWbd3Pxs5N4ZsISLjm2K6NvOZmjD/R/PhuGYRh7F5LKZukiMgz4K1AA/F1VH4t1/oABA3Tq1KlJ15cIH89exy9fnU7zxgWUOFsqfH3Xabw9bTUAlw3sxv5t6k76Xv12JaMmLqN/t7Y8fnGtS21FVTWfzFlPt/YtPH9Ef/XWLCYWbeZvPz6WBet2cHzPffl8/gauPakHv31/Lq9MXsHgg/dl0pItQCiE/JFd29Dn/jE1Zbxz4yAmL93KZ/M38Op1J9C8SQHz1+1gxNMTKa+q5rKB3Xjkh0fQyKVk7CytoKpaOf/piazcuoeHRvTlykHdmbZiK0Ubd7GnvIoHP5zHcz/uz7AjOvPd8q1MW7GNG04NKU1llVXc/fZsbhp6CFOWbWP4kZ1p1bSQKlXf8PZV1YoAjRoJ64pLUIUDHKVj2optNCloRD+XFav7PR/TrX0LvrrrNKqqle17ytnXcftVVUSED2etpWfHlvQ9IJTvkF+PprJaeema43hs9AKGHtaJDcWlnH/0AQw5tBMTFm6kx74t6dS6KTe9OoMbTj2Yzm2acekLk7no2K48+fnimvpn/vZMlmzaxbEHtfdsz/SV29hRUsHjYxbWCdDUqmkh3z94NpOKNiMiUa7YY+as44Z/Tee9X57Iqq17uPm1GUBIYb/i+IM4rU8nThj5Oet3lHq+RFhfXMpXizZxdLe2bN5VRuOCRmzaWcYv/j2dnh1a8tdLj6Ff1zZUVyuz1xRz1agpDD+yMyMviFYI1xeX8o9vlvLTk3pw5T+m0Kdza37/wyMobCTMW7eD47qH2n7eU19zfI99uf+8aKXLi8UbdvLV4s2c229/Xvt2Jbec3iumm+vuskp+//F8Xpuykmn3ncHvP57PuzPWsOj359CkMJRv2+5ylm3ZzYotu5m+YjsHtm/OyNEL+MWQg7nLUUi+WrSJAd3b8a/JKxg5egHNGjeitKKaRy44giuOP6imvuWbd/OfGWu4/YxeNZvOqyqfzFnPsL77M2v1dpZu2s2F/bvw+NiFPPvFktq8jw1n9bY9vDZlJc9MWMJRXdsw6urjKC6poHmTAl6etIKTDunA8T3bI0CVKlt3lzNmznquObEHa7aX8MjH8xj9/fqoYCl/GreQp8YX8dyP+/PejLUc0LY5vzzt4Jpxv7O0gp2llTX3zVeLNvH42AXMWbODLm2bM+HOIYwcPZ9bTu9F+5ZN6H7Px1F9/X8/HciVo6bws5N78OLXy4BQ0JZzjuhMk8JGqCrvz1xLp1ZNGXxI6KVQZVU1z3+1lB8NOJB2LRpTWBA674uFm2jfsgmTl27hxEM6cN5T3zCo574c170dT44vYsxtJzNu7gb+/Okibjm9F3ec2buOLM9/uYSz++5P9w7+L9NufX0G789cy6irBzC0z36+5yWDiExT1QHxzzT8SMdv89i567n9jZk0LmjEoxf249wMLC0wDMMw8gO/3+akFU9n8+tFuDa/Bi6L2Py6DvWpeIYj6Z135AE1a9LaJ+DmmC7KK6t5edJyrhrcnU/mrKN1s8ac5rIG7SmvZGdppaflIyjb95RTVa01E9swqsqC9Ts5LItbrewpr6SRCM0ScNstLqmgsqo6qj1Bqa5Wvi7aTL8ubRK65qrKyq17mLZiG/27tYs5kYZQv7dtESq/pLyKMXPX8cOju9QoQdt2l7O7vNLXKupF0cZdHOJhDc0XyiqrWLOthJ4d96Giqpqtu8tjju2dpRW8NHE5Pz+1J00L644RVWXTzjKaFhbw4ey1XHF8t5q+TZQ95ZW88NVSbhxyMKokNB5jsWLL7ihrakVVNR/PXsf5Rx1Q50VRPJZu2kW7Fk1oFzFm1xWXUFGpiEBhgdC0sKDOuC6tqKJJQaOE6orF4g076dS6WVy35kQIK/m3n9E7oTW6QTDFM3XS8du8eMNORo6ez8gL+9G5Tf6uXTYMwzBSJxOK5yDgAVU92/l+L4CqPuqXpz4VT8MwDKPhY4pn6thvs2EYhpFO/H6bU3n1HGsDa3fF14vIVBGZumnTphSqMwzDMAzDMAzDMPKRVBTPQJtf18deYYZhGIZhGIZhGEbukorimcrm14ZhGIZhGIZhGMZeQiqKZyqbXxuGYRiGYRiGYRh7CYXxT/FGVStFJLz5dQEwSlXnpk0ywzAMwzAMwzAMo0GQtOIJoc2vgdFpksUwDMMwDMMwDMNogKR3QzXDMAzDMAzDMAzDiMAUT8MwDMMwDMMwDCOjiGrUDiiZq0xkE7AiDUV1ADanoZxsY+3IHRpCG6BhtKMhtAEaRjvyoQ0Hqart1ZUC9tuct3JD/spuctcvJnf9YnL7/DbXq+KZLkRkqqoOyLYcqWLtyB0aQhugYbSjIbQBGkY7GkIbjPojX8dLvsoN+Su7yV2/mNz1i8ntj7naGoZhGIZhGIZhGBnFFE/DMAzDMAzDMAwjo+Sr4vlCtgVIE9aO3KEhtAEaRjsaQhugYbSjIbTBqD/ydbzkq9yQv7Kb3PWLyV2/mNw+5OUaT8MwDMMwDMMwDCN/yFeLp2EYhmEYhmEYhpEnmOJpGIZhGIZhGIZhZJS8UzxFZJiILBSRIhG5J9vy+CEiB4rIBBGZLyJzReRWJ729iHwqIoud/+2cdBGRJ512zRaR/tltQV1EpEBEZojIR873HiLyrdOON0SkiZPe1Ple5Bzvnk25w4hIWxF5W0QWONdkUD5eCxG53RlPc0TkNRFplg/XQkRGichGEZnjSku4/0XkKuf8xSJyVQ604Y/OmJotIu+KSFvXsXudNiwUkbNd6Vl9hnm1w3XsThFREengfM/Ja2HUP/HGbaznjd+9kCNy3yEi85zx/bmIHOQ6ViUiM52/D3JM7qtFZJNLvutcx7L5nIwn9/+6ZF4kIttdx7LZ377PRed4Tj4LA8h9hSPvbBGZJCJHuY4tF5Hvnf6eWn9SB5J7iIgUu8bDb13HsvYbGkDuX7lknuOM6fbOsWz2t6dOEnFO/YxxVc2bP6AAWAL0BJoAs4DDsy2Xj6ydgf7O51bAIuBw4HHgHif9HuAPzudzgU8AAU4Avs12GyLacwfwKvCR8/1N4FLn83PAjc7nXwDPOZ8vBd7ItuyOLC8D1zmfmwBt8+1aAF2AZUBz1zW4Oh+uBXAK0B+Y40pLqP+B9sBS538753O7LLfhLKDQ+fwHVxsOd55PTYEeznOrIBeeYV7tcNIPBMYCK4AOuXwt7K9+/4KMW7/njd+9kENynwa0cD7f6H5OArtyuL+vBp72yJu1ezPR5xtwMzAq2/3t1O35XHQdz8lnYQC5B4flAc7BNZ8Bloef9TnY30Nw5pupjLH6ljvi3B8A43Okvz11kohz6mWM55vFcyBQpKpLVbUceB0YkWWZPFHVdao63fm8E5hPSHEYQUgJwvn/Q+fzCOD/NMRkoK2IdK5nsT0Rka7AcODvzncBhgJvO6dEtiPcvreB053zs4aItCb0sPgHgKqWq+p28vBaAIVAcxEpBFoA68iDa6GqXwFbI5IT7f+zgU9VdauqbgM+BYZlXvoQXm1Q1XGqWul8nQx0dT6PAF5X1TJVXQYUEXp+Zf0Z5nMtAP4XuAtwR5zLyWth1DtBxq3f88bvXsgJuVV1gqrucb667+FskspzIpv3ZqJyXwa8Vi+SxSHGczFMTj4L48mtqpMcuSB3xneQ/vYjq7+hCcqdS+PbTydxUy9jPN8Uzy7AKtf31UR3XM4hIZejY4Bvgf1UdR2EBgLQyTktl9v2F0IT0mrn+77AdteE2y1rTTuc48XO+dmkJ7AJeElC7sJ/F5GW5Nm1UNU1wBPASkIKZzEwjfy6Fm4S7f+cvC4ufkrobSHkWRtE5HxgjarOijiUV+0wMkaQ6+33vMnmWEm07mupvYcBmonIVBGZLCI/9MuUAYLKfZHjEve2iByYYN5MELhux6W5BzDelZyt/g5CQ3gWRo5vBcaJyDQRuT5LMsVikIjMEpFPRKSvk5YX/S0iLQgpZ++4knOivyN0Ejf1MsYLk82YJbysNTm9H4yI7ENo4N2mqjtiGJxysm0ich6wUVWniciQcLLHqRrgWLYoJOQacbOqfisifyXk2ulHLrYBCa2BHEHox3o78BYh15lIcvlaBMFP7pxtj4j8BqgE/h1O8jhN8X7Zl9U2OD+QvyHkNhx12CMtp6+FkRGCXO9cHCuB6xaRHwMDgFNdyd1Uda2I9ATGi8j3qrokA3JGieORFin3h8BrqlomIjcQsjYPDZg3UyRS96XA26pa5UrLVn8HIRfHd2BE5DRCiudJruQTnf7uBHwqIgsci14uMB04SFV3ici5wHtAL/Kkvwm52U5UVbd1NOv9HamTRB72yJL2MZ5vFs/VhNYhhekKrM2SLHERkcaELvC/VfU/TvKGsNum83+jk56rbTsROF9ElhNyaRhKyALa1nH3hLqy1rTDOd6G5Nwp0slqYLWqht/uvE1IEc23a3EGsExVN6lqBfAfQus38ulauEm0/3PyujgL7c8DrlDV8MM4n9pwMKGXGbOc+7wrMF1E9ie/2mFkjiDX2+95k82xEqhuETmD0MuX81W1LJyuqmud/0uBLwhZCeqDuHKr6haXrC8CxwbNm0ESqftSItwQs9jfQcjbZ6GIHEloqdQIVd0STnf190bgXerPBT4uqrpDVXc5n0cDjSUU9C7n+9sh1vjOSn/76CRu6mWM55vi+R3QS0JRPJsQurD1GvksKM7aln8A81X1z65DHwDhiFBXAe+70q90okqdABSH3RCziareq6pdVbU7of4er6pXABOAi53TItsRbt/FzvlZfRulquuBVSJyqJN0OjCPPLsWhFxsTxCRFs74Crcjb65FBIn2/1jgLBFp51h/z3LSsoaIDAPuJjRh3eM69AFwqYQiffYg9KZ2Cjn4DFPV71W1k6p2d+7z1YSCEKwnj66FkVGCjFu/543fvZATcovIMcDzhO7hja70diLS1PncgdBL2Hk5JLc77sD5hNZsQXbvzUDPN+e3uB3wX1daNvs7CHn5LBSRboReUv9EVRe50luKSKvwZ0Jye0ZqzQYisr8zz0FEBhLSV7aQg7+hkYhIG0KeE++70rLa3zF0Ejf1M8Y1C9GVUvkjFHVpEaGoVr/Jtjwx5DyJkCl6NjDT+TuX0JqXz4HFzv/2zvkCPOO063tgQLbb4NGmIdRGte1JaPJQRMjls6mT3sz5XuQc75ltuR25jgamOtfjPUI/enl3LYAHgQWEHlivEIoUmfPXgtCbv3VABSHF5tpk+p/QOsoi5++aHGhDEaG1D+F7/DnX+b9x2rAQOMeVntVnmFc7Io4vpzaqbU5eC/ur/z+vcQs8REhhi/m88bsXckTuz4ANrnv4Ayd9sDPmZzn/r80xuR8F5jryTQD6uPJm8zkZU27n+wPAYxH5st3fXs/3G4AbnOM5+SwMIPffgW2u8T3VSe/p9PUsZxzV629RALlvco3vycDgWGMsV+R2zrmaUEA1d75s97efTlLvY1ycAg3DMAzDMAzDMAwjI+Sbq61hGIZhGIZhGIaRZ5jiaRiGYRiGYRiGYWQUUzwNwzAMwzAMwzCMjGKKp2EYhmEYhmEYhpFRTPE0DMMwDMMwDMMwMoopnoZhGIZhGIZhGEZGMcXTMAzDMAzDMAzDyCj/H7V4i7rdpRYEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# model.train()\n",
    "with exp.train():\n",
    "    loss_v_node = []\n",
    "    # loss_v_edge = []\n",
    "    acc_v_node = []\n",
    "    # acc_v_edge = []\n",
    "    ep = 0\n",
    "    for epoch in range(3):\n",
    "        ep += 1\n",
    "        node_correct = 0\n",
    "    #     edge_correct = 0\n",
    "        node_total = 0\n",
    "    #     edge_total = 0 \n",
    "        for batch in train_loader:\n",
    "    #         print(batch)\n",
    "            optimizer.zero_grad()\n",
    "            data = batch.to(device)\n",
    "            node_pred = model(data)\n",
    "    #         print(edge_pred.shape, data.y_edges.shape, node_pred.shape, data.y_params.shape)\n",
    "    #         print(edge_pred, data.y_edges, node_pred, data.y_params)\n",
    "    #         losses = [F.binary_cross_entropy_with_logits(edge_pred.float(), data.y_edges.float()), F.mse_loss(node_pred.float(), data.y_params.float())]\n",
    "    #         print(node_pred, data.y_nodes)\n",
    "    #         print(\"Losses: \", losses[0].item(), losses[1].item())\n",
    "            loss = F.mse_loss(node_pred, data.y_params)\n",
    "    #         loss = sum(losses)\n",
    "    #         loss_v_edge.append(losses[0])\n",
    "            loss_v_node.append(loss)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "    #         edge_pred = torch.sigmoid(edge_pred)\n",
    "    #         edge_correct += ((edge_pred > 0.5) == (data.y_edges > 0.5)).sum().item()\n",
    "            # A \"correct\" track parameter is one where the pred. is within 5% of the truth\n",
    "    #         print(node_pred, data.y_nodes)\n",
    "    #         print((((node_pred - data.y_nodes)/data.y_nodes)**2 < 0.05**2).sum().item())\n",
    "            node_correct += (((node_pred - data.y_params)/data.y_params)**2 < 0.1**2).sum().item()\n",
    "            node_total += len(node_pred)\n",
    "    #         edge_total += len(edge_pred)\n",
    "    #         print(out, data.y, )\n",
    "        node_acc = node_correct/node_total\n",
    "    #     edge_acc = edge_correct / edge_total\n",
    "        scheduler.step()\n",
    "    #     print(\"Epoch: \" , ep, \", loss: \", loss.item(), \", node accuracy: \", node_acc*100, \"%, edge accuracy: \", edge_acc*100, \"%, lr: \", scheduler.get_lr())\n",
    "        print(\"Epoch: \" , ep, \", loss: \", loss.item(), \", node accuracy: \", node_acc*100, \"%, lr: \", scheduler.get_lr())\n",
    "        acc_v_node.append(node_acc)\n",
    "        exp.log_metric(\"Test Accuracy\", node_acc, step = ep) \n",
    "#                        \"Test Loss\": loss.item(), \"Learning Rate\": scheduler.get_lr()[0]})\n",
    "    #     acc_v_edge.append(edge_acc)\n",
    "\n",
    "    #     if node_acc > 0.5:\n",
    "    #         break\n",
    "    fig, axs = plt.subplots(1,2)\n",
    "    fig.set_size_inches(16,4)\n",
    "    axs[0].plot(np.arange(len(loss_v_node)-10), loss_v_node[10:])\n",
    "    # axs[1].plot(np.arange(len(loss_v_edge)-10), loss_v_edge[10:])\n",
    "    axs[1].plot(np.arange(len(acc_v_node)), acc_v_node)\n",
    "    # axs[3].plot(np.arange(len(acc_v_edge)), acc_v_edge)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 8.9318],\n",
      "        [11.8650],\n",
      "        [11.2290],\n",
      "        ...,\n",
      "        [ 0.7272],\n",
      "        [ 0.6703],\n",
      "        [ 0.7702]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[15.2125],\n",
      "        [15.2125],\n",
      "        [15.2125],\n",
      "        ...,\n",
      "        [ 0.5958],\n",
      "        [ 0.5958],\n",
      "        [ 0.5958]], device='cuda:0')\n",
      "Accuracy: 39.7417%\n",
      "tensor([[13.8930],\n",
      "        [14.0282],\n",
      "        [17.2821],\n",
      "        ...,\n",
      "        [ 1.9400],\n",
      "        [ 1.5565],\n",
      "        [ 1.8785]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[40.8148],\n",
      "        [40.8148],\n",
      "        [40.8148],\n",
      "        ...,\n",
      "        [ 1.7936],\n",
      "        [ 1.7936],\n",
      "        [ 1.7936]], device='cuda:0')\n",
      "Accuracy: 40.2185%\n",
      "tensor([[0.6895],\n",
      "        [0.6651],\n",
      "        [0.6577],\n",
      "        ...,\n",
      "        [1.0724],\n",
      "        [1.4556],\n",
      "        [1.4310]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[18.9864],\n",
      "        [18.9864],\n",
      "        [18.9864],\n",
      "        ...,\n",
      "        [ 0.6766],\n",
      "        [ 0.6766],\n",
      "        [ 0.6766]], device='cuda:0')\n",
      "Accuracy: 40.0350%\n",
      "tensor([[1.1658],\n",
      "        [1.1077],\n",
      "        [0.6554],\n",
      "        ...,\n",
      "        [0.8939],\n",
      "        [0.8615],\n",
      "        [0.7666]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[18.9864],\n",
      "        [18.9864],\n",
      "        [18.9864],\n",
      "        ...,\n",
      "        [ 0.6721],\n",
      "        [ 0.6721],\n",
      "        [ 0.6721]], device='cuda:0')\n",
      "Accuracy: 40.3253%\n",
      "tensor([[2.3222],\n",
      "        [2.5033],\n",
      "        [2.3553],\n",
      "        ...,\n",
      "        [0.9325],\n",
      "        [0.8848],\n",
      "        [0.8664]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[2.8667],\n",
      "        [2.8667],\n",
      "        [2.8667],\n",
      "        ...,\n",
      "        [0.8440],\n",
      "        [0.8440],\n",
      "        [0.8440]], device='cuda:0')\n",
      "Accuracy: 40.4110%\n",
      "tensor([[12.1352],\n",
      "        [14.0357],\n",
      "        [17.5757],\n",
      "        ...,\n",
      "        [ 0.7509],\n",
      "        [ 0.8472],\n",
      "        [ 0.9719]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[15.9097],\n",
      "        [15.9097],\n",
      "        [15.9097],\n",
      "        ...,\n",
      "        [ 0.8866],\n",
      "        [ 0.8866],\n",
      "        [ 0.8866]], device='cuda:0')\n",
      "Accuracy: 40.0428%\n",
      "tensor([[0.9942],\n",
      "        [1.1208],\n",
      "        [1.1194],\n",
      "        ...,\n",
      "        [0.9826],\n",
      "        [0.8489],\n",
      "        [0.9707]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.6542],\n",
      "        [1.6542],\n",
      "        [1.6542],\n",
      "        ...,\n",
      "        [1.2293],\n",
      "        [1.2293],\n",
      "        [1.2293]], device='cuda:0')\n",
      "Accuracy: 38.8717%\n",
      "tensor([[ 3.4424],\n",
      "        [ 5.2534],\n",
      "        [23.1656],\n",
      "        ...,\n",
      "        [ 0.8685],\n",
      "        [ 0.5717],\n",
      "        [ 0.5412]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[45.7979],\n",
      "        [45.7979],\n",
      "        [45.7979],\n",
      "        ...,\n",
      "        [ 0.5170],\n",
      "        [ 0.5170],\n",
      "        [ 0.5170]], device='cuda:0')\n",
      "Accuracy: 39.7613%\n",
      "tensor([[ 9.1288],\n",
      "        [11.8913],\n",
      "        [11.2297],\n",
      "        ...,\n",
      "        [ 0.7391],\n",
      "        [ 0.9998],\n",
      "        [ 0.6972]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[15.2125],\n",
      "        [15.2125],\n",
      "        [15.2125],\n",
      "        ...,\n",
      "        [ 1.6608],\n",
      "        [ 1.6608],\n",
      "        [ 1.6608]], device='cuda:0')\n",
      "Accuracy: 40.3495%\n",
      "tensor([[ 9.4903],\n",
      "        [ 8.7859],\n",
      "        [10.1595],\n",
      "        ...,\n",
      "        [ 1.2201],\n",
      "        [ 1.3347],\n",
      "        [ 1.2632]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[12.6091],\n",
      "        [12.6091],\n",
      "        [12.6091],\n",
      "        ...,\n",
      "        [ 1.7351],\n",
      "        [ 1.7351],\n",
      "        [ 1.7351]], device='cuda:0')\n",
      "Accuracy: 39.1402%\n",
      "tensor([[28.3547],\n",
      "        [63.4533],\n",
      "        [81.7252],\n",
      "        ...,\n",
      "        [ 0.7373],\n",
      "        [ 0.6272],\n",
      "        [ 0.6428]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[214.2887],\n",
      "        [214.2887],\n",
      "        [214.2887],\n",
      "        ...,\n",
      "        [  0.6889],\n",
      "        [  0.6889],\n",
      "        [  0.6889]], device='cuda:0')\n",
      "Accuracy: 38.7976%\n",
      "tensor([[1.1035],\n",
      "        [1.3030],\n",
      "        [2.1136],\n",
      "        ...,\n",
      "        [0.5246],\n",
      "        [0.5279],\n",
      "        [0.5917]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[77.3769],\n",
      "        [77.3769],\n",
      "        [77.3769],\n",
      "        ...,\n",
      "        [ 0.5297],\n",
      "        [ 0.5297],\n",
      "        [ 0.5297]], device='cuda:0')\n",
      "Accuracy: 39.7328%\n",
      "tensor([[0.9875],\n",
      "        [0.9888],\n",
      "        [0.9942],\n",
      "        ...,\n",
      "        [1.1358],\n",
      "        [0.9484],\n",
      "        [1.0132]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.0031],\n",
      "        [1.0031],\n",
      "        [1.0031],\n",
      "        ...,\n",
      "        [1.0928],\n",
      "        [1.0928],\n",
      "        [1.0928]], device='cuda:0')\n",
      "Accuracy: 39.4175%\n",
      "tensor([[14.6370],\n",
      "        [19.4894],\n",
      "        [18.1789],\n",
      "        ...,\n",
      "        [ 0.9971],\n",
      "        [ 0.8656],\n",
      "        [ 0.9496]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[10.8352],\n",
      "        [10.8352],\n",
      "        [10.8352],\n",
      "        ...,\n",
      "        [ 0.6337],\n",
      "        [ 0.6337],\n",
      "        [ 0.6337]], device='cuda:0')\n",
      "Accuracy: 39.8321%\n",
      "tensor([[2.1587],\n",
      "        [2.2499],\n",
      "        [2.0766],\n",
      "        ...,\n",
      "        [0.8286],\n",
      "        [1.1595],\n",
      "        [1.3049]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.9701],\n",
      "        [1.9701],\n",
      "        [1.9701],\n",
      "        ...,\n",
      "        [0.6973],\n",
      "        [0.6973],\n",
      "        [0.6973]], device='cuda:0')\n",
      "Accuracy: 39.2101%\n",
      "tensor([[1.1895],\n",
      "        [1.2553],\n",
      "        [1.3382],\n",
      "        ...,\n",
      "        [0.6514],\n",
      "        [0.6962],\n",
      "        [0.6892]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.6419],\n",
      "        [1.6419],\n",
      "        [1.6419],\n",
      "        ...,\n",
      "        [0.7746],\n",
      "        [0.7746],\n",
      "        [0.7746]], device='cuda:0')\n",
      "Accuracy: 39.8810%\n",
      "tensor([[0.6917],\n",
      "        [0.6471],\n",
      "        [0.6083],\n",
      "        ...,\n",
      "        [0.9881],\n",
      "        [0.6713],\n",
      "        [0.7153]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[9.5957],\n",
      "        [9.5957],\n",
      "        [9.5957],\n",
      "        ...,\n",
      "        [0.5030],\n",
      "        [0.5053],\n",
      "        [0.5053]], device='cuda:0')\n",
      "Accuracy: 39.1109%\n",
      "tensor([[ 7.0304],\n",
      "        [20.3012],\n",
      "        [30.0342],\n",
      "        ...,\n",
      "        [ 0.7923],\n",
      "        [ 1.3118],\n",
      "        [ 1.0547]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[103.9713],\n",
      "        [103.9713],\n",
      "        [103.9713],\n",
      "        ...,\n",
      "        [  0.5964],\n",
      "        [  0.5964],\n",
      "        [  0.5964]], device='cuda:0')\n",
      "Accuracy: 39.0858%\n",
      "tensor([[ 9.0399],\n",
      "        [11.7952],\n",
      "        [11.1465],\n",
      "        ...,\n",
      "        [ 0.6030],\n",
      "        [ 0.7784],\n",
      "        [ 0.8715]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[15.2125],\n",
      "        [15.2125],\n",
      "        [15.2125],\n",
      "        ...,\n",
      "        [ 0.8757],\n",
      "        [ 0.8757],\n",
      "        [ 0.8757]], device='cuda:0')\n",
      "Accuracy: 39.4866%\n",
      "tensor([[0.9135],\n",
      "        [1.0143],\n",
      "        [1.2914],\n",
      "        ...,\n",
      "        [0.8385],\n",
      "        [0.8471],\n",
      "        [0.7578]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.0432],\n",
      "        [1.0432],\n",
      "        [1.0432],\n",
      "        ...,\n",
      "        [0.5182],\n",
      "        [0.5182],\n",
      "        [0.5182]], device='cuda:0')\n",
      "Accuracy: 40.1606%\n",
      "tensor([[1.6345],\n",
      "        [1.6915],\n",
      "        [1.5431],\n",
      "        ...,\n",
      "        [0.6044],\n",
      "        [0.7311],\n",
      "        [0.9333]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.9438],\n",
      "        [1.9438],\n",
      "        [1.9438],\n",
      "        ...,\n",
      "        [0.5980],\n",
      "        [0.5980],\n",
      "        [0.5980]], device='cuda:0')\n",
      "Accuracy: 39.1989%\n",
      "tensor([[10.2383],\n",
      "        [ 8.0740],\n",
      "        [ 7.1587],\n",
      "        ...,\n",
      "        [ 1.0393],\n",
      "        [ 1.1273],\n",
      "        [ 1.0783]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[7.2037],\n",
      "        [7.2037],\n",
      "        [7.2037],\n",
      "        ...,\n",
      "        [1.1689],\n",
      "        [1.1689],\n",
      "        [1.1689]], device='cuda:0')\n",
      "Accuracy: 39.5244%\n",
      "tensor([[0.7564],\n",
      "        [0.7558],\n",
      "        [0.6647],\n",
      "        ...,\n",
      "        [0.6475],\n",
      "        [0.6273],\n",
      "        [0.6765]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.7752],\n",
      "        [0.7752],\n",
      "        [0.7752],\n",
      "        ...,\n",
      "        [0.5488],\n",
      "        [0.5488],\n",
      "        [0.5488]], device='cuda:0')\n",
      "Accuracy: 40.0390%\n",
      "tensor([[ 2.1219],\n",
      "        [ 4.1635],\n",
      "        [31.8953],\n",
      "        ...,\n",
      "        [ 0.7448],\n",
      "        [ 0.6810],\n",
      "        [ 0.6063]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[103.9713],\n",
      "        [103.9713],\n",
      "        [103.9713],\n",
      "        ...,\n",
      "        [  0.6936],\n",
      "        [  0.6936],\n",
      "        [  0.6936]], device='cuda:0')\n",
      "Accuracy: 40.3640%\n",
      "tensor([[5.5135],\n",
      "        [4.3961],\n",
      "        [4.6118],\n",
      "        ...,\n",
      "        [0.6515],\n",
      "        [0.7172],\n",
      "        [0.7335]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[4.7416],\n",
      "        [4.7416],\n",
      "        [4.7416],\n",
      "        ...,\n",
      "        [0.5174],\n",
      "        [0.5174],\n",
      "        [0.5174]], device='cuda:0')\n",
      "Accuracy: 41.8565%\n",
      "tensor([[0.7226],\n",
      "        [0.8478],\n",
      "        [0.6117],\n",
      "        ...,\n",
      "        [0.7012],\n",
      "        [0.7081],\n",
      "        [0.6395]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[18.2959],\n",
      "        [18.2959],\n",
      "        [18.2959],\n",
      "        ...,\n",
      "        [ 0.5716],\n",
      "        [ 0.5716],\n",
      "        [ 0.5716]], device='cuda:0')\n",
      "Accuracy: 39.6079%\n",
      "tensor([[0.7796],\n",
      "        [0.8318],\n",
      "        [0.7697],\n",
      "        ...,\n",
      "        [0.9296],\n",
      "        [0.9273],\n",
      "        [1.0580]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.8145],\n",
      "        [0.8145],\n",
      "        [0.5644],\n",
      "        ...,\n",
      "        [1.2768],\n",
      "        [1.2768],\n",
      "        [1.2768]], device='cuda:0')\n",
      "Accuracy: 39.3381%\n",
      "tensor([[10.2519],\n",
      "        [ 8.6586],\n",
      "        [ 7.7297],\n",
      "        ...,\n",
      "        [ 1.2707],\n",
      "        [ 1.0032],\n",
      "        [ 0.6826]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[12.3305],\n",
      "        [12.3305],\n",
      "        [12.3305],\n",
      "        ...,\n",
      "        [ 0.6648],\n",
      "        [ 0.6648],\n",
      "        [ 0.6648]], device='cuda:0')\n",
      "Accuracy: 38.8127%\n",
      "tensor([[0.9996],\n",
      "        [0.8601],\n",
      "        [0.9708],\n",
      "        ...,\n",
      "        [0.6847],\n",
      "        [0.6144],\n",
      "        [0.5791]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.2671],\n",
      "        [1.2671],\n",
      "        [1.2671],\n",
      "        ...,\n",
      "        [0.6528],\n",
      "        [0.6528],\n",
      "        [0.6528]], device='cuda:0')\n",
      "Accuracy: 40.3468%\n",
      "tensor([[19.1213],\n",
      "        [16.8965],\n",
      "        [17.0006],\n",
      "        ...,\n",
      "        [ 1.2032],\n",
      "        [ 1.2859],\n",
      "        [ 1.1205]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[11.3675],\n",
      "        [11.3675],\n",
      "        [11.3675],\n",
      "        ...,\n",
      "        [ 1.1216],\n",
      "        [ 1.1216],\n",
      "        [ 1.1216]], device='cuda:0')\n",
      "Accuracy: 40.6948%\n",
      "tensor([[3.6168],\n",
      "        [5.2099],\n",
      "        [5.1324],\n",
      "        ...,\n",
      "        [2.8753],\n",
      "        [3.4861],\n",
      "        [2.0596]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[5.4117],\n",
      "        [5.4117],\n",
      "        [5.4117],\n",
      "        ...,\n",
      "        [1.7091],\n",
      "        [1.7091],\n",
      "        [1.7091]], device='cuda:0')\n",
      "Accuracy: 40.0502%\n",
      "tensor([[10.3463],\n",
      "        [ 8.6515],\n",
      "        [ 7.7605],\n",
      "        ...,\n",
      "        [ 0.6446],\n",
      "        [ 0.6453],\n",
      "        [ 0.7479]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[12.3305],\n",
      "        [12.3305],\n",
      "        [12.3305],\n",
      "        ...,\n",
      "        [ 0.5811],\n",
      "        [ 0.5811],\n",
      "        [ 0.5811]], device='cuda:0')\n",
      "Accuracy: 39.0119%\n",
      "tensor([[0.9783],\n",
      "        [1.0397],\n",
      "        [1.0651],\n",
      "        ...,\n",
      "        [0.6647],\n",
      "        [0.7699],\n",
      "        [0.8835]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[41.3389],\n",
      "        [41.3389],\n",
      "        [41.3389],\n",
      "        ...,\n",
      "        [ 2.3012],\n",
      "        [ 2.3012],\n",
      "        [ 2.3012]], device='cuda:0')\n",
      "Accuracy: 40.6840%\n",
      "tensor([[0.6630],\n",
      "        [0.6878],\n",
      "        [0.6975],\n",
      "        ...,\n",
      "        [0.8447],\n",
      "        [0.8768],\n",
      "        [1.0330]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.7537],\n",
      "        [0.7537],\n",
      "        [0.7537],\n",
      "        ...,\n",
      "        [0.7055],\n",
      "        [0.7055],\n",
      "        [0.7055]], device='cuda:0')\n",
      "Accuracy: 39.3853%\n",
      "tensor([[3.9761],\n",
      "        [4.5978],\n",
      "        [6.2752],\n",
      "        ...,\n",
      "        [0.6555],\n",
      "        [0.9929],\n",
      "        [0.9387]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[114.3767],\n",
      "        [114.3767],\n",
      "        [114.3767],\n",
      "        ...,\n",
      "        [  0.5394],\n",
      "        [  0.5394],\n",
      "        [  0.5394]], device='cuda:0')\n",
      "Accuracy: 40.5195%\n",
      "tensor([[0.9683],\n",
      "        [1.0184],\n",
      "        [1.0687],\n",
      "        ...,\n",
      "        [2.1319],\n",
      "        [2.1325],\n",
      "        [2.3955]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[2.1535],\n",
      "        [2.1535],\n",
      "        [2.1535],\n",
      "        ...,\n",
      "        [2.0440],\n",
      "        [2.0440],\n",
      "        [2.0440]], device='cuda:0')\n",
      "Accuracy: 40.0183%\n",
      "tensor([[1.0741],\n",
      "        [1.1577],\n",
      "        [1.1884],\n",
      "        ...,\n",
      "        [0.7665],\n",
      "        [0.7874],\n",
      "        [0.8470]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.0568],\n",
      "        [1.0568],\n",
      "        [1.0568],\n",
      "        ...,\n",
      "        [0.5111],\n",
      "        [0.5111],\n",
      "        [0.5111]], device='cuda:0')\n",
      "Accuracy: 38.4389%\n",
      "tensor([[1.6255],\n",
      "        [1.6056],\n",
      "        [1.8206],\n",
      "        ...,\n",
      "        [0.7076],\n",
      "        [0.8283],\n",
      "        [0.8569]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.5487],\n",
      "        [1.5487],\n",
      "        [1.5487],\n",
      "        ...,\n",
      "        [0.8363],\n",
      "        [0.8363],\n",
      "        [0.8363]], device='cuda:0')\n",
      "Accuracy: 39.5117%\n",
      "tensor([[0.6603],\n",
      "        [1.6645],\n",
      "        [4.5902],\n",
      "        ...,\n",
      "        [0.7540],\n",
      "        [0.7077],\n",
      "        [0.6972]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[34.2068],\n",
      "        [34.2068],\n",
      "        [34.2068],\n",
      "        ...,\n",
      "        [ 0.5591],\n",
      "        [ 0.5591],\n",
      "        [ 0.5591]], device='cuda:0')\n",
      "Accuracy: 40.2999%\n",
      "tensor([[10.2161],\n",
      "        [ 8.5189],\n",
      "        [ 7.6812],\n",
      "        ...,\n",
      "        [ 0.6470],\n",
      "        [ 0.6614],\n",
      "        [ 0.5862]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[12.3305],\n",
      "        [12.3305],\n",
      "        [12.3305],\n",
      "        ...,\n",
      "        [ 0.5768],\n",
      "        [ 0.5768],\n",
      "        [ 0.5768]], device='cuda:0')\n",
      "Accuracy: 40.0605%\n",
      "tensor([[0.9681],\n",
      "        [1.0175],\n",
      "        [1.0679],\n",
      "        ...,\n",
      "        [0.6423],\n",
      "        [0.6531],\n",
      "        [0.6582]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[2.1535],\n",
      "        [2.1535],\n",
      "        [2.1535],\n",
      "        ...,\n",
      "        [0.5419],\n",
      "        [0.5419],\n",
      "        [0.5419]], device='cuda:0')\n",
      "Accuracy: 39.3605%\n",
      "tensor([[0.9883],\n",
      "        [0.9892],\n",
      "        [0.9956],\n",
      "        ...,\n",
      "        [0.7058],\n",
      "        [0.7919],\n",
      "        [0.9199]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.0031],\n",
      "        [1.0031],\n",
      "        [1.0031],\n",
      "        ...,\n",
      "        [0.7900],\n",
      "        [0.7900],\n",
      "        [0.7900]], device='cuda:0')\n",
      "Accuracy: 39.9850%\n",
      "tensor([[1.4345],\n",
      "        [1.4077],\n",
      "        [1.2925],\n",
      "        ...,\n",
      "        [1.1382],\n",
      "        [1.3562],\n",
      "        [1.0842]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.5719],\n",
      "        [1.5719],\n",
      "        [1.5719],\n",
      "        ...,\n",
      "        [1.4316],\n",
      "        [1.4316],\n",
      "        [1.4316]], device='cuda:0')\n",
      "Accuracy: 39.1546%\n",
      "tensor([[3.0630],\n",
      "        [2.7623],\n",
      "        [3.2913],\n",
      "        ...,\n",
      "        [0.6739],\n",
      "        [0.5828],\n",
      "        [0.6157]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[49.4070],\n",
      "        [49.4070],\n",
      "        [49.4070],\n",
      "        ...,\n",
      "        [ 0.5294],\n",
      "        [ 0.5294],\n",
      "        [ 0.5294]], device='cuda:0')\n",
      "Accuracy: 40.2626%\n",
      "tensor([[11.9847],\n",
      "        [13.2803],\n",
      "        [10.4962],\n",
      "        ...,\n",
      "        [ 0.7002],\n",
      "        [ 0.5775],\n",
      "        [ 0.5533]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[103.9713],\n",
      "        [103.9713],\n",
      "        [103.9713],\n",
      "        ...,\n",
      "        [  0.5163],\n",
      "        [  0.5163],\n",
      "        [  0.5163]], device='cuda:0')\n",
      "Accuracy: 40.2963%\n",
      "tensor([[1.0355],\n",
      "        [0.8637],\n",
      "        [0.8407],\n",
      "        ...,\n",
      "        [1.0183],\n",
      "        [0.8709],\n",
      "        [0.9781]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[7.4545],\n",
      "        [7.4545],\n",
      "        [7.4545],\n",
      "        ...,\n",
      "        [0.5515],\n",
      "        [0.5515],\n",
      "        [0.5515]], device='cuda:0')\n",
      "Accuracy: 39.2724%\n",
      "tensor([[1.6573],\n",
      "        [1.7834],\n",
      "        [1.5776],\n",
      "        ...,\n",
      "        [0.7427],\n",
      "        [0.7284],\n",
      "        [0.8567]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.6736],\n",
      "        [1.6736],\n",
      "        [1.6736],\n",
      "        ...,\n",
      "        [0.5688],\n",
      "        [0.5688],\n",
      "        [0.5688]], device='cuda:0')\n",
      "Accuracy: 40.3407%\n",
      "tensor([[11.2320],\n",
      "        [13.0427],\n",
      "        [16.5677],\n",
      "        ...,\n",
      "        [ 0.6959],\n",
      "        [ 0.6481],\n",
      "        [ 0.7528]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[15.9097],\n",
      "        [15.9097],\n",
      "        [15.9097],\n",
      "        ...,\n",
      "        [ 0.5118],\n",
      "        [ 0.5118],\n",
      "        [ 0.5118]], device='cuda:0')\n",
      "Accuracy: 39.8308%\n",
      "tensor([[3.6314],\n",
      "        [5.2184],\n",
      "        [5.1166],\n",
      "        ...,\n",
      "        [0.6317],\n",
      "        [0.6797],\n",
      "        [0.5500]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[5.4117],\n",
      "        [5.4117],\n",
      "        [5.4117],\n",
      "        ...,\n",
      "        [0.5336],\n",
      "        [0.5336],\n",
      "        [0.5336]], device='cuda:0')\n",
      "Accuracy: 39.5466%\n",
      "tensor([[ 0.6881],\n",
      "        [11.6658],\n",
      "        [ 6.5289],\n",
      "        ...,\n",
      "        [ 0.6949],\n",
      "        [ 0.7447],\n",
      "        [ 0.8752]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[18.9864],\n",
      "        [18.9864],\n",
      "        [18.9864],\n",
      "        ...,\n",
      "        [ 0.5053],\n",
      "        [ 0.5053],\n",
      "        [ 0.5053]], device='cuda:0')\n",
      "Accuracy: 39.3142%\n",
      "tensor([[8.2489],\n",
      "        [7.2578],\n",
      "        [7.3865],\n",
      "        ...,\n",
      "        [0.6723],\n",
      "        [0.6653],\n",
      "        [0.6261]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[7.0816],\n",
      "        [7.0816],\n",
      "        [7.0816],\n",
      "        ...,\n",
      "        [0.6795],\n",
      "        [0.6795],\n",
      "        [0.6795]], device='cuda:0')\n",
      "Accuracy: 39.4279%\n",
      "tensor([[42.7952],\n",
      "        [48.9230],\n",
      "        [46.6251],\n",
      "        ...,\n",
      "        [ 0.6276],\n",
      "        [ 0.6505],\n",
      "        [ 0.5349]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[214.2887],\n",
      "        [214.2887],\n",
      "        [214.2887],\n",
      "        ...,\n",
      "        [  0.5513],\n",
      "        [  0.5513],\n",
      "        [  0.5513]], device='cuda:0')\n",
      "Accuracy: 40.1457%\n",
      "tensor([[1.6206],\n",
      "        [1.4773],\n",
      "        [2.0015],\n",
      "        ...,\n",
      "        [0.8882],\n",
      "        [0.9505],\n",
      "        [0.8475]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[2.0819],\n",
      "        [2.0819],\n",
      "        [2.0819],\n",
      "        ...,\n",
      "        [0.9285],\n",
      "        [0.9285],\n",
      "        [0.9285]], device='cuda:0')\n",
      "Accuracy: 40.2122%\n",
      "tensor([[1.3236],\n",
      "        [1.3064],\n",
      "        [1.1554],\n",
      "        ...,\n",
      "        [0.6435],\n",
      "        [0.6298],\n",
      "        [0.6163]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.2227],\n",
      "        [1.2227],\n",
      "        [1.2227],\n",
      "        ...,\n",
      "        [0.5580],\n",
      "        [0.5580],\n",
      "        [0.5580]], device='cuda:0')\n",
      "Accuracy: 39.8365%\n",
      "tensor([[0.6796],\n",
      "        [0.7947],\n",
      "        [0.8771],\n",
      "        ...,\n",
      "        [0.7633],\n",
      "        [0.9490],\n",
      "        [1.0927]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[18.4249],\n",
      "        [18.4249],\n",
      "        [18.4249],\n",
      "        ...,\n",
      "        [ 0.8176],\n",
      "        [ 0.8176],\n",
      "        [ 0.8176]], device='cuda:0')\n",
      "Accuracy: 37.5792%\n",
      "tensor([[1.0066],\n",
      "        [0.8420],\n",
      "        [0.9938],\n",
      "        ...,\n",
      "        [0.7329],\n",
      "        [0.8331],\n",
      "        [0.7742]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.7756],\n",
      "        [1.7756],\n",
      "        [1.7756],\n",
      "        ...,\n",
      "        [0.8164],\n",
      "        [0.8164],\n",
      "        [0.8164]], device='cuda:0')\n",
      "Accuracy: 40.5097%\n",
      "tensor([[0.7226],\n",
      "        [0.8432],\n",
      "        [0.6172],\n",
      "        ...,\n",
      "        [1.5072],\n",
      "        [1.1013],\n",
      "        [1.3276]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[9.5957],\n",
      "        [9.5957],\n",
      "        [9.5957],\n",
      "        ...,\n",
      "        [1.1248],\n",
      "        [1.1248],\n",
      "        [1.1248]], device='cuda:0')\n",
      "Accuracy: 39.2459%\n",
      "tensor([[1.1484],\n",
      "        [1.2153],\n",
      "        [1.1721],\n",
      "        ...,\n",
      "        [0.6316],\n",
      "        [0.6301],\n",
      "        [0.6537]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.7752],\n",
      "        [0.7752],\n",
      "        [0.7752],\n",
      "        ...,\n",
      "        [0.5931],\n",
      "        [0.5931],\n",
      "        [0.5931]], device='cuda:0')\n",
      "Accuracy: 40.2219%\n",
      "tensor([[2.6271],\n",
      "        [2.6988],\n",
      "        [3.3629],\n",
      "        ...,\n",
      "        [0.6870],\n",
      "        [0.6212],\n",
      "        [0.6703]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[3.5017],\n",
      "        [3.5017],\n",
      "        [3.5017],\n",
      "        ...,\n",
      "        [0.6006],\n",
      "        [0.6006],\n",
      "        [0.6006]], device='cuda:0')\n",
      "Accuracy: 39.0849%\n",
      "tensor([[0.6733],\n",
      "        [0.6504],\n",
      "        [0.6597],\n",
      "        ...,\n",
      "        [1.0149],\n",
      "        [1.2987],\n",
      "        [0.9066]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.6718],\n",
      "        [0.6718],\n",
      "        [0.6718],\n",
      "        ...,\n",
      "        [0.6416],\n",
      "        [0.6416],\n",
      "        [0.6416]], device='cuda:0')\n",
      "Accuracy: 40.3180%\n",
      "tensor([[0.9803],\n",
      "        [0.9679],\n",
      "        [0.9910],\n",
      "        ...,\n",
      "        [0.6828],\n",
      "        [0.6682],\n",
      "        [0.6200]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[5.4676],\n",
      "        [5.4676],\n",
      "        [5.4676],\n",
      "        ...,\n",
      "        [0.7276],\n",
      "        [0.7276],\n",
      "        [0.7276]], device='cuda:0')\n",
      "Accuracy: 40.6631%\n",
      "tensor([[14.7487],\n",
      "        [19.8781],\n",
      "        [18.3167],\n",
      "        ...,\n",
      "        [ 0.6486],\n",
      "        [ 0.6342],\n",
      "        [ 0.6402]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[10.8352],\n",
      "        [10.8352],\n",
      "        [10.8352],\n",
      "        ...,\n",
      "        [ 0.6237],\n",
      "        [ 0.6237],\n",
      "        [ 0.6237]], device='cuda:0')\n",
      "Accuracy: 39.3348%\n",
      "tensor([[1.0005],\n",
      "        [1.1341],\n",
      "        [1.1276],\n",
      "        ...,\n",
      "        [0.6037],\n",
      "        [0.6033],\n",
      "        [0.5284]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.6658],\n",
      "        [0.6658],\n",
      "        [0.6658],\n",
      "        ...,\n",
      "        [0.5664],\n",
      "        [0.5664],\n",
      "        [0.5664]], device='cuda:0')\n",
      "Accuracy: 40.1885%\n",
      "tensor([[16.8904],\n",
      "        [20.8417],\n",
      "        [20.3371],\n",
      "        ...,\n",
      "        [ 0.7235],\n",
      "        [ 0.6742],\n",
      "        [ 0.7453]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[62.8775],\n",
      "        [62.8775],\n",
      "        [62.8775],\n",
      "        ...,\n",
      "        [ 0.7276],\n",
      "        [ 0.7276],\n",
      "        [ 0.7276]], device='cuda:0')\n",
      "Accuracy: 39.7209%\n",
      "tensor([[0.8520],\n",
      "        [0.8058],\n",
      "        [1.3407],\n",
      "        ...,\n",
      "        [0.8619],\n",
      "        [0.9751],\n",
      "        [0.8014]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.1269],\n",
      "        [1.1269],\n",
      "        [1.1269],\n",
      "        ...,\n",
      "        [0.7729],\n",
      "        [0.7729],\n",
      "        [0.7729]], device='cuda:0')\n",
      "Accuracy: 40.4717%\n",
      "tensor([[0.6712],\n",
      "        [0.6263],\n",
      "        [0.5953],\n",
      "        ...,\n",
      "        [0.5279],\n",
      "        [0.5198],\n",
      "        [0.5139]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[9.9181],\n",
      "        [9.9181],\n",
      "        [9.9181],\n",
      "        ...,\n",
      "        [0.5163],\n",
      "        [0.5163],\n",
      "        [0.5163]], device='cuda:0')\n",
      "Accuracy: 39.5578%\n",
      "tensor([[0.8464],\n",
      "        [0.8655],\n",
      "        [0.9052],\n",
      "        ...,\n",
      "        [0.8541],\n",
      "        [0.9607],\n",
      "        [0.9901]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[28.2485],\n",
      "        [28.2485],\n",
      "        [28.2485],\n",
      "        ...,\n",
      "        [ 0.5693],\n",
      "        [ 0.5693],\n",
      "        [ 0.5693]], device='cuda:0')\n",
      "Accuracy: 38.7823%\n",
      "tensor([[0.9980],\n",
      "        [0.8870],\n",
      "        [0.9275],\n",
      "        ...,\n",
      "        [1.7981],\n",
      "        [1.4017],\n",
      "        [1.2597]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.8174],\n",
      "        [1.8174],\n",
      "        [1.8174],\n",
      "        ...,\n",
      "        [1.5891],\n",
      "        [1.5891],\n",
      "        [1.5891]], device='cuda:0')\n",
      "Accuracy: 39.9508%\n",
      "tensor([[0.9969],\n",
      "        [0.9015],\n",
      "        [0.9987],\n",
      "        ...,\n",
      "        [0.6854],\n",
      "        [0.7988],\n",
      "        [0.6971]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.6272],\n",
      "        [0.6272],\n",
      "        [0.6272],\n",
      "        ...,\n",
      "        [0.5574],\n",
      "        [0.5574],\n",
      "        [0.5574]], device='cuda:0')\n",
      "Accuracy: 39.1757%\n",
      "tensor([[5.6407],\n",
      "        [5.8813],\n",
      "        [5.0981],\n",
      "        ...,\n",
      "        [0.6493],\n",
      "        [0.5865],\n",
      "        [0.6767]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[4.2333],\n",
      "        [4.2333],\n",
      "        [4.2333],\n",
      "        ...,\n",
      "        [0.6468],\n",
      "        [0.6468],\n",
      "        [0.6468]], device='cuda:0')\n",
      "Accuracy: 39.7403%\n",
      "tensor([[2.6274],\n",
      "        [2.7943],\n",
      "        [3.6031],\n",
      "        ...,\n",
      "        [1.1684],\n",
      "        [1.0011],\n",
      "        [1.0178]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[3.5017],\n",
      "        [3.5017],\n",
      "        [3.5017],\n",
      "        ...,\n",
      "        [1.0661],\n",
      "        [1.0661],\n",
      "        [1.0661]], device='cuda:0')\n",
      "Accuracy: 38.7006%\n",
      "tensor([[3.1372],\n",
      "        [4.5701],\n",
      "        [7.6276],\n",
      "        ...,\n",
      "        [0.5648],\n",
      "        [0.6136],\n",
      "        [0.7399]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[18.9803],\n",
      "        [18.9803],\n",
      "        [18.9803],\n",
      "        ...,\n",
      "        [ 0.7229],\n",
      "        [ 0.7229],\n",
      "        [ 0.7229]], device='cuda:0')\n",
      "Accuracy: 39.4491%\n",
      "tensor([[12.9539],\n",
      "        [12.2334],\n",
      "        [11.3214],\n",
      "        ...,\n",
      "        [ 0.6566],\n",
      "        [ 0.5275],\n",
      "        [ 0.5185]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[10.0892],\n",
      "        [10.0892],\n",
      "        [10.0892],\n",
      "        ...,\n",
      "        [ 0.5517],\n",
      "        [ 0.5517],\n",
      "        [ 0.5517]], device='cuda:0')\n",
      "Accuracy: 40.2761%\n",
      "tensor([[3.6755],\n",
      "        [3.6912],\n",
      "        [4.2473],\n",
      "        ...,\n",
      "        [0.8420],\n",
      "        [0.9263],\n",
      "        [0.8659]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[4.1409],\n",
      "        [4.1409],\n",
      "        [4.1409],\n",
      "        ...,\n",
      "        [0.6053],\n",
      "        [0.6053],\n",
      "        [0.6053]], device='cuda:0')\n",
      "Accuracy: 39.6207%\n",
      "tensor([[0.9930],\n",
      "        [1.1189],\n",
      "        [1.1182],\n",
      "        ...,\n",
      "        [1.0980],\n",
      "        [0.9079],\n",
      "        [0.9797]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.6542],\n",
      "        [1.6542],\n",
      "        [1.6542],\n",
      "        ...,\n",
      "        [1.0477],\n",
      "        [1.0477],\n",
      "        [1.0477]], device='cuda:0')\n",
      "Accuracy: 40.8742%\n",
      "tensor([[2.3119],\n",
      "        [3.0012],\n",
      "        [3.4853],\n",
      "        ...,\n",
      "        [0.7756],\n",
      "        [0.8348],\n",
      "        [0.7605]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[4.2260],\n",
      "        [4.2260],\n",
      "        [4.2260],\n",
      "        ...,\n",
      "        [0.8264],\n",
      "        [0.8264],\n",
      "        [0.8264]], device='cuda:0')\n",
      "Accuracy: 39.4977%\n",
      "tensor([[0.9552],\n",
      "        [0.9512],\n",
      "        [1.1259],\n",
      "        ...,\n",
      "        [0.7132],\n",
      "        [0.8764],\n",
      "        [0.7109]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[18.9803],\n",
      "        [18.9803],\n",
      "        [18.9803],\n",
      "        ...,\n",
      "        [ 0.5077],\n",
      "        [ 0.5077],\n",
      "        [ 0.5077]], device='cuda:0')\n",
      "Accuracy: 40.1390%\n",
      "tensor([[1.0961],\n",
      "        [1.0131],\n",
      "        [0.9886],\n",
      "        ...,\n",
      "        [0.9215],\n",
      "        [0.8816],\n",
      "        [1.0348]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.6300],\n",
      "        [0.6300],\n",
      "        [0.6300],\n",
      "        ...,\n",
      "        [0.6538],\n",
      "        [0.6538],\n",
      "        [0.6538]], device='cuda:0')\n",
      "Accuracy: 39.7940%\n",
      "tensor([[0.9016],\n",
      "        [1.0014],\n",
      "        [1.2399],\n",
      "        ...,\n",
      "        [0.6275],\n",
      "        [0.6268],\n",
      "        [0.6755]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.0432],\n",
      "        [1.0432],\n",
      "        [1.0432],\n",
      "        ...,\n",
      "        [0.6954],\n",
      "        [0.6954],\n",
      "        [0.6954]], device='cuda:0')\n",
      "Accuracy: 39.1318%\n",
      "tensor([[0.9803],\n",
      "        [0.9692],\n",
      "        [0.9925],\n",
      "        ...,\n",
      "        [0.5200],\n",
      "        [0.5255],\n",
      "        [0.5377]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[5.4676],\n",
      "        [5.4676],\n",
      "        [5.4676],\n",
      "        ...,\n",
      "        [0.5036],\n",
      "        [0.5036],\n",
      "        [0.5036]], device='cuda:0')\n",
      "Accuracy: 39.6506%\n",
      "tensor([[19.0480],\n",
      "        [23.0835],\n",
      "        [21.6653],\n",
      "        ...,\n",
      "        [ 1.5495],\n",
      "        [ 1.5566],\n",
      "        [ 1.3213]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[66.0761],\n",
      "        [66.0761],\n",
      "        [66.0761],\n",
      "        ...,\n",
      "        [ 1.1489],\n",
      "        [ 1.1489],\n",
      "        [ 1.1489]], device='cuda:0')\n",
      "Accuracy: 39.7656%\n",
      "tensor([[10.7145],\n",
      "        [ 6.7402],\n",
      "        [ 8.7517],\n",
      "        ...,\n",
      "        [ 0.8141],\n",
      "        [ 0.7114],\n",
      "        [ 0.8462]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[9.6110],\n",
      "        [9.6110],\n",
      "        [9.6110],\n",
      "        ...,\n",
      "        [0.7663],\n",
      "        [0.7663],\n",
      "        [0.7663]], device='cuda:0')\n",
      "Accuracy: 38.7527%\n",
      "tensor([[10.4775],\n",
      "        [ 8.1635],\n",
      "        [ 6.9152],\n",
      "        ...,\n",
      "        [ 0.8971],\n",
      "        [ 1.0843],\n",
      "        [ 0.8726]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[7.2037],\n",
      "        [7.2037],\n",
      "        [7.2037],\n",
      "        ...,\n",
      "        [0.8917],\n",
      "        [0.8917],\n",
      "        [0.8917]], device='cuda:0')\n",
      "Accuracy: 38.9668%\n",
      "tensor([[32.4640],\n",
      "        [32.3690],\n",
      "        [43.8462],\n",
      "        ...,\n",
      "        [ 1.8761],\n",
      "        [ 1.6347],\n",
      "        [ 1.5169]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[77.1132],\n",
      "        [77.1132],\n",
      "        [77.1132],\n",
      "        ...,\n",
      "        [ 1.6917],\n",
      "        [ 1.6917],\n",
      "        [ 1.6917]], device='cuda:0')\n",
      "Accuracy: 39.9223%\n",
      "tensor([[0.9733],\n",
      "        [0.9466],\n",
      "        [0.9591],\n",
      "        ...,\n",
      "        [0.5838],\n",
      "        [0.5999],\n",
      "        [0.7291]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.9247],\n",
      "        [0.9247],\n",
      "        [0.9247],\n",
      "        ...,\n",
      "        [0.5667],\n",
      "        [0.5667],\n",
      "        [0.5667]], device='cuda:0')\n",
      "Accuracy: 39.6083%\n",
      "tensor([[1.0348],\n",
      "        [0.8618],\n",
      "        [0.8457],\n",
      "        ...,\n",
      "        [0.8780],\n",
      "        [0.9837],\n",
      "        [0.5215]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[7.4545],\n",
      "        [7.4545],\n",
      "        [7.4545],\n",
      "        ...,\n",
      "        [1.2338],\n",
      "        [1.2338],\n",
      "        [1.2338]], device='cuda:0')\n",
      "Accuracy: 39.0186%\n",
      "tensor([[1.0539],\n",
      "        [1.0836],\n",
      "        [1.0144],\n",
      "        ...,\n",
      "        [0.9879],\n",
      "        [0.8594],\n",
      "        [0.7651]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.6624],\n",
      "        [0.6624],\n",
      "        [0.6624],\n",
      "        ...,\n",
      "        [1.0003],\n",
      "        [1.0003],\n",
      "        [1.0003]], device='cuda:0')\n",
      "Accuracy: 40.4452%\n",
      "tensor([[0.6560],\n",
      "        [0.7484],\n",
      "        [0.8034],\n",
      "        ...,\n",
      "        [0.8664],\n",
      "        [0.8966],\n",
      "        [0.7544]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.5559],\n",
      "        [0.5559],\n",
      "        [0.5559],\n",
      "        ...,\n",
      "        [1.0908],\n",
      "        [1.0908],\n",
      "        [1.1814]], device='cuda:0')\n",
      "Accuracy: 39.5069%\n",
      "tensor([[0.6713],\n",
      "        [0.6502],\n",
      "        [0.6608],\n",
      "        ...,\n",
      "        [1.1412],\n",
      "        [1.2072],\n",
      "        [0.9134]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.6718],\n",
      "        [0.6718],\n",
      "        [0.6718],\n",
      "        ...,\n",
      "        [1.1390],\n",
      "        [1.1390],\n",
      "        [1.1390]], device='cuda:0')\n",
      "Accuracy: 39.1823%\n",
      "tensor([[1.6302],\n",
      "        [1.2663],\n",
      "        [1.7483],\n",
      "        ...,\n",
      "        [0.7675],\n",
      "        [0.7321],\n",
      "        [0.6926]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.7191],\n",
      "        [1.7191],\n",
      "        [1.7191],\n",
      "        ...,\n",
      "        [0.7097],\n",
      "        [0.7097],\n",
      "        [0.7097]], device='cuda:0')\n",
      "Accuracy: 40.5287%\n",
      "tensor([[0.9975],\n",
      "        [0.9531],\n",
      "        [0.9131],\n",
      "        ...,\n",
      "        [1.1248],\n",
      "        [0.7863],\n",
      "        [1.0304]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[3.8672],\n",
      "        [3.8672],\n",
      "        [3.8672],\n",
      "        ...,\n",
      "        [0.5144],\n",
      "        [0.5074],\n",
      "        [0.5074]], device='cuda:0')\n",
      "Accuracy: 40.6298%\n",
      "tensor([[0.8473],\n",
      "        [0.8746],\n",
      "        [0.7258],\n",
      "        ...,\n",
      "        [0.6936],\n",
      "        [0.7743],\n",
      "        [0.7342]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.0484],\n",
      "        [1.0484],\n",
      "        [1.0484],\n",
      "        ...,\n",
      "        [0.7965],\n",
      "        [0.7965],\n",
      "        [0.7965]], device='cuda:0')\n",
      "Accuracy: 39.0264%\n",
      "tensor([[8.5380],\n",
      "        [7.4409],\n",
      "        [8.9080],\n",
      "        ...,\n",
      "        [0.9483],\n",
      "        [0.8375],\n",
      "        [0.7380]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[6.1454],\n",
      "        [6.1454],\n",
      "        [6.1454],\n",
      "        ...,\n",
      "        [0.8144],\n",
      "        [0.8144],\n",
      "        [0.8144]], device='cuda:0')\n",
      "Accuracy: 39.3465%\n",
      "tensor([[0.9399],\n",
      "        [0.9246],\n",
      "        [0.9130],\n",
      "        ...,\n",
      "        [0.6532],\n",
      "        [0.6188],\n",
      "        [0.6643]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.7312],\n",
      "        [0.7312],\n",
      "        [0.7312],\n",
      "        ...,\n",
      "        [0.6452],\n",
      "        [0.6452],\n",
      "        [0.6452]], device='cuda:0')\n",
      "Accuracy: 38.6133%\n",
      "tensor([[17.3492],\n",
      "        [21.3735],\n",
      "        [20.9378],\n",
      "        ...,\n",
      "        [ 0.7750],\n",
      "        [ 0.7245],\n",
      "        [ 0.8494]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[62.8775],\n",
      "        [62.8775],\n",
      "        [62.8775],\n",
      "        ...,\n",
      "        [ 1.1988],\n",
      "        [ 0.5820],\n",
      "        [ 0.5820]], device='cuda:0')\n",
      "Accuracy: 40.3640%\n",
      "tensor([[0.7486],\n",
      "        [0.7896],\n",
      "        [0.9446],\n",
      "        ...,\n",
      "        [0.9740],\n",
      "        [0.8918],\n",
      "        [0.8121]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.9247],\n",
      "        [0.9247],\n",
      "        [0.9247],\n",
      "        ...,\n",
      "        [0.6448],\n",
      "        [0.6448],\n",
      "        [0.6448]], device='cuda:0')\n",
      "Accuracy: 38.9745%\n",
      "tensor([[0.7439],\n",
      "        [0.7359],\n",
      "        [0.6380],\n",
      "        ...,\n",
      "        [0.5949],\n",
      "        [0.8614],\n",
      "        [0.9297]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.7752],\n",
      "        [0.7752],\n",
      "        [0.7752],\n",
      "        ...,\n",
      "        [0.6405],\n",
      "        [0.6405],\n",
      "        [0.6405]], device='cuda:0')\n",
      "Accuracy: 40.1297%\n",
      "tensor([[1.0853],\n",
      "        [1.0139],\n",
      "        [1.0190],\n",
      "        ...,\n",
      "        [1.0269],\n",
      "        [1.0126],\n",
      "        [0.8494]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[1.0948],\n",
      "        [1.0948],\n",
      "        [1.0948],\n",
      "        ...,\n",
      "        [0.8433],\n",
      "        [0.8433],\n",
      "        [0.8433]], device='cuda:0')\n",
      "Accuracy: 39.8106%\n",
      "tensor([[0.7379],\n",
      "        [0.7385],\n",
      "        [0.6666],\n",
      "        ...,\n",
      "        [0.7068],\n",
      "        [0.8132],\n",
      "        [0.8594]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[0.9289],\n",
      "        [0.9289],\n",
      "        [0.9289],\n",
      "        ...,\n",
      "        [0.5385],\n",
      "        [0.5385],\n",
      "        [0.5385]], device='cuda:0')\n",
      "Accuracy: 40.2084%\n",
      "tensor([[35.6363],\n",
      "        [40.4104],\n",
      "        [31.6950],\n",
      "        ...,\n",
      "        [ 0.9692],\n",
      "        [ 1.0891],\n",
      "        [ 1.0799]], device='cuda:0', grad_fn=<AddmmBackward>) tensor([[81.0337],\n",
      "        [81.0337],\n",
      "        [81.0337],\n",
      "        ...,\n",
      "        [ 0.7141],\n",
      "        [ 0.7141],\n",
      "        [ 0.7141]], device='cuda:0')\n",
      "Accuracy: 39.1416%\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "for batch in test_loader:\n",
    "#     print(batch)\n",
    "    data = batch.to(device)\n",
    "    node_pred = model(data)\n",
    "#     edge_pred = torch.sigmoid(edge_pred)\n",
    "#     print(edge_pred, data.y_edges, node_pred, data.y_params)\n",
    "    print(node_pred, data.y_params)\n",
    "#     edge_correct = ((edge_pred > 0.5) == (data.y_edges > 0.5)).sum().item()\n",
    "    correct = (((node_pred - data.y_params)/data.y_params)**2 < 0.1**2).sum().item()\n",
    "    acc = correct / (len(node_pred))*100\n",
    "#     edge_acc = edge_correct / len(edge_pred)*100\n",
    "#     print('Accuracy: {:.4f}%'.format(acc), ', edge accuracy: {:.4f}%'.format(edge_acc))\n",
    "    print('Accuracy: {:.4f}%'.format(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13.484405994415283 13.484405994415283\n"
     ]
    }
   ],
   "source": [
    "torch.cuda.empty_cache()\n",
    "torch.cuda.reset_max_memory_allocated()\n",
    "torch.cuda.reset_max_memory_cached()\n",
    "print(torch.cuda.memory_allocated(0)/1024**3, torch.cuda.max_memory_allocated(0)/1024**3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(full_graphs[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.8922e+01, 3.6313e+00, 3.0467e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([ 46.2187, -35.0857,   0.6955], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([ 1.8613e+01,  3.5876e+00, -1.0034e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([ 1.8996e+01,  2.9768e+00, -4.8119e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([25.8152, 16.9260, -1.2471], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([25.4800, 16.8262, -1.2453], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([24.8583, 16.6111, -1.2062], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([21.5880, -2.6879,  0.8381], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([16.9862,  3.8774,  0.1122], device='cuda:0', grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([ 41.1885, -32.8211,   1.0185], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([ 40.7710, -30.7252,   1.2782], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([25.7639, 18.2869, -1.1607], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([16.9630,  2.1817,  0.0223], device='cuda:0', grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([24.5119, -4.3527,  0.6603], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([ 44.5465, -35.2413,   1.0523], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([ 45.1856, -40.0252,   1.1841], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([24.6620, 16.4957, -1.1209], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([1.7948e+01, 5.2398e-01, 7.7200e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([ 45.7709, -35.0205,   1.2817], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([ 44.2098, -33.7250,   1.2481], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([22.9125, -3.6225,  0.6602], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([16.7419,  3.0845,  0.0594], device='cuda:0', grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([26.6395, 17.5330, -0.8474], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([23.7234, -3.3299,  1.1437], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([27.5753, 17.9557, -0.5238], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([21.8177, -2.6738,  1.0679], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([ 44.8839, -32.5191,   1.1000], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([21.9020, -3.5113,  1.1277], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([1.8384e+01, 6.1529e-01, 3.2547e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([25.4626, 16.8129, -1.2414], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([25.4351, 17.4593, -1.2281], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([16.8515,  3.8994,  0.0608], device='cuda:0', grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([23.3092, -1.9636,  0.5303], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([25.9646, 17.7169, -1.2468], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.9215, 18.2583, -1.0000], device='cuda:0')\n",
      "tensor([ 46.9991, -33.8223,   0.1652], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([21.1459, -2.2798,  0.9505], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([23.1696, -3.5011,  1.1510], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([21.5713, -3.3363,  1.0770], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.3713, -3.2696,  1.0000], device='cuda:0')\n",
      "tensor([ 46.2006, -35.3662,   1.1758], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 43.2096, -32.9716,   1.0000], device='cuda:0')\n",
      "tensor([1.7263e+01, 1.0393e+00, 1.6066e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.0778,  3.4214,  1.0000], device='cuda:0')\n",
      "tensor([27.7684, 17.3738,  0.4669], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([45.3506, 18.0112,  0.9953], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([50.5027, 18.5959,  1.0513], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([37.3744, -5.8386,  0.6888], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([ 46.2743, -27.2447,   0.5555], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([ 51.6387, -31.0397,   0.8468], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([24.5524, 19.4569,  0.9050], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([24.2394, 18.8239,  0.8832], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([45.0087, 16.7298,  0.2292], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([44.5156, 15.5085,  1.1685], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([ 1.7541e+01,  2.0131e+00, -4.8630e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([48.6423, 16.9927,  1.0624], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([27.6775, 16.9265,  1.0528], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([23.6196, 18.5726,  0.9916], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([1.7953e+01, 9.1990e-03, 5.6586e-01], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([28.0753, 17.4585,  1.0053], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([20.5719, 12.1217, -1.0629], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([21.2473, 11.5207, -0.8434], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([34.1981, -3.6442,  0.7675], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([35.5160, -5.8147,  0.7135], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([17.7641,  0.0203,  0.2386], device='cuda:0', grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([26.3771, 16.5180,  1.0986], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([23.0741,  8.3208, -0.6991], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([21.7606, 12.2207, -1.2206], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([ 51.5355, -31.3089,   0.8295], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([47.0115, 18.1877,  0.6074], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([38.1979, -4.7878,  1.0353], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([24.2371, 19.3842,  0.5825], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([24.3078, 19.6725,  1.0566], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([37.0417, -5.9561,  0.7059], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([ 1.7006e+01,  1.8718e+00, -3.3896e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([ 47.2504, -27.9689,   1.1573], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([20.9418, 12.1344, -1.0992], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([45.6598, 17.6850,  0.9116], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([22.2479, 12.4092, -1.2084], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([47.0971, 18.5995,  1.0467], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([ 48.1816, -27.8349,   1.0276], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([26.3400, 16.4928,  1.0442], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([27.0130, 16.4426,  1.0671], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([36.6436, -5.3281,  0.9428], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([ 49.8435, -29.2734,   0.8564], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([27.7259, 17.4203,  1.0643], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([ 4.2559e+01,  1.6479e+01, -7.1920e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([24.3839, 18.9240,  0.9832], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([1.8226e+01, 6.0504e-01, 6.6561e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([ 47.6424, -27.5069,   1.3200], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([18.5114,  0.3200,  0.0315], device='cuda:0', grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([27.6347, 17.0694,  1.0836], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([17.6981,  0.1550,  0.0968], device='cuda:0', grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([ 45.3533, -26.1198,   0.2132], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([28.8183, 17.7870,  0.8339], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([20.1620, 11.9863, -1.1321], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([48.3909, 18.0284,  0.7943], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.5379, 18.5337,  1.0000], device='cuda:0')\n",
      "tensor([37.8586, -5.2764,  0.7718], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([ 46.1128, -26.8198,   1.3595], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([36.0578, -6.1242,  0.8403], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([38.0319, -5.8003,  1.1152], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([1.7983e+01, 1.5616e-02, 4.1088e-01], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([20.5953, 12.0102, -1.1903], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([ 1.7714e+01,  8.4727e-01, -1.6490e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([27.2588, 16.8424,  1.0697], device='cuda:0', grad_fn=<SelectBackward>) tensor([28.3298, 16.5919,  1.0000], device='cuda:0')\n",
      "tensor([24.0996, 19.1311,  1.1166], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([25.2157, 19.9593,  0.5026], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([37.2529, -6.4199,  0.9335], device='cuda:0', grad_fn=<SelectBackward>) tensor([35.6597, -4.4337,  1.0000], device='cuda:0')\n",
      "tensor([21.1143, 11.9102, -0.9114], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([24.5080, 18.7854,  0.8182], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([ 1.7304e+01,  1.5815e+00, -2.6718e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([16.4344,  1.3468,  1.0000], device='cuda:0')\n",
      "tensor([ 52.6274, -29.6235,   0.7102], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.5908, -30.7045,   1.0000], device='cuda:0')\n",
      "tensor([21.7299, 12.2777, -1.1904], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1091, 12.4644, -1.0000], device='cuda:0')\n",
      "tensor([24.5432, 19.5908,  1.0037], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.6944, 19.6988,  1.0000], device='cuda:0')\n",
      "tensor([ 48.1639, -28.5145,  -1.3257], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([32.0019, 26.6397, -1.1649], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([32.1165, 12.5905,  0.0456], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([28.7304, 14.3607,  0.9624], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([2.0859e+01, 8.6323e+00, 3.1196e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([47.7461, -8.3849,  1.1690], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([25.8597, 20.8922, -0.9945], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([46.4431, -6.8762,  0.9621], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([18.2406,  5.1905,  0.0713], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([ 54.1077, -30.4465,  -1.2699], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([17.1021,  3.1857,  0.1222], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([17.2286, 13.7583, -1.1431], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([4.6069e+01, 4.6449e+00, 1.8598e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([24.2882, 11.2915,  1.0395], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([24.4959, 11.5860,  0.8428], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([43.2275,  4.4356, -0.1100], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([ 46.4589, -29.3233,  -1.2642], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([29.5395, 13.9458,  1.0556], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([1.5784e+01, 1.2392e+01, 4.4193e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([32.3649, 12.6060,  0.7098], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([ 42.0881, -26.7149,  -0.8660], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([31.0216, 26.6414, -1.1206], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([23.7804, 11.2889,  0.9542], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([ 42.3342, -26.9702,  -1.2818], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([31.6662, 26.6305, -1.1397], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([ 41.3861, -26.5129,  -1.1880], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([30.7623, 25.6517, -0.5856], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([ 46.0461, -10.6148,   1.0503], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([ 30.8343, -17.1875,  -1.1948], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([1.8842e+01, 4.8756e+00, 1.1441e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([1.7010e+01, 1.0998e+00, 6.0374e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([ 41.6420, -26.5655,  -1.1666], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([27.8856, 23.0059, -0.4784], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([ 4.6144e+01,  4.0165e+00, -1.2932e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([15.9573, 12.7325, -0.0453], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([30.1603, 25.6759, -1.1075], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([33.0890, 28.8386, -0.8282], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([44.9079, -7.6834,  1.2577], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([ 48.7111, -25.7130,  -1.4429], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([ 41.6861, -26.8061,  -1.3106], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([1.7741e+01, 3.0308e-01, 9.8991e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([17.5335,  4.3753,  0.2962], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([33.2634, 12.2586,  0.3365], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([24.2539, 11.3417,  0.9970], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([29.9662, 26.8041, -1.0218], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([49.1592, -7.7543,  1.1602], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([42.5250, -7.2680,  1.1907], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([45.2285, -9.9877,  1.1006], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([ 4.4435e+01,  6.0937e+00, -3.4538e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([24.4202, 11.0299,  1.0440], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([29.6907, 24.8335, -1.1392], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([2.8395e+01, 9.9921e+00, 1.9658e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([ 4.5632e+01,  4.1621e+00, -3.9749e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([31.0132, 26.7924, -1.1592], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([24.7528, 11.4088,  1.0851], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([46.1363, -8.4528,  1.1295], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([ 41.6044, -27.0621,  -1.0020], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([ 40.1575, -25.1728,  -1.2345], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([ 4.7806e+01,  5.0180e+00, -1.6837e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([4.1045e+01, 6.2937e+00, 1.2046e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([29.7276, 14.2327,  0.9999], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([28.7566, 14.1472,  1.1039], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([1.7636e+01, 1.1737e+01, 2.9489e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([17.8668, 15.0071, -1.1991], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([28.8045, 14.2918,  1.0584], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([3.8797e+01, 6.2271e+00, 3.3374e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([29.5629, 14.0177,  1.0133], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([ 47.2604, -29.8391,  -1.1907], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([1.6598e+01, 1.2691e+01, 2.2851e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([ 39.0897, -25.0488,  -1.1440], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([46.1941, -9.5500,  1.0726], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([4.0250e+01, 6.5744e+00, 1.9062e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([17.0370,  1.2464,  0.0194], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([23.4191, 10.2125,  0.8028], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([ 52.5991, -30.6360,  -1.1983], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.1714, -30.0839,  -1.0000], device='cuda:0')\n",
      "tensor([22.7708, 11.3404,  0.5429], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([32.0703, 26.3512, -1.1672], device='cuda:0', grad_fn=<SelectBackward>) tensor([32.1656, 27.2873, -1.0000], device='cuda:0')\n",
      "tensor([ 4.8828e+01,  6.4398e+00, -7.5582e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([16.4979,  2.3589,  0.0554], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([28.8272, 13.8015,  1.0916], device='cuda:0', grad_fn=<SelectBackward>) tensor([29.0483, 13.4259,  1.0000], device='cuda:0')\n",
      "tensor([ 37.9628, -24.4737,  -0.9050], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([ 43.8031, -28.4171,  -1.2870], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([1.7140e+01, 6.3404e-05, 6.3291e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.3100,  4.1804,  1.0000], device='cuda:0')\n",
      "tensor([23.0901, 18.6425, -1.1632], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.6615, 13.3337, -1.0000], device='cuda:0')\n",
      "tensor([22.5966, 11.1689,  0.8177], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.9298, 10.3572,  1.0000], device='cuda:0')\n",
      "tensor([49.2041, -8.4592,  1.0448], device='cuda:0', grad_fn=<SelectBackward>) tensor([48.2970, -9.6577,  1.0000], device='cuda:0')\n",
      "tensor([ 46.2717, -30.4173,  -1.0776], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([ 41.9591, -26.0762,  -0.9106], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([ 43.6489, -29.1443,  -1.1057], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.0909, -26.7089,  -1.0000], device='cuda:0')\n",
      "tensor([ 4.4414e+01,  4.0448e+00, -8.6032e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.4910,  4.0459,  1.0000], device='cuda:0')\n",
      "tensor([20.4732,  7.7521, -0.8678], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([ 3.0361e+01, -9.0684e-01,  1.4020e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([2.3867e+01, 1.5737e+00, 6.2276e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([2.8793e+01, 1.6137e-02, 1.9421e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([18.1009,  7.2010, -0.5217], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([20.4995,  6.1244, -0.3843], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([ 2.4375e+01,  5.3006e-01, -2.2653e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([ 2.9346e+01, -2.1110e+00,  6.9283e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([ 1.9825e+01,  2.6374e+00, -5.3469e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([ 2.2607e+01,  2.6042e+00, -5.4687e-06], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([2.2622e+01, 1.7777e-01, 8.2684e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([19.3176,  7.6769, -0.7170], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([2.3651e+01, 3.5793e-01, 6.9781e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([2.3212e+01, 2.8957e-01, 1.6484e-07], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([20.5678,  6.4029, -0.9347], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([20.6440,  7.0543, -0.5203], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([19.4577,  6.7390, -0.8307], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([25.7861, -2.1049, -0.9565], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([26.9044, -2.0418, -0.9875], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([32.0293, -2.8769,  0.0618], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([ 3.4346e+01, -4.0661e-01,  6.4398e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([25.9423, -1.9959, -0.4250], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([26.4303, -2.0085, -0.1088], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([ 2.1563e+01,  2.6869e+00, -2.6040e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.0031,  2.1301,  1.0000], device='cuda:0')\n",
      "tensor([26.0292, -1.0818, -0.6359], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([20.7453,  6.8437, -0.9572], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([18.3152,  6.8623, -0.6716], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([28.0052, -2.1346, -0.9193], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([18.1486,  7.7711, -0.3882], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.1449,  7.8380, -1.0000], device='cuda:0')\n",
      "tensor([26.0147, -2.3663, -0.7187], device='cuda:0', grad_fn=<SelectBackward>) tensor([27.8089, -1.0582, -1.0000], device='cuda:0')\n",
      "tensor([37.6316, 17.6384,  0.6327], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([38.4320, -5.6872,  1.0864], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([39.4602, 29.9535, -1.1194], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([37.6067, 28.3178, -0.9100], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([41.1823, -6.8072,  0.9684], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([ 30.2778, -12.4251,  -1.1322], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([37.0519, 18.6641,  1.0256], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([ 30.2364, -11.8132,  -1.0391], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([ 4.4481e+01, -6.5600e-01, -5.0757e-08], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([ 30.7749, -12.7462,  -1.1717], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([38.3575, 19.1815,  1.0025], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([41.0319, -3.1314, -0.6708], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([36.5603, 18.9082,  0.9219], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([ 29.2684, -12.0046,  -0.7118], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([40.8006, -8.1951,  0.8819], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([ 4.8053e+01, -2.9964e+00,  2.2435e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([23.6557,  5.3420,  0.4148], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([20.6691,  5.0161,  0.4992], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([27.3522,  2.0463,  0.0655], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([39.3495, 30.9496, -1.0776], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([ 28.3781, -12.0150,  -1.0010], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([37.1208, 18.8004,  1.1111], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([ 2.9178e+01, -1.0264e+01, -3.6540e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([24.6353,  4.4781,  0.1336], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([39.0446, 18.4038,  1.1690], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([ 29.7751, -11.9870,  -1.0337], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([39.1089, -7.1186,  1.3860], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([39.6200, 29.3303, -0.4647], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([39.4916, 30.8150, -1.0854], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([39.6167, -8.1484,  1.0453], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([43.2562, -3.6169, -0.9345], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([40.7500, -7.5464,  0.9650], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([41.2075, -1.7224, -0.2210], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([41.8914, -6.7456,  1.1842], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([22.6407,  4.9817,  0.0567], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([41.0046, -2.9481, -0.6774], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([37.8384, 29.9754, -1.1338], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([41.1329, 19.6691,  0.1152], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([41.6940, -3.3429,  0.1971], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([ 4.5591e+01, -2.7471e+00,  3.4420e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([40.9958, -6.8056,  0.9397], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([40.8906, -7.7229,  0.9011], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([39.2874, 31.5213, -1.0686], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([ 30.1761, -11.8717,  -1.0767], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([25.5830,  4.6412,  0.3656], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([ 30.0866, -11.9596,  -1.0088], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([24.0881,  5.5311,  0.3139], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([43.6374, -3.3421, -0.7984], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([38.3118, 17.2446,  0.8301], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([39.1915, 29.7655, -1.1314], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([36.4061, 18.3137,  1.0203], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([39.7809, -8.1087,  1.0083], device='cuda:0', grad_fn=<SelectBackward>) tensor([38.5897, -6.2761,  1.0000], device='cuda:0')\n",
      "tensor([ 2.1692e+01,  3.9961e+00, -1.3708e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([38.3595, 19.1461,  1.0342], device='cuda:0', grad_fn=<SelectBackward>) tensor([37.1612, 18.2663,  1.0000], device='cuda:0')\n",
      "tensor([37.6707, 29.5908, -1.0714], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([40.0692, 32.2023, -1.0258], device='cuda:0', grad_fn=<SelectBackward>) tensor([41.6190, 31.7445, -1.0000], device='cuda:0')\n",
      "tensor([2.2357e+01, 4.9793e+00, 2.5894e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([ 30.3413, -12.3630,  -1.1304], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 30.7383, -12.9980,  -1.0000], device='cuda:0')\n",
      "tensor([22.5654,  6.4018,  0.1331], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.6845,  5.2091,  1.0000], device='cuda:0')\n",
      "tensor([ 4.8627e+01, -1.5483e+00,  1.0832e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.2524, -2.5645, -1.0000], device='cuda:0')\n",
      "tensor([25.0635, 20.6621,  0.9118], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([ 32.3412, -21.4295,  -0.4783], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([24.8563, 20.7228,  0.9519], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([ 30.5998, -21.6088,  -0.9220], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([ 38.3277, -27.7195,  -1.0626], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([24.3583, 21.3949,  1.1643], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([24.7492, 20.6866,  1.0289], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([ 36.1638, -25.0830,  -1.0250], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([25.0523, 21.1409,  0.4789], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([23.5665, 20.8937,  0.5312], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([ 32.5937, -21.2133,  -1.0758], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([23.3976, 20.0892,  0.9583], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([24.1440, 20.5512,  1.0356], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([ 33.6129, -23.0956,  -0.9738], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([ 30.0960, -15.5063,  -1.1366], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([ 38.0785, -29.6126,  -1.0845], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([24.8373, 21.4133,  1.1694], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([23.4217, 19.6150,  0.9108], device='cuda:0', grad_fn=<SelectBackward>) tensor([24.3244, 21.0145,  1.0000], device='cuda:0')\n",
      "tensor([ 30.4280, -20.8386,  -0.9225], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([ 32.3170, -22.2782,  -1.0892], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 29.8802, -19.5947,  -1.0000], device='cuda:0')\n",
      "tensor([27.2292,  7.9937, -0.8391], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([ 31.8698, -18.5931,   1.4277], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([ 48.1254, -30.4452,  -0.8618], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 32.5653, -19.2922,  -1.1136], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 33.5700, -20.4197,   1.2511], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 39.1715, -22.6338,   1.1657], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 33.3085, -20.2111,   1.0551], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 42.2549, -26.4990,  -1.3056], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([24.7784,  8.8442, -0.4713], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([27.0000,  8.2584, -0.2980], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([ 44.0308, -23.1981,   0.7422], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([ 42.6640, -24.9193,   1.0270], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([ 48.1079, -32.5155,  -0.8624], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([ 43.0763, -25.1173,  -1.0208], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 32.7897, -18.1567,  -1.2757], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 32.2616, -19.1610,   0.9457], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 50.7671, -27.0119,  -1.0360], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 31.1080, -17.7125,   0.6052], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([ 39.6060, -23.2370,  -1.2991], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([26.9951,  8.1046, -0.8100], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([ 29.6949, -16.4559,   1.0694], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([ 36.1768, -20.4686,   0.6178], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 45.3915, -21.4775,   0.9351], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 29.9549, -18.3827,   1.1334], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([ 36.7118, -21.7746,   0.8704], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 51.9191, -35.4677,  -1.1198], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([ 46.7065, -30.8349,  -1.2729], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([ 45.9194, -26.0154,  -1.3024], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([ 27.8645, -16.0591,   1.1560], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([25.9221,  9.7349, -0.3596], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([ 40.1617, -22.3173,  -1.3455], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 33.0416, -20.6618,   1.0726], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([25.0358,  7.8745, -0.6312], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([ 35.2067, -17.9522,   1.1347], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([26.5732,  8.0706, -0.7725], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([ 35.0165, -15.6279,   0.7247], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 48.9671, -30.7986,  -1.3131], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([ 41.9918, -24.1578,  -1.2832], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 46.2006, -30.1672,  -0.8878], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 43.5084, -25.8403,  -1.3615], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 48.5722, -31.7141,  -1.0249], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([25.8978,  7.6763, -0.7499], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([24.7793,  7.4680, -0.6518], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([24.9587,  6.3740, -0.5758], device='cuda:0', grad_fn=<SelectBackward>) tensor([26.9997,  8.4866, -1.0000], device='cuda:0')\n",
      "tensor([ 48.6529, -26.0149,  -0.9965], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([ 46.3356, -30.3369,  -0.7518], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.1559, -27.4095,  -1.0000], device='cuda:0')\n",
      "tensor([ 50.1795, -34.1627,  -1.0129], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 49.4276, -31.1958,  -1.0000], device='cuda:0')\n",
      "tensor([ 29.1484, -15.9217,   1.3702], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([ 44.0147, -25.0389,   1.0110], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 42.3723, -23.8952,   1.0000], device='cuda:0')\n",
      "tensor([ 38.0040, -21.0520,   0.8675], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 32.0675, -16.8338,   1.0000], device='cuda:0')\n",
      "tensor([19.1694, -8.4264,  0.9726], device='cuda:0', grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([19.3052, 14.7226, -1.2652], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([19.9093, 14.4648, -1.2532], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([43.7420, -9.4681, -0.9711], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([24.6558, -1.0284, -0.7798], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([40.4085, 23.6050, -1.1083], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([ 45.7367, -11.8549,   1.2197], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([15.9912,  1.8245,  0.0539], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([20.4814, 14.6420, -1.1478], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([ 1.7310e+01,  1.3532e+01, -2.5872e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([ 45.8967, -11.2985,   1.1787], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([19.1737,  6.5416,  0.4160], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([17.7633,  4.8228,  0.3674], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([ 2.0003e+01,  1.8229e-02, -5.8753e-01], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([45.6128, -9.8662, -1.0718], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([39.9717, -9.6078, -0.5851], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([ 28.3255, -17.3660,  -0.9996], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([ 50.5374, -10.9733,   1.1294], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([2.2874e+01, 3.3643e-04, 2.0508e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([1.7383e+01, 5.4251e-01, 2.2851e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([41.7526, -9.6131,  1.1053], device='cuda:0', grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([38.8679, 22.0989, -1.0209], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([42.2459, 24.2224, -1.1425], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([23.0029, -0.4130, -0.5072], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([37.1308, 20.9270, -0.6662], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([24.6513, -1.0274, -0.7799], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([36.7437, 21.7721, -0.5055], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([38.8172, 22.1275, -1.0668], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([32.5541, -7.1424, -0.4088], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([ 21.7859, -10.9481,   0.9601], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 27.7086, -17.2293,  -1.0346], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([ 32.4475, -15.1250,   1.1521], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 29.3695, -17.8955,  -1.0002], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([17.9851, -7.8031,  1.0950], device='cuda:0', grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 4.4604e+01, -9.2402e+00, -2.2697e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([ 34.6389, -22.0240,   1.0612], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([2.1917e+01, 3.7318e-04, 1.0870e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([45.7130, -9.6971, -1.1591], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([ 48.5334, -11.1791,   1.1318], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([20.6993, 13.5720, -0.8054], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([ 2.2089e+01, -3.6863e-02, -1.7284e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([ 27.2232, -16.6841,  -1.0463], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([ 28.3570, -17.2013,  -1.0628], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([18.6096, -8.1778,  1.0952], device='cuda:0', grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 41.1040, -10.5154,  -0.4078], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([2.2622e+01, 4.7181e-05, 9.8465e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([ 46.9590, -10.0802,   1.0000], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([ 31.0548, -19.3935,  -0.9932], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([ 32.7773, -15.3948,   0.4964], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 33.4972, -22.4312,  -1.0192], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([ 46.2448, -12.3461,   1.2108], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([17.7250,  3.3450,  0.0559], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([ 45.7980, -10.8379,  -1.0088], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([ 47.2279, -11.1517,   1.1422], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([18.1354,  0.1791, -0.6717], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([ 28.9363, -17.8795,  -1.1328], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([21.1007, -9.8037,  1.0609], device='cuda:0', grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 2.0315e+01,  1.5277e+01, -5.1279e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([22.0932, -0.3574, -0.8531], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([17.2614, 13.4807, -0.1066], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([45.1314, -9.8459,  1.3050], device='cuda:0', grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([39.3175, 22.6438, -1.1115], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([45.1563, -9.9030, -1.3124], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([23.0284, -0.3491, -0.1983], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.4631,  0.6105, -1.0000], device='cuda:0')\n",
      "tensor([17.9584, 13.7529, -0.9381], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([40.2954, 23.0237, -1.1225], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([39.6436, 22.8016, -1.1094], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([16.3021,  4.4033,  0.4693], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([49.8576, -9.7559,  1.2227], device='cuda:0', grad_fn=<SelectBackward>) tensor([ 48.9321, -11.7934,   1.0000], device='cuda:0')\n",
      "tensor([45.7060, -9.8630, -1.2551], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([ 1.8570e+01,  1.4343e+01, -5.7742e-08], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([36.4904, 20.6650, -0.9052], device='cuda:0', grad_fn=<SelectBackward>) tensor([42.2122, 23.1191, -1.0000], device='cuda:0')\n",
      "tensor([ 43.3137, -10.5829,   0.0456], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.5450, -9.8693, -1.0000], device='cuda:0')\n",
      "tensor([16.3764,  2.3267,  0.0347], device='cuda:0', grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([ 26.4741, -15.8986,  -0.2672], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([ 30.3962, -15.5043,   0.7691], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 1.9314e+01, -3.2709e-01,  8.9996e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([15.7065,  5.4932,  1.0000], device='cuda:0')\n",
      "tensor([20.2633, 14.9293, -1.2018], device='cuda:0', grad_fn=<SelectBackward>) tensor([19.3354, 15.3564, -1.0000], device='cuda:0')\n",
      "tensor([19.7943, -8.5712,  1.0701], device='cuda:0', grad_fn=<SelectBackward>) tensor([18.5312, -8.2375,  1.0000], device='cuda:0')\n",
      "tensor([ 27.5721, -16.4542,  -1.0861], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 26.6597, -15.5906,  -1.0000], device='cuda:0')\n",
      "tensor([ 36.8004, -14.0253,  -0.1525], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 37.6987, -12.7500,  -0.9818], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 36.8323, -13.1476,  -0.8427], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 37.0691, -13.0756,  -1.2426], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 37.7271, -12.7907,  -1.1204], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 4.7225e+01,  3.1429e+00, -5.7094e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([4.8059e+01, 4.4811e-01, 1.0074e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([5.0588e+01, 5.3723e-01, 1.1402e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([4.5107e+01, 1.1715e+00, 3.5800e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([ 4.4905e+01,  2.2869e+00, -3.3138e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([ 35.8117, -13.7389,  -0.5532], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 4.6455e+01,  3.2318e+00, -2.4588e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([ 37.7274, -12.8226,  -1.1574], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 37.3657, -12.9295,  -1.1164], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([42.8013,  3.1678, -0.0764], device='cuda:0', grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([ 37.8884, -13.3035,  -1.2201], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 4.4989e+01,  3.4468e+00, -2.9004e-02], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([ 36.7744, -12.8364,  -1.1663], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 38.0250, -13.6089,  -1.0000], device='cuda:0')\n",
      "tensor([ 4.5628e+01,  3.1252e+00, -8.4812e-03], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([ 4.8861e+01,  1.6628e+00, -6.3367e-04], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([47.3939,  0.8490,  1.0000], device='cuda:0')\n",
      "tensor([21.1452, 14.2958,  1.0402], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([ 34.3848, -21.0683,   0.4999], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([ 36.1567, -22.2870,   0.9020], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([20.7149, -8.4735, -0.9143], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([22.1994, 14.6790,  0.9295], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([ 31.6820, -19.2839,   1.0733], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([43.4429, -9.6276, -0.5950], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([ 45.2725, -13.6588,   1.2365], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([ 47.2706, -11.9890,   1.1404], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([37.7684,  9.9255, -0.3758], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([21.2967, -9.2416, -0.7403], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([46.9732, -9.0880, -1.1894], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([ 41.1978, -11.2936,  -1.3125], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([45.6473, -8.7864, -1.0961], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([ 33.3934, -13.4733,  -0.0717], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([ 46.2767, -13.3457,   1.3039], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([ 35.7915, -23.1989,   1.4267], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([24.5251, -8.3163, -1.0842], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([19.2190, -8.1142, -0.0356], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([23.9683, -9.7375, -1.1503], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([21.8635, -9.6428, -1.1142], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([ 47.6516, -11.7902,   1.0446], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([28.3642, -9.4880, -1.0699], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([45.1132, -8.6117, -0.9859], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([34.4113,  8.6637, -0.7571], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([ 4.4134e+01, -8.3850e+00,  1.2796e-05], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([22.0105, 14.7682,  0.4353], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([ 42.1805, -14.1110,  -1.3199], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([ 33.1469, -12.6442,  -0.5486], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([22.9424, 13.8233,  0.7787], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([ 39.9691, -30.4766,   1.3957], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([ 46.1071, -32.6982,   0.8277], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([ 36.9858, -22.7988,   0.9687], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([ 47.4768, -12.7926,   0.9172], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([43.5278, -8.5838, -0.9152], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([ 37.6509, -23.5464,   1.0267], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([36.0453,  9.8700, -0.2599], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([ 37.4979, -26.4122,   1.0523], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([22.5580, 15.2765,  0.9946], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([ 39.3663, -27.1656,   1.0368], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([20.3253, -8.3980, -1.0026], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([34.1808, -8.9038, -1.1762], device='cuda:0', grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([35.4480,  9.0929, -0.6705], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([36.7542,  9.2998, -0.5362], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([ 35.5665, -14.0888,  -0.1037], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([ 43.2548, -30.9784,   0.8108], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([ 43.7007, -11.5399,   1.3480], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([ 40.7450, -11.0795,   0.8547], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([ 36.3603, -22.9301,   0.9941], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([25.3835, -8.7454, -1.0970], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([ 43.7655, -32.5143,   1.4511], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([21.9805, 14.6549,  0.9896], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([ 42.7131, -14.0731,  -1.2423], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([36.3246,  9.1050, -0.6384], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([22.2379, 15.5507,  0.3626], device='cuda:0', grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([41.2401, -9.1022,  0.0708], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([ 47.5700, -32.0633,   0.7148], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([ 44.4812, -29.9610,   0.1999], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([21.5474, -9.3747, -0.6336], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([22.3830, -9.6972, -1.0111], device='cuda:0', grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([44.0308, -9.5210, -0.1934], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([ 24.7156, -11.5502,  -1.0125], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([ 46.2647, -12.7126,   1.1559], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([21.9339, 13.5515,  0.6462], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([22.3797, 15.6557,  0.4092], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([44.3779, -9.2008, -0.8399], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([39.1298,  8.9210, -0.5924], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([39.6513,  9.4132, -0.8297], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([ 22.2594, -10.0362,  -1.0848], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([21.1108, -8.7940, -1.0000], device='cuda:0')\n",
      "tensor([ 39.1314, -27.1075,   1.1799], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([ 48.3564, -12.1008,   1.0983], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([28.6191, -9.6940, -0.8227], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([ 43.2633, -15.0926,  -1.2950], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([24.7170, -8.7339, -1.1214], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([29.0598,  3.9818,  0.2646], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([21.7672, 14.6257,  0.9070], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([34.6777,  9.0611, -0.7263], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([25.9458, -8.9930, -1.1087], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([36.9219,  8.8780, -0.7726], device='cuda:0', grad_fn=<SelectBackward>) tensor([36.8643,  8.5166, -1.0000], device='cuda:0')\n",
      "tensor([ 36.9911, -23.7474,   1.3183], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([34.7247, -9.9405, -0.9778], device='cuda:0', grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n",
      "tensor([ 34.3366, -22.1937,   0.8130], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([21.8759, 14.8030,  1.0215], device='cuda:0', grad_fn=<SelectBackward>) tensor([22.2387, 15.0933,  1.0000], device='cuda:0')\n",
      "tensor([46.1893, -9.2882, -1.2502], device='cuda:0', grad_fn=<SelectBackward>) tensor([45.9062, -8.7200, -1.0000], device='cuda:0')\n",
      "tensor([ 46.0888, -12.8163,   1.2269], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 45.2991, -12.1745,   1.0000], device='cuda:0')\n",
      "tensor([ 30.9451, -11.3676,  -0.3238], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([ 46.2089, -33.2701,   1.1292], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 46.1681, -33.3355,   1.0000], device='cuda:0')\n",
      "tensor([26.8774, -9.5045, -1.0295], device='cuda:0', grad_fn=<SelectBackward>) tensor([23.7561, -8.1230, -1.0000], device='cuda:0')\n",
      "tensor([ 37.6442, -23.3092,   0.8890], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 37.1662, -23.3223,   1.0000], device='cuda:0')\n",
      "tensor([ 36.6121, -12.8179,  -0.8288], device='cuda:0',\n",
      "       grad_fn=<SelectBackward>) tensor([ 44.6086, -15.7559,  -1.0000], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "for pred, dat in zip(node_pred, data.y_nodes):\n",
    "    print(pred, dat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AGNNConv',\n",
       " 'APPNP',\n",
       " 'ARGA',\n",
       " 'ARGVA',\n",
       " 'ARMAConv',\n",
       " 'ChebConv',\n",
       " 'DNAConv',\n",
       " 'DataParallel',\n",
       " 'DeepGraphInfomax',\n",
       " 'DenseGCNConv',\n",
       " 'DenseGINConv',\n",
       " 'DenseSAGEConv',\n",
       " 'DynamicEdgeConv',\n",
       " 'ECConv',\n",
       " 'EdgeConv',\n",
       " 'FeaStConv',\n",
       " 'GAE',\n",
       " 'GATConv',\n",
       " 'GCNConv',\n",
       " 'GINConv',\n",
       " 'GMMConv',\n",
       " 'GatedGraphConv',\n",
       " 'GlobalAttention',\n",
       " 'GraphConv',\n",
       " 'HypergraphConv',\n",
       " 'InnerProductDecoder',\n",
       " 'JumpingKnowledge',\n",
       " 'MessagePassing',\n",
       " 'MetaLayer',\n",
       " 'NNConv',\n",
       " 'PPFConv',\n",
       " 'PointConv',\n",
       " 'RENet',\n",
       " 'RGCNConv',\n",
       " 'Reshape',\n",
       " 'SAGEConv',\n",
       " 'SAGPooling',\n",
       " 'SGConv',\n",
       " 'Set2Set',\n",
       " 'SignedConv',\n",
       " 'SignedGCN',\n",
       " 'SplineConv',\n",
       " 'TopKPooling',\n",
       " 'VGAE',\n",
       " 'XConv',\n",
       " '__all__',\n",
       " '__builtins__',\n",
       " '__cached__',\n",
       " '__doc__',\n",
       " '__file__',\n",
       " '__loader__',\n",
       " '__name__',\n",
       " '__package__',\n",
       " '__path__',\n",
       " '__spec__',\n",
       " 'avg_pool',\n",
       " 'avg_pool_x',\n",
       " 'conv',\n",
       " 'data_parallel',\n",
       " 'dense',\n",
       " 'dense_diff_pool',\n",
       " 'fps',\n",
       " 'glob',\n",
       " 'global_add_pool',\n",
       " 'global_max_pool',\n",
       " 'global_mean_pool',\n",
       " 'global_sort_pool',\n",
       " 'graclus',\n",
       " 'inits',\n",
       " 'knn',\n",
       " 'knn_graph',\n",
       " 'knn_interpolate',\n",
       " 'max_pool',\n",
       " 'max_pool_x',\n",
       " 'meta',\n",
       " 'models',\n",
       " 'nearest',\n",
       " 'pool',\n",
       " 'radius',\n",
       " 'radius_graph',\n",
       " 'reshape',\n",
       " 'unpool',\n",
       " 'voxel_grid']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(tnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "COMET INFO: ----------------------------\n",
      "COMET INFO: Comet.ml Experiment Summary:\n",
      "COMET INFO:   Data:\n",
      "COMET INFO:     url: [OfflineExperiment will get URL after upload]\n",
      "COMET INFO:   Metrics [count] (min, max):\n",
      "COMET INFO:     sys.gpu.0.free_memory    : (14839906304.0, 14839906304.0)\n",
      "COMET INFO:     sys.gpu.0.gpu_utilization: (0.0, 0.0)\n",
      "COMET INFO:     sys.gpu.0.total_memory   : (16945512448.0, 16945512448.0)\n",
      "COMET INFO:     sys.gpu.0.used_memory    : (2105606144.0, 2105606144.0)\n",
      "COMET INFO:   Other [count]:\n",
      "COMET INFO:     offline_experiment: True\n",
      "COMET INFO:   Uploads:\n",
      "COMET INFO:     git-patch: 1\n",
      "COMET INFO: ----------------------------\n",
      "COMET INFO: Uploading stats to Comet before program termination (may take several seconds)\n",
      "COMET INFO: Starting saving the offline archive\n",
      "COMET INFO: To upload this offline experiment, run:\n",
      "    comet upload comet/8530386ae84a48c9865e12c4e31519fe.zip\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/common/software/pytorch/v1.2.0-gpu/bin/python\n"
     ]
    }
   ],
   "source": [
    "!which python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch-v1.2.0-gpu [conda env:root] *",
   "language": "python",
   "name": "conda-root-pytorch-v1.2.0-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc-showmarkdowntxt": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}