rnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用于识别不定长数字串\n",
    "\n",
    "# 实现工具：\n",
    "* tensorflow 1.9.0 \n",
    "* CUDA 9.0\n",
    "\n",
    "# 网络结构\n",
    "* CNN + LSTM + CTC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/luo/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import time, os, json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import tensorflow as tf\n",
    "from PIL import Image\n",
    "import math\n",
    "import timeit\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "def get_session():\n",
    "    \"\"\"Create a session that dynamically allocates memory.\"\"\"\n",
    "    # See: https://www.tensorflow.org/tutorials/using_gpu#allowing_gpu_memory_growth\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allow_growth = True\n",
    "    session = tf.Session(config=config)\n",
    "    return session\n",
    "\n",
    "# for auto-reloading external modules\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "test_mode = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 载入CVL数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def loda_cvl_string(path):\n",
    "\n",
    "    label = []\n",
    "    begin = True\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for file in files: \n",
    "#             print(\"file: %s \" % file) #当前路径下所有非目录子文件 \n",
    "#             img = Image.open('/home/luo/文档/dataset/handwritting digital string/cvl-strings/train/' + file).resize((200,50), Image.ANTIALIAS)\n",
    "            img = Image.open(path + file).resize((200,50), Image.ANTIALIAS)\n",
    "            if begin:\n",
    "                date_train = np.array(img)\n",
    "                begin = False\n",
    "            else:\n",
    "                date_train = np.concatenate([date_train, img])\n",
    "            string = []\n",
    "            for s in file:\n",
    "                if s == '-':\n",
    "                    string = [int(i) for i in string]\n",
    "                    label.append(string)\n",
    "                    break\n",
    "                string += s\n",
    "\n",
    "    data_label = np.array(label)\n",
    "    num = data_label.shape[0]\n",
    "\n",
    "    date_train = date_train.reshape(num, 50, 200, 3)\n",
    "    for i in range(num):\n",
    "        data_label_temp = np.array(label[i])\n",
    "        data_label[i] = data_label_temp\n",
    "\n",
    "    num_training = 1088\n",
    "    num_validation = 160\n",
    "    # Subsample the data\n",
    "    mask = range(num_training, num_training + num_validation)\n",
    "    X_val = date_train[mask]\n",
    "    y_val = data_label[mask]\n",
    "    mask = range(num_training)\n",
    "    X_train = date_train[mask]\n",
    "    y_train = data_label[mask]\n",
    "\n",
    "    print('Train data shape: ', X_train.shape)\n",
    "    print('Train labels shape: ', y_train.shape)\n",
    "    print('Validation data shape: ', X_val.shape)\n",
    "    print('Validation labels shape: ', y_val.shape)\n",
    "    \n",
    "    return X_train, y_train, X_val, y_val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试载入数据，若图与label不一致，则出错\n",
    "* label:0 1 2 3 4 5 6 7 8 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape:  (1088, 50, 200, 3)\n",
      "Train labels shape:  (1088,)\n",
      "Validation data shape:  (160, 50, 200, 3)\n",
      "Validation labels shape:  (160,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB2CAYAAADY3GjsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXlsnEd24H/F7uZ9kyJFkaJE3ZKtW5asw7d8e+yJPZ7DyWRmM8FggQ2QQbLYTDbAIn/sAsku9sACwQROMjuezEzs8SlfsiXLsnVYlnWflESRoiTe9012k921f7z62B+bh5pSi01R9QOkbn79Ha+r63j13qtXSmuNxWKxWO58EuItgMVisVhig+3QLRaLZYZgO3SLxWKZIdgO3WKxWGYItkO3WCyWGYLt0C0Wi2WGYDt0i8VimSHcUoeulHpKKXVRKXVZKfXzWAllsVgslsmjbnZhkVLKA1wCHgdqgCPAD7TW52MnnsVisViixXsL124ELmutqwCUUq8DLwDjduhKKbssdZqggESfvE9NUgAkJcmEze/XdPWFAAiG4iGdZbqjXO9to54SWrTWs2500q106MXAddffNcCmyJOUUj8FfnoLz5nRqDFaxlQ0EK8H5hfKw9csTAKgbF4KAFVX/Xx2tA+Att4pEMZyx+H1yKtSEBiKryx3CVejOelWOnQ1xrFRfZHW+lXgVbAaOoQ7cK2VeYWxuvDwebF5pnOf2bny+sS2VFYszQRgMEFU9czMdACWrAkSTLoGwI7PBgAYCoV/8Lv+R7yLmZMnr/PnSG1o74KKGqkRQ8F4SWVxuJUOvQaY6/q7BKi7NXFuEyr8Eu9cZM7zlekWfR7wGNOHNuaNoWBsGod7UMhOk/ff+wPp0detLuLYmU4AvviqBYBgsA2AvAIfffIRSYlyk6EBbTvyu4xIpSIzHZ5+PFX+GJSDn37ZT8ia5aYNtxLlcgRYrJQqU0olAt8H3o+NWBaLxWKZLDetoWuth5RSfwZ8CniAX2qtz8VMshgwbCKIwjbttmXHSouPNFF4E6CkRI5u2iia8uIFGaR4RMVpb+0HoKa2m3OVfgDOXZKrQ6HozTCR5yUnwZOPiDll7pwMAHbtauLT/aKRt3RGTAcuuKW33K041cypCWtX+li+PAeArw50AVDTMvVyTYq7zHt7KyYXtNYfAx/HSBaLxWKx3AK31KFPd5wBOTVLXnPyPDReF23Usft5TAkM+m/hQYoxR3/nkMcYtlbeo3hkszgiH3t0HQAFRbk0VYsDu61ZtJ65hYncd79c8+n+dgA++6wvKlule6bhS5TXp55MZ9Ma8WYdO9IMwLt7+ugLjL5mWPZ4OxsscSNyhpdu/C+rV6Qz2CPvL1X1j3t+PBmuy8NBBxpHTR+u5kpPC1lvBzO2Q8/L97B8tZgXHtwunee8AsWeT48C0GvirLOz5We+UDlA+dlBAPq6J/lrh+vMcGx3IACJJrRr42YJB9y8PpuhOokD/OKdCgBCCV7OV9TLiUpkml3oYdtjJQA8tkluePxINS3t5rQbNCDn+LoNyQBsW5vDxWNy8a79Eo7YF4AEc5/QDK3cltjgrFMoyEuhpU468uqa6ROr6G4P4TYhb5QPdFDeO0EHM9n0YnO5WCwWywxhxmjokVrr4hL48Yv3ALD1ka0AdFRegpX5APT6RcNIS5MxbcMaxcEF4iT8+L0OAPp6R9/XHdPtkJaRQGmxnOjvEjWgfwhWrpEQr20rxdyRFILffS1z1so6Ma+ENPRFmFK2rvex1dhX/O0i52DAZdcZQ8Nwy1lQJN/p6YcK5NpWP599Lc9r7jHTT6WtZm6JimSZYNLe0M2xU2Kb7DEmyrHaw1QxaqbqgbwimRbPXyBCFxZ7GAzKsX5jJSo/0UXrNJphxBKroVssFssM4Y7R0Mdy3Lk11eEQPTEbU5jvIzUoF13eJ9GUn3zwNY0N4hRMz5CxbMFy8fisWF3KQ0tlJD+eI9psVW9YdR6hDZg/CufI+QtLwBcQAaqb5XXz5gyeeXYpAPWXZL3VF/ubOFcr9xxybq3CzprcQnldvSmHNJ/c+9w5WeHT2auHT5xII0pKVTyxXXwHxcaZ9d6udqpanO+hb3gPy92NUzd8xh+0eq2khujoH+JkZSBOUo3EPTMonCvd2H1b8ll9r4RVLswTmbPSNF6vfB5Scmz/6iZe+6crALQ2Bkfd705m2nfo0XjQVQLMMz/qmpUS2pHn8/DGv0lHXnlFHKHlDQM4Edfp5pv/QabU2ntWK1prZJl7a2vIPFuNivbIK/SxoFSeUZgrdwv2BjHjBH3iVyXQpSlIkYe0JMn073jF0HBH7kS+hDQkGFkef1J69FWLczh3Qm64/0RP+HuaV7dEkeWzZJGPNYslrOfI19KL7z0dcJz+NrrcMiHuji03S2rLivliOmyoGaB3IF6SjURrKCkT7e17LxUDsH5pJv0t3QA0XZAggIrOIAkBaXT3rpX29cT6OVw6IYrShx81T6ncYMrY+cPVmNWoN5MfZKzJxWKxWGYI015Dd0aorDwZe3LzfCgzDrXVi5djxbJEli+S6VRzjajIu88NcL1tpLdRKTU8+A0myo3z5kpceEKi4tjFVgA6h0NsNQlmyFu9Vs7bfH8GQy3iNG2ok2e1dChqWkRbbxFrDf0a/Fo+b2oWD1Jrt+t7ub7f/ZvERPLYfaJBtNe18/pO0a7rOhzZR4/W7mP5+WKiuX9NOl1t8gU+PWxCFAenV6yw5c4gP0cqf1+nqOVnyweGZ5hTXZ8inzd7dgKvvCgBDmvmyyz77MErfLxPNO+mVjmxsw9CZlr+3V4xF71SnEZJfuLUCM7oWfG44caj3kweq6FbLBbLDGFaaujOaKyUYt16s7Jym4Tg5Wf6aGmRxTktbaKBJnv7OXxEtNJjFyQcSbtulGCGPE14hdh9a+S+962WhJFXq1s5cqZvhBxFsxPZ/qhoAevulfPbapv48oI8v9Jkg2/thgFjO3cW6yyenznsoL1SL3bwoRAoc0LIxAwuX5LMy0/LIqIURIN44/NGLjeOLIuxRnWtId0kv3vxW5IbZu2yDN55r0a+U3P4HlYzt0yW4hLx/WTli0+mpbthVH6XqcKpv0nGUfv0AzksmS2z8k92VgOwc98Azd1jXGykvVQjqnp3r8YzZpRF7JFg45Her5SUBJYsyQZg/jzJsZSUCDU1InxNnczo6+r6GRqaXMOdVh16ZOe1dUs2P/7uQgCKs6VyNTe0kZQvJ/p8MsH4YGcfpy46c8Hwi+PQ1K77Fs4S08RjD8wBwGMK+c33a2iQMHQWL5TwkGcey2XuLPn88mlZzbn3UCeXa+W8sRL752aLTIvLMujvlh/mek1vWCbTkS9ZJM/4wbfnUJgtP8Pu/dIR7zseGDuqJ6Kc0LBmpZTLijKpGJcutvP1yekRiWC5c3C3vVSjiKxaLHU0PUPME/3++GgFyhUJtna59OiL53nZd1C0nrd2i0moLzBGNJwrZ3ZGhlybmOQjNVW+k2NSjXkKYFd5pqWaNr9QtK9N67PYsEYUxaF+UfZ8vhADIVmv0t0twnx9op1de8SG29cfXT5ta3KxWCyWGcK00dDdZoFFZaJ1fv9bc1laJlOTpkbRchvb2qlqkFFr115j+rimR2n3rnDxYbIyFS89K+aNxfNE+/jygMSjllcN8L3nRWt/6AHZuq+9tZXX3qg1n5tdWVwjuXLNBpxVlznZMgMoyPfS3mGcoW1hM9As47z8zrfEAVpakshHn4oM7+6W0TowjhMz8lhRoZeHNompZcgvzzrwTQ/d/WOfb7FEw4IS6RZm58tr+WWZujZ3hCv/VNQpd/rrwhz5655F0n5OXujgg11S5/sD4fNHyeX+22Tg6+3rI8TQ8L1jKnNEmyvMTWD7Nulr1qyRttrV7eeTnZUAXKwUM296WgK5WVLeD9wvsfTPPJJHTY005qMnrYZusVgsdxXTRkPXhBfbPLhBnDClRcn0mxyvfX0yklXU9LDjU9FkG13J9ScaaYezDy7PYNtGca6euSCa9+93iOdw6ZJMXviWLFC4VNkkn73dwPnLI2+s3ClVxvAQpadKkSYlJXKxWmYQtY1yYnZGAo9uktG3rETsaQe+quNt8336XelsJ1xIZV43r8ui1IRd7v1CUvCevzxoVw9ZJk14a0RYMEdszb4ksTNXXzUpp4NTO+tzP6JsjnQOAWPs/vLQ4HB7uVHWUKc5ZBq7eWJSIg0tYncP52gavYhwsrhnCLNyZSbx/PZM1qyStn6uQiwL7+3spK4p0mgfAhMUMRgS5+gr389lrnH8HiW6FV1WQ7dYLJYZQtw1dLedrLhAxpflC8TmlJySSnePaK+fH6oG4MNdXTSbaJSJtAW3lpuXLic+sDGP2iZZqfP7D0ULbzWbIZcU+jh9XiJZ/vnXEm3S0joyomS8Z7mPzZ0lRepJ8FJtUgkkyyDLy88VsXSRRKN8c0oWMb31UfsIzTya77NknmhQD2/OpcfYzvcfFVtb74C1nVuiJ7KueL1QUiw+LP+gaJG1TVMbNRWZ4iIjCTLNJjXHy8X23dgartxjaeZjRe2UFIuGrnxDNDRF7mhz843F/awiRzN/QgTOylZ8ul9MCQe/MT619tHXKhX+Hv39YVmU1zMpWeLeobuLcalJeblgkZhFVFIC5WckNGm3WQHW3DbxFMs9QGSbGO3nHpTCzcjz8u4uCR6vrJKKkWBKtCvQz/5DMni0tJp73cj04fohnY0tli+RwcjnCVGSLRXohYfFGbJmTTZnKuXm73ws36uje7RDdyy0hiTz2z64UZ6RkaPYsUsGJsesY2POLbdCZqpidqE0nMYGMXPW1E9th+5UX6edr1joIaBF2btweXDS9yswOWmy0uS1vKKFy1f6JrpkUrjb2/bN0tesWC6rv9/8uJavjkhfE3JFVusxrnUOppqGnpaeykBgcvZTa3KxWCyWGULcNXT3BGvJQtFk55WJc7KqoYUPPhWnZZ2zS9t4GmiElpvshZcel/DD9Zvkdfc3TRw5OjDi/JC54MgJP6Gh8DPc94oGn1cuSkuT+V1qcgoPbikV2ZtldnGsqoW3d4hm3toSnUbtno0sKZWR+97logVcaRzg8In+8S61WCbN0tJkFpZJO/zmlJgK/O4qFuWMNRbki5LL/DkeKpvl5kEnes+t5t6AhaVi8yxbKAt3Dp7toK5upKZ/MzJHft+5uYp1a6RtXm+X+x8/PTSsmU+0iMn9/HSzoUh3Vx91DZObHVkN3WKxWGYI00BDl6EpyQv5uWJz9hknRrDXz0CTsT8NhxeNHpjdg3Wyk+d8awabHpDFO8evS7jQni9bGIx0QJrr2lqiC9wfE5eZKy3daOhZaQyZkMvys6Khv/dhA80uWzeMrxlEfp6WCE9tk+XCObnyjI8/qaW+bqTc1n5umQyRDsjZhV48yaJCXm4Qn9KgqWI3mk3Guu4lJot0iekedH1E+7xBiKJ7pr5ppWjN2bPE91ReWcegUdBvdlbhDlFMMn3Ok1uzyDZ92I5D4tvq6w8/YyzN3P38LOPzWzRX7nH1egc1Db2TkmsadOhCSMOAklLuD8iXmJOTxMMbZKVo+XWZ/g26V2q635jCfWitdHbPf6eMAxVyze9/3wBAV/v4P+DNOBOH7xWCWWZ6mJMvUSwpOfmcvi7RMu++L/ai+uvhB9z4WXJzbb7YphXJbNkmK1nPXhPH6tFveiR8ldhPdy13B0518Zj6M7c4hZZ2icY4eza6ziTZmBvnZMndGrrCG73cCk0dcr/6jkFS0qI0JkSMUGWzE1i0TNZ+nKmUAersmZ4xLpwc2tXnpJmdwZavzaWxU5S4kydcTtcoB8E50tVRYFboXqgfoLN9cklmblhKSqm5Sqm9SqlypdQ5pdSfm+O5SqndSqkK85ozqSdbLBaLJaZEo6EPAX+ptT6ulMoAjimldgM/BvZorf9OKfVz4OfAX01eBBlSB4Oaw2b/zM2PyPBeWpjD+i3iIF1xVII3T1YHh50Lw3lbNNy3QO7z7DNyfnUvvP2OOCDbGpwt5SZILj8JzTbSuVGYDc9vF4dLbp5o6H0BD1cqZMVX1aXJqSsipwiUlyHfa9vWHAJKHnjQlEVb2+gppsUSLe72kCHVlgWLshkKSQVvbxtfO3TPCJcvEhPBk5vEZvD5oW6OVATN5zdfMQPGPHrseBBfUnQmUedxKSaM+PFt2SRminyff3wNgI720C3PaN1mXr9p3l2BIQI9opn3dk7+xotM0ENRqejGx652DJuGouWGGrrWul5rfdy87wbKgWLgBeA1c9prwLcn92iLxWKxxJJJ2dCVUvOBtcBhoFBrXQ/S6SulCsa55qfAT8e/aTg94qGvREPPnnUJgJe/dQ/F80Tz/eGLswHo/k0tlU0jDWX3LfPyk1cki2K7WYDw/35dQYNJaB/LnMfKdZ98SaPCC09lsHJ9EQBNjWKfCw5UMtQtIZfOYq+hm/C7LlssPoGVa/JoNnvjnT/vss9FerUslpsg0QQipGZ66e01m5+PlxwlgjQzi0wvlJuEfL1oPcZmATdJe1eUldulNt+7WBrdfZtzuV4rvoAzpztjJpNbszdpprh2fYBtJmxx470yiz54Ojh204xotwXZ8MBWCeLwpck9rl5vmrRcUXfoSql04G3gZ1rrLhXljh9a61eBV809Rn831xG/KZj3fy+rOVuu+/nZny4DYOsjstFFfzCBf/iNOBu9pqN+7skCPBniUXjvN5cBuHS+f7jMYtKRu6ZopbPlj+e3S4++/N5ZHDlhMoWZQNkXnppPSZFU8ExT4ds69MQdsOsZzvPml8p00e+H46fFhNPQEByWyZpaLLHAZxpTosdLp4k8GBoaXVkjTRWeBCiZI3HeIXN+Y2to3PNvK1rhS5AHbVgrnaIvOZkjRyUoot3sz5uQECN5Ir7biZOdPPWgmHxffmkeAFcbq6hpDD/XOd/pP511MMuXJDLPxMlfuSZKYfmFMbdfmpCoXMdKKR/Smf9Wa/2OOdyolCoynxcBkx9OLBaLxRIzbqihKxlK/gUo11r/L9dH7wM/Av7OvO64ZWmceM2gjFoHDjSyqFQcLX/6imjoq9bO5o9NXp3UdBHfm5DAr39XBcDhY+GQpJgMwhGjcNm8BP7ddyV8MNOsCv3siw4+2SMa+kMml0NqRjJer3hmPK7JjMvCNCElc+SipUvk+9fV97D/oEzjAhExtBbLrSM10pvkJZQg3shoTC45WQk8+XgZAH3dsgq7q7vuNsl4IzQr75U2t2yZxBLW13dzoVLaTbQmpEk8bgTnL/g59LXotY88LCbiZ57K4u33xdTjTsrlOIudlOFr12WTlCL92flzJs9Tx+TNVtGYXLYCPwTOKKVOmmP/GenIf6+U+glwDXh50k+3WCwWS8y4YYeutT7A+FsmPBZTaZzVoI6tKQQf7pSNKFYsllCedWvyeOwZcUA2N8nI9+ovjnDwmDHAx9hJ6GjmWSZ951Pbc5g7W/54a4fI9tG+juHzne3mWtr68QdHb3M1oVjO9wfWrJKEDsVzRNPYu7eNy1dEA7KauSXWODlSQsFB8vNk5pmVKQ2xdYLwxaw0D7lZUld7umTqGIz1hstRo1ixWGa0hWaBX0//ENufENt0fqkEFVy+2EObSb/b73QbMfBHBYLwlllEmGhW/G1aPQtlOqV3PpB+wuNTBE0bLsyWN4vLcmi4brabOyG28xCT90FMm5WibvTw6kdFS7tM/85XyFRmy8PLSEmRDrWrXQogkBB0FkxGbdKYCHch5pgC/9Z2WQq6ZtEsvvpKVmruMVE5bu96R4/I293rJy1NKnpSouOqCI493kQcTEmBJfOkQvo84nA6faF3uKFEmzbAwTpOLTeit0cqSVtzgJWrJJnd0qWiTHx1yO2cG1lZE7xquO0FhuRYKFoFJla4FmOUnxWlZ1aStNGUNB9r7pGUGVvvk4Hq4uUWLlXJd/p8v6QFqb12C6k/XNQYh/Cv3pKO/amGLJavlHJ87lFjKx4MUWj2Ss7Kk75s3twidn1aAUDFlXA2tMm2XZucy2KxWGYI01JDD2ugmnuWiIhrV4u221bTRIIWzTgjVUbDxx4rpvziFQA6um59owfnuvQ0eOxBGV3LiuT5p08088HnMnUaMBpJQkJYK/EZbTzNp+g2eR38gxMLEqm1FxZ4mF8qz60zmwzUN0e/ZGzU906ABBMLH4pBjg3LzMBdT3qNUlhR2c3994tJ88EtoqkfPSba7ODg6B28ZuV7STLJvBoazYx5jG1tb6um7rr58YuiBV+8IjN6n0+xdqlscbZ0kZhjZhek8NB60dqLZsuxDz9p5vz5wK0La754k5nU/O7TTjZelTZcXCKNsLdtiOJieb921XwAhvr8VF+Xfs3vslBMdqWt1dAtFotlhjCtNPTIvCSLFnp5+lFxhnr8Mnq+9qujJJhdH155ZQkA6+8tZMkSGZG/Odpj7jGJDPhE5LUwK0Cf3p7NhhXyx7HjMsp/+kUPHRG7V7kH0ZRkGSNT0pPRrWKXG3I5ifQYRvTIQbi4MJkcY38/cdJsiNHhvsn438PrURQVyWKk3BwJ4SpbmEJakjz4pPketU0huozd1AkTtdx9DKd2NVXgwNE2HnxQ1MuNxpa+ca20rQOHR4cEL5qfhtc4rq5ckxWZI1ZEj1XhbyPOU3qcfSECms+Py8xhr3mdk6N42vjEHn5ENqHJ+nYK/9hWDcD1eglmuKlZvnO++dpDIfjqvEyLEyvkVQchd47Y+je0yWx/794q9n0zMgvkzeTBmTYdurvwykqkI/qjF0tINzmCf/OWJNb55nSQNHNs/SZxfNy/oZSnHjG7HFVWAtDSPnTTq9S2bxOHxfatczj0tUSyfHZIKmtH31hOyfDgUWKmcMkpyQS19Px63CChkTj3XbogbXhZ2dUamUKGxu3PTc5os0Lu6ScLeOmVjQC0Xr0qMrc0U5gvcqUFpbF2BhT7vpEKVt8wOOL51ol6FxHxW5+9OMBHn8lK7ZefWwzAH78kq7W7O8s5dUHaQYFsasTmdXkEjUmx+mrYeVqcL204M1tMC5VX/QRuYHq83ThPr23XvP6ROEO9qaIwPf5gKU89ItlLfvmGxNEHb8VP6vqqTut31o8keSHHlEv1NXn+R3vb6Lj1rL7W5GKxWCwzhbhr6M6IEtJQXCBj2Y+/I0lqiooTefsj0ZC/Ph3eNqW7T4a/HZ/JSFqyIJvNmyQ51+kzorW/90lLVJM8t1a6qFS0ik1rZap5ta6DHZ/JSN7WPr6z1T01ys+Rb6SDQZraRJvp64vOXJJkUn6WFqegvHJincsZ6jFLTkNmfqw1pCfL+/lSZOTlBOjolVw35y5L2R39unV4VW2ST2SZPctHYCBit6PxRbPMUIYtBI6JYAh2fCxJ5YoKTVrch2SV9o++U8qRY/KZE+e9qGwWPQNiPlhSIrPAkvxUFi6WylxdI22gsnrg9n6RKHA0ZZUAJsstO/dIf7FxfQFb7hNH6UefiNZc1xaM6azViV7ecI+P1EyTCvuUPL+qNjbB+1ZDt1gslhlC3DT04UyI5jU3XfH0g+IATc2WcebfdlzjwMGRu9onKE3IOFqOHJdh9sipRr79qGgM962Se+w90EpnT1irhpGjbOSxotwEnnlAru3qF61i5+fNIzTzUfcwr5pwWGBKqmgm/qCmrknkGxy2HY7hqHUdSjSOy5TkRAJ9Ugb9PWEtOsdsQmAehdejyM2Vaxxl+80d7fz+fUkaMWRSQQT8EArJHwtL5b6DfYN0dUdoBVZFv2tx6rVS0GE2Z3jzA1kcE/BI5Vo+L48/+qHY02flyiw2wZuIp0ccey+/LBr9hapaDp8WR+rBw+IDCsQum+5NM1y9Q+C03vp6aQOVV/tZf684BgrzpFusa4vNYiPnuSWFct+N67IZMj3fsZPSv4VCsfFhxaVDd2/0bPo/nt6STlaBdFW/e19MKSeOj64FoRAoc/WQ8WSfPNPF41vk3HuXyLRpXnEtpy/2jbrewSk0n+kdn3wgl0WLxPO984Ck2zx7bjDqQjY+SdKSfcNfsqM9MOIcpfTovUwJl0WGyUmdluElM13i0DculWgXz2A3ucaREjA36Qsq2vqlYjQ2OilPnbsyfGf3ytGQqaP9PXo48qWjz/bkFsGdurmyUirLq/8kHfviZZ1s3CKOwxdN6uhZKYkkZUu7Ka8QU8Uv36zlSqVJAzAN1z24a7sTkdPRESA5VZSdPGNqosJ/6w9T4DEPXL/YRJ+lwRGzGrSmJjxoxMKsY00uFovFMkOYcg3d0UgdjfbJdeJI2fbQbN78XJx5J09OPD+LDG09fbqbi7Wyyuq+ZZKIZ+7spGENXU+QsausSMa09aszudYlo+Yxkw9ixMq48b6Mue2sLLlPcYHkXgmGoKt35MU3GoCT5VJ8Hk32LNHQF64wK0a7ehgyuTZrjfbT3gXBoYggRjX6SVpDqtHGZzlmG5/Cl2JOsBq6xUWkptgjcQGc+KaPikvVAJRli7b5wP3zOVYlJ/ziF+UAXK8Kq+XTcUMt6YP08HsAPRggYGKDg56xr5vUM1wz+23LjHP5UQmtbg35OfeROEOdfVMnt2pmfKyGbrFYLDOEKdfQHe38qeWijj777FwAznQOcOSY2Ky0K6vgWHal8Kgv79pbQjRWy6IG/1KJ3wt4wmPVRBkYH1orjpD0vHT27hLbeVPj5Ld4K5kttvNZBTLjOF/dzvlLI7eQutEoHDDmtARPKglJokpfN1r+qSua/j6ZuQz5x7/LCHldE4T1S6W8Fy4Q9eNkxQCd3dNJb7JMV9xbpyWayXOiUS27+nvZd0DyKF2v8pvzVTi0dmpFjQ4V7hNmyxpC5ucn0dsjHU/bTWwsMXzrCJ/bstmKl5+TzS6yF8jD9n91jat1IwMSYlVOcTG5PLEqiT/5Q1m2fyVJOsA3f32KbrO8fSJHpMLdocsJxZlQkikdamWNRHhcNMuQAbQKT6+GCzpfaun2J5YDcH5gkPIL0TlBIq0wHgWL5/hGyH78TBPNjSOdojf60bp65Yx+vHiN/aXqslSu7vZo7iAkRCznLslU3GM69HqzJrq2IUQgEDHtjOrulrsNp827lW2uAAAPuElEQVRoDQW50mUUFEja1+bmAc6fFcXFnVRvuuNIuGahtIslS/O42CNt7Xp9YJyrJsatsKWYQfCJdZkULBCn8dfGafz5520M+V0XuQW6RazJxWKxWGYIU6qhe5SE5q3blk9XqcSxfvSu5BupqxiYWDN3PnN9nmmcF997KJO5ZeLh23lWkk/V1Yxemaa1wmuGwu3bxNSSWCzhV7tfO0Vvh2NqUeb8sYdNZ0clJ7/K0kJYt0qmU61mtnbyXHd4B6IoQx/7jchtbd0M9YmTN3EwHNaUEPHc8YjID8S65UmkpMpfDcZh1dsf1iemvz5liSduJXJWlsxEc3LFJFjbNkBznTGVRtT36YZ75lpk8kE9sFrWnrQnePhwv5hc25pNP8AkQwldKvr6Mmmsy1ZnUuuX++3cLY7Q65cGb9u02GroFovFMkOYUg09wQOpmdAU8HP47bMAHN4jq8zco7p7gHcGMGek9AAm8o6XN4pWvvXJEiqMZvzZQRkFh8ZMHaHJNBfPX2EWQ5h8J+cO1I/x1DFkUmENeYEJVXx2Yxo5c0VD//KK2BOrL4UFmGgQdmsAg+Y71DX3MzAgdrylhfLkfYzc2ms8ElT4PLOJOIUlPmr65T5VZkP2EfllLJYoKZ4jNufUNKlkzTX9+Aenb11Sw/8xvMI8Cc1L90s48MqtspnHhxfb+eYzyVMzYoobRZtzp/0ukdvyyGaZ+aeVZLDHpKy+eGr8hY6xYko7dK0h6Ie2q+10XJFpSH/7xNckmTlEsemItxZ7KJ5jHDObxWxzggR2fCA9VdVxk4NynB8j2Xgruo2jtLJcHBU9XcFRU0V3lMvwrTQsNCs6Xza7Ga1+tJh9DdKBf/SB3C8YiN7UMmziMT3xuepOnhyUaeyWDWIa+nBfG3Vdzpx29H3dea3NNqiYfGUMJmkOmlzQzfWTj+Cx3J0Mp+cw9SQByMkVk0tISSd+/VovPT2j1z3cdqHGqPsjDrvbrXlfaBLebVkAWx6XFa+XQ3Lxrj1N+LvCbQOiaLfOo8wbr4KH15hosnvElHOxJcC+PSYVR7/rhrepjKzJxWKxWGYIU6qhJ/mgrEixak4iC5JlNMwwYXQBP3SbaCEzaJLgBZ/RhlffK5rBg48W0qxE7N1m/8Aj712lpdxo5mPsvuMe1J1PW+vE1JNgQgtTgJ4xRk1n0ZjZiIiH5nt49lEx1+SsklH+WLeH90z+mdbLIsekNOCIQPnKij6ae0WuJYtFQ9+4LJF3DztxvqO/m6NFzU+HHzwgpqiCRfL6+ik/dRcGRpxvtXPLZEn2QnqK2RfTWA/qageH8wNNyQYp5t5ZiRAytbk7MPqBptugNBUWm5nq1rWiPResyOFyokz5d7xjZvYnXWHOUbfbkTItLoC1G8T0qkwmvQM76miu9o91+m3BaugWi8UyQ4haQ1dKeYCjQK3W+jmlVBnwOpALHAd+qLWeMCJ/CEWb8tFfnMsiycLJnHmOzdtLt9k3tK1Z7L19A9BiVkU2Z8ro+k4FXLwsuSOunDK7kXdFn9Ktz6Sybe6Va0rMxhCr5ykq6uSzTBPS1OuHdfNEI1m/RGYIC9cVQInYx768JCPvhztqqS8XmW5GA448t7E2wBfGFzD/acn/8OIzxdTWyoq8b2rCXhtncdUySTLJT7+dTdlD8wH49Jzc49SJ6vDN7SoiS7RE1JWUVCjIly6jq1vaT11dDPZNi4JErwhzT4EIU1oIHWZKfUYin5mVCqUmnXRhici5aG4iC5eLg3IgS3xeBy8H+eI9yRt15ZTM1KOKOHDhnoFnml700XVeCsvkGadqpC87fcxVPlPQ9iajof85UO76+++B/621Xgy0Az+JpWAWi8VimRxRaehKqRLgWeC/AX+hJCzjUeAVc8prwN8Cv5joPv4BzaULg/zqrS7K5ksGsgyzhRqJCfT2ihbc2xo05wfpN7a69jMy4vX01BPqH5l4/kb2avdnnSZ5/56DEiO4ZKkUgcrycm+aHCucLce8aPLNtlra7Ip7sC2RC0dk0c+pw5LEv7MhHI50U4NvxIKMkF+z8z0JpywyW309s2Euf/ojsYPr34rd73qjpkQCffju82K7W/7IQvack/J56w1ZKDHQffMbZlssDl6vIjFdrNNdPaKhNzfEIGf4OLjrbIoxiq/aKG/WLE1FmbSIj3eIDHn5XgJB0VGvt5uUGamJHO8w9uzdMqM/fbwNf/fNLe93cDejtctF0HvXF9CnxW+1d7e00Y5m/5S2vWhNLv8H+E9Ahvk7D+jQWjtZbGqA4qjuFNI0Xuqm8ZL52z1HGA5nHSsSPYx71ShMsqDMuS21fvMqP6wvMYGsDLlx1oC8Zmf6UO1mhWWLxJK2Ng/S3z1yF6VYhQCO2DWmXirfG/9aCcCc3FWsuU/y3/ylSdHb29FNapq8z5ojSc5OX07gnd9clO925SYctBbLOOTnKhLN/i2NDVI/2zpu3w4W7jrr7CN86Ly0R783kZxMUQoLZokJ9PpQiPOXZB3KlQpRfroH/LR1igLW0eZ04vqmO1n3daXGXLvNrDrX6ZkcNWbg86c7J3fjGHFDk4tS6jmgSWt9zH14jFPHLBql1E+VUkeVUkdvUkaLxWKxREE0GvpW4Hml1DNINFAmorFnK6W8RksvAerGulhr/SrwKoBSEp+nXIt+tGuRWXiUCOcYUREfunO53AqRmeEGA0FaZHCnpdXJOh8E7az4DAs6SqYYa7/ubcBqq8Vp88+vXuCZx2USdM+SOQDklHroMlPHj/aI2efD3bVcvSazCWtmsdwKkfVm2eIk8nJEBzxzXjTQgYHQuOfHEmd19iWzAruyyo/PK91Xeppo6v7AIH190h6CE4QvcxMz1kirgMcDWzaIeaV0vpg7m9oHOPSVmGH7BsKZY6ey/d1QQ9da/7XWukRrPR/4PvC51voPgb3Ad8xpPwJ23DYpLRaLxXJDbmVh0V8Bryul/itwAviXaC8cb8Qa63DkUt5YMdHGGWHNO+x8dWu7t0smNzriIRfKG7lSJUuIi8z2dFkpPnoHxI1xrVFsd4HA4G2bOVjublJTE/GYlX4dJg3FYHCiK24H8tzgkCZodol38h6Nx6iZ6q20C3NtcbHinlWSEz4tScrk6sUmKq74xzx/qphUh661/gL4wryvAjbGXqT4MVFHHe/OUSnwmzj96prxK7B1gFpuFwO9iiFj+mjtDO/qEw/T3ngpeseSISZyOflsTI+5ZmM6pWWyUrzzqjiGDx3qZMCsc4mXudOuFLVYLJYZwpRvQWe5OdyO0vE+d79aLLdKpJZZW9VPsEtMCpnFRheM04xwqp4ZWQZzF0rc5vqV+aT65cODJ1sAKG8Iz5zj1Q6thm6xWCwzBKuh30FY7dsypUTUt9OX/ZzeL7G9a5eJI3DfLEVjU3ztxlPJ4nkSqlg2K4u2Rlm4d+KiBCuEiH8Z2A7dYrGMyXAKONNJ9YTgd59Jh77xupge1GACIKEuM7EjH/5Opgxy0uRNug+6E+V9SoEYOjxVEIzz5k3W5GKxWCwzBKuhWyyWCXFr3tdMytprx24tudWdgDsE2Gdy2MybJ4m+fN4EUryiD2eazXtvkD18SrAausViscwQrIZusViiJt5Ov3hh9tcgNcn4CxL91HZIPpuzZnvHkI5/+dgO3WKxRM3d1JG7v6uTXWDnFxJzXt3ZxbnjEpNfXTs1CcqiwZpcLBaLZYag9BQOKUqpZqAXaJmyh948+Ux/Oe8EGcHKGWusnLHlTpBzntZ61o1OmtIOHUApdVRrvWFKH3oT3Aly3gkygpUz1lg5Y8udImc0WJOLxWKxzBBsh26xWCwzhHh06K/G4Zk3w50g550gI1g5Y42VM7bcKXLekCm3oVssFovl9mBNLhaLxTJDmLIOXSn1lFLqolLqslLq51P13BuhlJqrlNqrlCpXSp1TSv25Of63SqlapdRJ8++ZaSBrtVLqjJHnqDmWq5TarZSqMK85cZZxqavMTiqlupRSP5sO5amU+qVSqkkpddZ1bMzyU8L/NfX1tFJqXRxl/B9KqQtGjneVUtnm+HylVL+rTP9xKmScQM5xf2Ol1F+bsryolHoyznK+4ZKxWil10hyPW3nGDK31bf8HeIBKYAGQCJwCVkzFs6OQrQhYZ95nAJeAFcDfAv8x3vJFyFoN5Ecc++/Az837nwN/H285I373BmDedChP4EFgHXD2RuUHPAPsRBKn3g8cjqOMTwBe8/7vXTLOd583DcpyzN/YtKdTQBJQZvoCT7zkjPj8fwL/Jd7lGat/U6WhbwQua62rtKQkex14YYqePSFa63qt9XHzvhsoB4rjK9WkeAF4zbx/Dfh2HGWJ5DGgUmt9Nd6CAGit9wFtEYfHK78XgF9r4WsgWylVFA8Ztda7tNbOrsxfAyW3W44bMU5ZjscLwOtaa7/W+gpwmSnaYH4iOZVSCvgu8G9TIctUMFUdejFw3fV3DdOw01RKzQfWAofNoT8z09xfxtuUYdDALqXUMaXUT82xQq11PcjgBBTETbrRfJ+RjWW6lSeMX37Ttc7+CTJzcChTSp1QSn2plHogXkK5GOs3nq5l+QDQqLWucB2bbuU5KaaqQx9re+NpFV6jlEoH3gZ+prXuAn4BLATWAPXI1CzebNVarwOeBv6DUurBeAs0HkqpROB54E1zaDqW50RMuzqrlPobYAj4rTlUD5RqrdcCfwH8TimVGS/5GP83nnZlafgBIxWO6Vaek2aqOvQaYK7r7xKgboqefUOUUj6kM/+t1vodAK11o9Y6qLUOAf/EFE0RJ0JrXWdem4B3EZkaHVOAeW2Kn4QjeBo4rrVuhOlZnobxym9a1Vml1I+A54A/1Mbga0wYreb9McQ2vSReMk7wG0+rsgRQSnmBF4E3nGPTrTxvhqnq0I8Ai5VSZUZz+z7w/hQ9e0KMHe1fgHKt9f9yHXfbS/8AOBt57VSilEpTSmU47xFH2VmkHH9kTvsRsCM+Eo5ihPYz3crTxXjl9z7wxyba5X6g0zHNTDVKqaeAvwKe11r3uY7PUkp5zPsFwGKgKh4yGhnG+43fB76vlEpSSpUhcn4z1fJFsB24oLWucQ5Mt/K8KabK+4pEDVxCRr2/ibc32CXXNmT6dxo4af49A/wrcMYcfx8oirOcC5BIgVPAOacMgTxgD1BhXnOnQZmmAq1AlutY3MsTGWDqgUFEa/zJeOWHmAn+wdTXM8CGOMp4GbFBO/XzH825L5m6cAo4DnwrzmU57m8M/I0py4vA0/GU0xz/FfDvI86NW3nG6p9dKWqxWCwzBLtS1GKxWGYItkO3WCyWGYLt0C0Wi2WGYDt0i8VimSHYDt1isVhmCLZDt1gslhmC7dAtFotlhmA7dIvFYpkh/H+mwsfkgVZ3GgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8 8 6 4 3 9]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB2CAYAAADY3GjsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztvWmMXll6Hvacu31r7TurSFZx62aT7G16xiONNBrbiTJSFI2zYhzDmSACBgFiQIITROMICPQjAawEcYIAgY0JJEQJZMsJbEOTRIY1kmekyJpp9d5sks29SBarWHt99a13PfnxPud+XxWLxSqyFrJ0H6D7Fu93l7Pdc97zLs+rtNbIkCFDhgwvP6zDLkCGDBkyZNgbZBN6hgwZMhwRZBN6hgwZMhwRZBN6hgwZMhwRZBN6hgwZMhwRZBN6hgwZMhwRZBN6hgwZMhwRPNeErpT6ulLqulLqllLqO3tVqAwZMmTIsHuoZw0sUkrZAG4A+NcBzAB4D8Bf11pf3bviZciQIUOGncJ5jnu/BOCW1voOACilfhfANwA8cUJXSmVhqRmOPCzV/tsM+Jc5IFuxPrYFONzTKx6jWP4DXu46vgRY0loPPe2i55nQxwE86Pj3DIC/tPkipdS3AXz7Od5zYDADd6uBqdTj57IBnKFzzFic5Eo5OSYJEGm5II7NOY3kJRs3niN16CkDg2X528vJcaGSYGlNKhRGcv1+fhfmM3zJmnAvcG8nFz3PhL7FFPd4O2utvwvgu8CLJaFvnqC1fnwgKtU+l03eO8djbYuOD/GItOPmxV8p4OwpOXnquAcAcDTgJwkAIPDlunpD48G8zHyPFg6uvM+CgV6pz4VXZJooFoB8Qc7lPDmejRUadanjzJysWjfuJmj6e9fRG77DPXvqs8Hmot1VAnq65B8Wt2SJTqA4MOoNKel6XcP3D658zzOhzwA43vHvCQCzz1ec/UXnRLN5YnEcoLfbBgAMDMgx5ykETblwdlE+wiQBgpAfabDPBX5JsdWkfdgf4l7D1DFPaXx8TOHcOZnIh/vzAADXshBFmtfLMVHAwGATAPCxlgG0uAzEyUGV/OkosE7vfKEAADg+LvVaX60jl+PkXnTlWHARs459/VIv223h2k0512w9ezk6F02PM5UlnyZaBzhJAkB3SY5nT0lBJiZcdJWkXWItnWc7QMGV36v1EADwaDHEtZvSz/Pz0iYK+/c9PI+Xy3sAziqlppRSHoBvAvje3hQrQ4YMGTLsFs8soWutI6XU3wLwLwDYAH5La31lz0q2B9i8Le6UHHt65MfBQZE0RgddnDnZDQDo7pNmsS0NS4tIMPtIpA8NCw8e1QEA731QAwCE4VGTP58drtNuW7M9zXkKIUXQlRU557/ku5s+qiMuviJS2tgxF55r6s2K2xbCFnd2lMk828Kx0SIAoFCQcXb1ZhN3pqmHDg+m/JvR+a2Mj8s3cfKEiKU6ks6yLQvlguw+igW5Rlk2gkQK7dpSn6kpD7Yr9b76uahh6g29rY1qK5jrykXg1An5DsvU4U/PxJh9tP/fXU66FxcuyPvPn5U28WwbNi3EIa3CyrZQ4g09ZTkO9EcolRsAgB+/J3PI8vL+lfd5VC7QWv8+gN/fo7JkyJAhQ4bnwHNN6C86zArvsJb5AjBMifytN2SlHR8XqdxTFlzq5xxbpEkNjTJ1hcMDsuJatov+EZGwrt4QBeHaSrhr6eOoIS+CG147Z+PECWnwyBcpzfZsxGyXq1dF2rszfdAl3DuMHQMunJNxcWKiCwBgqQSNukhgirs629GwnY0WYlsBjpLfxziOvJKFpi+7vvv3D3YAbR63jg1MTspYNzuOKBFJtKs7j1ze4jnjq5gg5s2a7jtFz8GpKbm3WpXrPr+5+2+jJM2D199wMTEqY6qnIGXr7w/QaEl7r63t7rk7hW0rnJ6Uv0/S0J33pP46juDaYmzI064QxwkSSutOQfrY0xZOj8tckyTSJu9+0MTK8v7085Gd0IslYGhQGn98VBq+p9/G8KD8PTokjVzMyYfZaAQItXSGjqXhFTQi+pvlOLhhAVDGgvUXdPbugEcD2sWLMoDfvtAFHUn7rNfkaLk2bOpfBgakPafvJ0heMv+9wX45XjjnYGRY6hsEsqgrS0FD6htx5nIV4Oak3iWOsySMEPvSBjZNWMP9BVy6KPcsLcnE3mhs70a7V9j87BMnbIyPyeSl6aHj5CnpREAYGecAudG2LFi2/G7RchkGPjQNhSo108U7rw+vO3dOBtdbF3vQrEs7O0reMTig0dcrltG1tb2xKG8u30A/cJ6G7m5aii2Lfeu6KOalTx2zyLVCJFysTTCChkKJqrVXT8niX2lovPuu1CfeY3VtxuWSIUOGDEcER0JCV6q9PTOS09RUAcdGaZjoltXVcRS464FPYxUohVgKcF253k2NWzZyDqUPSl+NRgS/IWoDrbdyxd9bGB/Xckmjt1fK0k+3yiROUOGWtiF2F7i2AoUorKxSYoz2r3wnT4hM8No5cXHryntYXJatsAmmsV0FZSQWikH6JZLOC/S9Pn1a6trTbafGy4Suh6VCDvkcpTjWcWEphN+SPjhxQnRSCkkqvRq5MgcLk9yWT03J865cCQ9UfWek08kTORSpQkjYgWaUB60EAXdfHq+xlYLLvo0ZPhoqC62WFH55tV2JndbHRKOOj8iYGuwtYq4u0rg27ovx44Ewz+sOaMpnbNrjoxZ6eqh+4k7Lo+HXUkgLEAfSTmGoYLl8GNvJsVTq1mmzdKPDeZRLUp8Kg7I6fe2fB5mEniFDhgxHBIcuoW8X7PMkGFeiwQFZj05MeBii0bKHwUHdZRcel3qPEoRlWalUZIJWjW7XApCnLtBlcECiLSPApyt/nGgUPLmuVJB7K6s7K/eTsFU4c2+vPHvyuBxHh13094sEWCrL+2s1Hy1fChhGrKNOEFAin1sQaW/6rkZl/fn1jJt1jH29Cq+ckTLZPFetBYhYE5u7m3LeQ6XJ6EhGSb7o8nlnXXu65B9DQ8adVaVSuEudLrRCyFotr0i7X70aoMnANE3RbazfgWXLvSZUXiFCgfrY05Oy1bx9u4JW6/Gy7BeG+lnHfg/sNtiUuNcZ9XjrbhOVmkij5RKNlF0WRoa5M+Gz4kQiJAGgWt99obmBSd0BwzhKd8OKOmzEClG86dnPIaJ33ponrUFvtwXD05DOE5xTFDRC/qYY7WS7FgApc0x7nJUotPhBtupyzlMJCkV5R4UG3b3q20OZ0Dsbb6uKGHIjxwFK3O729djpcXBAij0+IdvY4aF8qppYr8lXEDRD2Nq4rcjHYns2XM0O8WjcyJkmUNDca/kcSEHow+GgdvkR5soe+ti53Qz9fd7wWNMExqtgbFhhkhFpoyNS9oJnI0cLu9Fl6ChBgYNvoJeeJWGCJkNYu0ty70BZ47NroiNYWHn2iT0Nc+e/Txx30E+f/YjbzkRF0MoYQ+W6wNK4fV/eP/dCxxK3Yeqac4GxYfmbcxgcz4LNVnA5Fip+iNt3pd0XHkn9Zx/p9DnBjysAgC+8mceJcZm0lfn4Y4WQs3sXx3lPj41WK96n2nWAnTk+IZPy4IALh545po7rFfmmPr0SYE2qAceOWV7g1Gkp+7Ex6fCV5RDT0/K7WdB2AzNP16jajKMETo5qRi3vWm8GaGymF3jKq7bjY+q81dAaWHaSzgVl3mwM3k0/wkpNyhKDHjCuQn/ZGMHTGQ4FRpSGEcNbwyhdNNOyPb34O0KmcsmQIUOGI4JDkdA1xB8XAHI5sxoq5Ck1jw/J8jU47GFgSKRwE3lVzLWNVMWySBWJBqoNMcSFsZEUEygug8rh1shWUJQ6/FCua3HbFEUW7j+oSpmUSAZnJnuRsw1bHv1LXRsqlOfGz7FP2rClZ/Tbq3SRmjyRg2t2EKxDFMeo12SFT+vQDGEZacIYHWMNbYyRlkiKE8dcgCqmK1cpqa/qVCTYbS1MJOj4mNV2T+PuJtYalYYUYGFJjr4f4t5duc4YSvfKCLRfMP0zNWHjJCMnzZi1tYLH4IaWRTfMhwGufGZ2du1nmOcsLkllr90McGxMxnQX3QHr9RDUnMHjS0qF/anXZhgD5Ai/s+5SLt1tVbnTm12WcVette8z3DOrK8Bl8pbMkZyrVgXWK8/fuY26SMALq420HR2qXFbXQtRrG69/2hu3G2+d49F81wkUQho3Tf9odu7D2SY+uSrtUqNDguMkOH9WOu6Lb/YAADxXpxGlRlKPKiH81v4M/kxCz5AhQ4YjgkOR0PMe8OqUSNfHyOTm5hQcLsPDg3LO9hQKZTrv0/AQhjG8vNGnyXoUBBEadROswQitvIPEuFHRVzEIIiyuyqp//yEDOKiHq1U1Ht6VFffcKSnbxfOjYMAXalX5rdVMsEbD0Nras+s4jTQwNuTi7TdEpzoxTtc2laSGlBK5MxpNH0Es0oEx0OTzOcT0SQy54luWToMvjPQMAAN90gaXLshvn16NsbC8O3260fudYRTgYLeDkMyThr8kjBWm70mZbt6U9omCx6WjF1k6BwCXeuTRUQs5Gr8jmtQdDbRoLb8zK+Pi86vxBskckDqav436dmUlht+iXYjGbZ1E6Q6rtyhjf2gwhzvTjfQ5e4lOXfLJMRkPxwZo8NCAZllWKlKhO9NSx7hjuHfSBhtq4NmZ9nh6Ht5y2+iXuXt+NNdAkX0wOCDfpt9Q6Xu3QmcfGPvX8bF2BOwidx0r1ce3qSl3fWwjChgktyZ2BJfS9nIlwuxDup92fEYLZdnVhK8xINF2AI4Ll62ShAmi4AhEiiolNJjvXMjjzS9I2J1HjxWlAYuTtks1Qr3mI2hKa6i8iUxL4Nky+Ez4saM0iuZBhpvYBZbWpNPWl6WR1+oxbl8X1czCrEw6Kb1p0rauW1ZZypFzoEk8ZLnSkYlWmF2Qvd5ape3gvduPboheLF98u4STkzJpWxzAQYi2hw6j75IkTnurykhDK2/DMZ8O6+FaQJwYNZFRgwCKlTPUB+fP2qjRgNzY4Ycx0Ct/v/mqbCeHR7qxsCRqKp8Rk2u1GA/ukWiKz91/b/29R6kodSgUNTSNlxYFhMjSuDMj9b16RcZHrdq+t3MsbB4XfgtYXZd7BhlPYFkKNlUJhmpidLwL1kfyjmSPuXVNmVwHuPCajL1eRlU3WgFsqpMSo25otKfn7QjvnsVjbSsYp4gCJ3HPsdKQemM81k+RpTrff/6U1PFrXx0FANh2jOV5mQf+9AOx8t6ZabPF+RTyqo0YA2yXmKu18UxaqyYbFjUA8Fzg5LgsOHSEQ+jHsPPGmi4HN6dgbZp592p6z1QuGTJkyHBEcKASumOL7/jFN3owSgrRFg0fWmvkuJIp7mG0ayMx3Ak5E4KVIKKBL+SxFodY5Qq+XpNzlUqE2Qci4SzSH7vR0mg9wVijVJvudXRIymZZgN8UMdNNJWUPM/dlKxwGJspLYafJtrto7PrpL4u4e+pcGaHfZCHkkCsqhJSuA0Yi+irBA+44bt2UYy6nUChR/cJ2mjjmobco4kGjJvcmus2oYbHNjh2zcXZVXniFyQiiLQTBzmpNkaDo9Bnx48uXimnkp0lDNneriio5XKAef8bLgpEBGt4LKo3wU6zIw+UIn12VcVszxFA79DsLQ2C9bnaHFDMtDe0Y0it5SN0P9z2atrsEDDB+w+buOE4i2Q2iwwhsb3n7Y3ieft4qmUVPUf7o7XbQJNFbkxHeTV9vUHU8CeWchXfeEm3AMI3bYSvGyRMirTfo2nz/0aM0otocZ+YSFKiq9FxGijMye3Y2fkxCf3XSxZnTQvbnMDAjihO0jG6Iu/xIteNhOuufRYpmyJAhQ4YUB2sUVWJwma+HGKRBs9wnDGT1RhMBJQGXCigFL2XuW2zIsrlcDRHEjLwKZWl8NFPHIiMQmzRyNOsJfEa4PbYcokPf16ELtDa5cDmw0tyIFvV51RawsBBuuBe7SJX61qsi/Z+/IFKDVkCLSuwmdbWLSwFafEWdEsnSQoR5Bues0Zhp25L2CmhnYV+qJHj9NeEFKVHU0S0/5VKJTVlVjJMnRGJ5tCzvmF9KNrYNOgJsHODsKdlV9PRLHerNEB6lqOqSHO/di9N2SaWuHbfO4aBTOjJ5Ik9MSH0KORuJNtKWHGcfJqitbqrVDiupASSRCT6h223kwyV3c5hIX6wuNXe869spNuu/J4Zd9PbkeY766pyXGpOKnoyPwZLcuLysD2S3ZRg8y91SpmLJgwXjnitlK7hJqmvvDBjdXMczJ3I4PUWK40R2rJEGitwqT5yUMV0ozKNa1RuesTivUaULruPRwM/o3WaHy+RYv5Tz7Te6kefu2Ejljm2hQTdQvynHuzMBKpWNdd6rds0k9AwZMmQ4IjhQCT2KgOXlBB9+vIYquRnGjskKubZaw3pNJNB8idwICvAZOnzvrrgZLs+GqbO/Mf43qwmwy5Rmm/Vf0ECJsd39/UxmAcCmR42mGLC21sJ6dXd5woYocZwe9/D2G0MA2uHFjVqERiB/f3JPdOl3rgcIuJobvoiwqaE3sSbGCRBvKsrtqxEaLWmrM5NSnxPdLowFomECr+IYPd1M0jAm71ik5J/ojVI6ABTyCsfGZVdhmSAivwWLHhEzC/KM9bW2qPEy6s7LtHEYWgcNK/VyoWAN5lUQPIN/nvFk6e8Vb6EoXoPFpCoBdbqGm2cvsbk/BgY89PdIhZt0z4viOOU+GuwT6f2Lb8j1Y3113H9ETp4V7oRDvdUG+LlgbDkJlfjFvI08bQwmL8HCoIJly+A380DnTsu03qkTBZR65Xuu03ahoNBoSicarvKugoPqpu9aa6C5vnXHWgD6GOD4Nnnbu3tttPhc837bslKPnFpV3j97L0ASbdwNvJRcLlqLS9DsPY3lVWG0ynUxL6cfo1UlB4jhiLdV+qE0KmyVfaS3GBkyRhgZAK04gkXSJI8GjbX1NVTXn7x6dH7fx/vknp/9yqA8/3gReX4kNj8D11V4xI7+7EMxtvpV84TdI4mAhzfleXW2Wf9PlTDaxYhTQxaVIM2mc3xc6n17Wu5bqT7unjbY76K3X9oioAuXUgnq9EO/e6eePvflzN7EyYPZ3V26yWqlU7//OuMPqjW96a6d95ZnAQMcA6VeZrxZt1IeoblVadv5xf1LuupydS/3urA4BiyGQqo4SSlqbWahGqL/dr6QYHBYyhfR8L2+HuPju3JumRGTz8tLkiZDogDjOEBsuHMYbenDx2Zurk6M05h5+nQZTXIdN6nHdC2FFlflHLMOlUseUofxLVBkRPsxWYMx1mPjzJQshgPHpaESlaSD3jYR3ErD4/hx+duAA5i0ohvyWzyP8z6RqVwyZMiQ4YjgUCJFwwAIaYCrLW0T1XLAGCPHRqEszbKyUk2Z55y8LM2LKzGCTQxyWj++uI50KfzlnxDJ/NJPjAAAVpst1FpS3x5uu1cT4PM7RjJvb8OeS7rlvWvz8sf1OyHKF5gXklvMIAF8Bij1MfpuZMhI6GFH8g55xuiIDcumC6WJSnUd3L4lZV+YaffjdkXfjvHusLAhEIhbes1+V7GCYlqx6hp3PrugIt68W/HyQIEqxQYjr7SykCczprXEQKWOKOS93vGY59meDW6w0lSLnqdSA2Qa0JS6DjvoI81ugS6A+ZyN7iHZnf3hjyW6qtp6PmEzHSKGutZtT1MmirO+HiPZInGLuffNi2IIHRopY70iY9TE97iuSgPtWjEZHa3td10XTsr384YJxOrPo2dQnAMC+k82/RC2GT98iAsbEduxu0va7EuXihgsiXX1xqxx9tAI9kD7kEnoGTJkyHBEcHgJLgzHhXEb3Erx1inNHYAUV+pmCD4jKfxWCE1dmAlYmnvYetwtT7eL10dj2te/OoALb0kAjkl7V19tIaDir6dbVve7D2q4f52JhvenWrh1LcRQv3T1F84xA3krQp18MSZma3BISmDfbTPOddNl7dxUCYqSRotMlbGbw73bDLJqmaTA6HDnbJdhq1DxzTgsBkZ5r7x4ica+VdpzBgoKId1jLZM8/DlC3C1LwTHji7aTlp/ApY0jR37+nGenRry9QOfnZdz9XFul5Tc833nXSznezW8h+1vZFkC3QU2Dpdebw9nXZQAtcgfzo0+aRrh+JkndTm1ocmz5Ok0oUuSPvTknDXxKU7Zr4CRTUF58XQJ8IhUjZHKVYj/dkR0JvweAKm0W1XrbZtFZ1j4mN5k6zd37gOjNraKH0HBFhSYRh4LShiKEjhvQaPobRe/usTwuMX/B5KS89+a9EB/elLK0noPn5fAmdL3hsHWPH8DH3WkVd8g+FXLv47pAjr7c049k4pq5V3/s3s6v5RR5Iy59YQQu1RsNRpsiStCiFeQhJ457d1qIzGS4T1vroJXgIQmzLpymB5GO0zypWpHEn9mecp5KScuMP/aJk10ITbSeb1QvDlZXNhqSlNVBVtRRD0PTSq4x2FbbO6FB3949pizZMTrbe515WFfp8TM4ZaNJ4rd6YPLPtjt8132mADMFOVb72CRn0fKqtO3mSeC50TFGzYRp51SaMEIbrpQoTo2NqSqBYzbSGpa38RmVSiuNcTh5WlR3C0s+bpG4qvP72mkTFSlE5Oh1FoRJOlHlekUgmTheQr6wDgCoN9pPnpqUMvT0y3Vh4MPjmFPpwLSQK8h1KxX5Np9EtDcySlKwY3R/Yr3X6wEC46lCT41WK06DWRKGmzoK6YpjStkMdKqSKTGBzRt9LuDKHPP+ZzIGgmdYz5+qclFKHVdK/UApdU0pdUUp9cs836+U+r5S6iaPfbt/fYYMGTJk2CvsREKPAPznWusPlVJdAD5QSn0fwH8M4I+01n9XKfUdAN8B8Kv7V9S9xWbJqlwGeploolUTKdy1NQJusz//XNws1yvNLZ/BBR+vnRe/erfoYJ0ZAVqUvuABy4vy9yd/9gAAMDvj7xvnSefzZmdl1X/wUMTh4yMKioY/EwnZRTrXUjfQWJT7uvtEJHNyCn5d7nUcETnml1tYXNho1E6SthReEtJK9HZZmBilMamb70SS0s3OL8lz78/oNIIu2kf31O1gXOaq9KevNWLcvS/1vXWLvuIdvma77TPXtVDgzi2hZJsruAjprPyIdLytxrO/Yyt0PsP4uK9VYvjcbWlt2AwD2KRfjugQXm/JNcq10EeHAZdlX12P4dMt0HPlJZcueohoQL9zb2fbrs7d7uiwSK1lUmcnqu3rnlDO7R3w0liBOqM5u8rAwKjc02jQRdFO0nythp5bQcNiHU00efCEhBPGXdJwR5tdlR+GaYpAE2HuOjYU1ZcmjEBHCTTlZpNwJwojRIY1kmqbYtHBhQuyq6gwBuXq9QR6l2qKp0roWus5rfWH/LsK4BqAcQDfAPDbvOy3Afy1Xb05Q4YMGTLsKXalQ1dKTQJ4C8C7AEa01nOATPpKqeEn3PNtAN9+vmLuP3r7XJSZ4LWyKhJjd6+VJrO4e7NNvrCV693goDTlmZMSCYpYo2GS3NLXKlcqoEZ97K3rJNA+ADuBUkCDksiskdCPlVLFtnEF66FbVX+fg0XuJEyCkUQDMZXcTl6k9nvTTayvbQyAGTkGvPOmuIwND4nesZyz0MWoVGME9/0wlQCP0wVuYtzHLCMRjWS3unI4Po0zTNbQbCg8WjQ60ucvS3+/gxz5PprkJ7ItGx5d83yKdhr7F6AV0Oj22ad1jDDJ91C/ES0thIxWra5L+Rp0CLDtBD4lW5Wj1KnjdFdjkjZ09eTw9jsyRqot2aUuzic7q48GPEqyhn/esWKQvLC92000euhCaXJpf/mLRUxOyNgL6R6MvA0vZ3iM5JSybMT8R54cOo7TZlnsxJpszLG6Ii85xncWC3YqeQcMynJtDwUjhSt5WODEKR+TgaUVPHoimNwPYaxRKMsu9uLrct3c4hpWVrZpqy2w4wldKVUG8E8A/IrWel1tNattAa31dwF8l894YWMHy70uXGYW930ZNI5bgO8zb2Bti8i9jm3iIDOeWw4zCAWAaxmPBX4syksHRqenzEF6dphJ1HUceCxfos3WkaRoHbvknh5GyjoWAg7gKJT7VivtNhmdlN9+5iu9uDAl/vcOvSUajWa6LTWGTx22JwCX9ArH+hW6SvLMQrf0waeXI6zvclDvBar1tsrleUiyNt96/LgHh0a0JnPEOpYDpyQ6u95uOTpOdcsJZi8x89DHh++LYHF+Srb7caywVhGV49KqGOkMuVTOttDTJf0zNCZjpX/EQ4kLfJWEVEmsMDEmXlynz8q5xQV/+3He4Q6zyG/EjNWuvJ0mS/HpQOAVHbzxpuj0LLq7nJvoQd6TgWYWS2iV6kQ0B5+GasdZGEN3Z1k6yrm8JM+5+rG0SXJG+qerlEOB2aUqq7Jo3V+ro5jfKLiUyw5s8/3bUq982U1zARvdoh/G0KR/GGTGtolTLlZoJN+p4LcjP3SllAuZzH9Ha/1PeXpeKTXG38cALOzslRkyZMiQYT/wVAldiSj+mwCuaa3/XsdP3wPwLQB/l8ff25cS7jU2G0O5hTp3sQt5rpoRV2/bdeHY7QQc6SOM/ytPDR9z8dol8XvViag0Wr5KOTGKdL9aWtGYnenIVQYcOLes7THKUyVpmjsTcWe4X+YXYpRovBygtKCgU4OURclpcFjh1OsiqX3pi0Lc9dpYNxTpjevMBl9tNFEkJ2quSPcvJ0EckWOD9MFhM0z75xjTs4XnE3z0Pg1X/sFv8J6LwrbDV693QNp68mwZniMnq5TUExWjyUQZlXkaKRMLW/I+7yE0gM9uipH/wUzb/XSNRrmt6m6RBe+iTwl92E1VGsalMYkTePTDnhilRNvto0qt5Zaql46/q7XIvAwA4Oa8VK2TcPuYc4ALk+KAoBnJq+IYig8yEd46UVD8UE0wqOdYCDlGHy2KdN3pIthZLNMDV26bZDkyVkuejS7yyqxQtfpoJYTL3YCRlId6HHQzxqCHx+MTeQxQzWg4XxKoNI+tKfvZ0wVM35F+Wd+h6nEnKpevAPibAC4rpT7muf8KMpH/n0qpXwJwH8C/v6M3ZsiQIUOGfcFTJ3St9Z/iyUGMf3Vvi3MA2LTQnb8kHC0Xz/bDqosNn7E5AAAgAElEQVSeTHkm6ilBfUXW6DDolJY2xr8N9LgYZ9IHOw3WUbDp3mfS162tNLG0slEXfxAyp9btBBilHhPdFiFhoEWLRp2bTG23up5g/DiTG1CHHsdxGvEaUaF5ciKHybNS7/EeJseNY9QYgBRQ9NcAqkuUPHtEqnHzDgoxh18X03s1Q1gJmen404mxHB5NyPvu3X6GKJVDRGcxR8ekfSZGupBjME8X28x2XVz/RCTFazclWCZO9oR876kwu8zVxtMV9kq1r18kXWBY17BNIBANjNpK0sQ0feSoGRx2UK20x8N2MJTZfpVuhv0WLMVk7Zaxy/jIO+Z9vD4KoUgUHZqk6XnrsWAfFSdo8e9KR4b07Yy2JjXgbJrCMgEWH2dnbG3aVVUWA4AuwGayHbvXwNuvi/H2lTdk/il16dTwaxJGjg9ZGOoTO8b6ypOZIDtxeJGihwyXxpNTJyQeqmwD65zgPE8mnfXVCJ9+IiO32WpHBiabcj0Wii5c8oIryECxHIV8QQxNJu/k55+tpZPnQaBzgA4fk/qePC4TsE4UwDDlCj15Pr/XHtwu6YLtNAxcQzE9UhzI9rMn56JgnM5JTRpBQfv8wPP8zVP44IYxscgAHRnKw3HlvcOTooYJ+gGfnNQmn2ZXwcXIoJx7cJd87YcUUfo8GCAntxO1+b4HeuRjDhIP0/fF8kv36UOjQdgOneVZouHy6pUa3s7J5GQby7dWiOhFYtQcx8ZzmHlA4aj1eAd2Ll51GqRnH4gQMH6slGaLcpQxbGo0ORAMBbDtOEioLrU42Rc8NyXi8lmBZjOARZ7kQrnIN7cjwLdDWs6tRNwt+quzHw2f/oO1BMlnonodnaBANFmC3jSwc9pKHQt2ioycK0OGDBmOCP5CSeidW2Djjjc6LCt1FLY5P/NMW3Ptah1Xbm8yYm6B7m4LpVybbhUAbNuGDZHKrlwR6fTDK2vpPfu5nd68dSyULLz5phhth3tJxu9HsOi2aOUMTWq7VHnjfqXahpo83a8SuqdZtkotqoGhX3U82PmN0v3cQg2f0KjUNFF9+QYKzPjyFWZUP3a8C3UYN1G69GmNkrdRDePvX+6HPcGGDPa0AY+OyR9hGKY8KIY7qLYOrNV3lwXrsGFody/fCWF78o2cOSXj3cs5UHRJDRkOPNpno485QheMhN7xQXbSIpkg4ekHMmZeu6ThmqwcRop1NAI+x0qY6czz0uxAhpDGKzsIuaM0H10Qx8hZZgexuy9QP/bHU67fwsKqAMytyT9mFmWcnzjThdi4UHJbk8s5KHbR0oyd0YxnEnqGDBkyHBH8hZLQOyl6jeq3m65HjhWmUWM5BhgtVFpoBB20sKCBcZME7FcjaBphtInKiRIszEuY2fuXlwAANb9t6TpI/ejwkIezx0XP6dBdy080Ykoxij5hRnIEgDx3HEYqth0bHqUj323LATGfpyhVWJ6NGhnsphlde/lKCysV4won9zUCDTBfY99N0V8OjxTSVH8mutaPkjQoaYexbC8UjlFHevaUtH/RQ0cSBAYv1eto+i+XhG7gh8CnN5iSkO6Y58468JONbIs9JQu9PVLvBWNO2eIb6Dw1y2Qfd++v4uxpcVH0zC5RJ1AM1DE5X/N5N2XLXFkWd8xybx5OgQyjLUaMtiKE9FOsrHUmiD0YaABU6yOgrUgDaNGAYnau+aKHYn53MvdfqAm9U+ViIro8mpatRKfW5ZBb+lql1Xbv6dCRbJ6Mc/ky3BJVNw3xUtBK4cZd2YreuS8T3EEZusw7LG4rz54qo4ehxi36zPphjLlHshW9fE2O6x0JcUNGsJkk1NoTNRIgIc4A0IwATWNojoRT640IP35PDMkffyQfSzN4vN6dbXHrrrzr9OkGhnqNVw2vs612BKu18f7Our6oOHVcJqLhPhp+a3XYliGdknpNP6xgfv7xBMUvAxQAww59465MkoNDQUr7QLs7PMdKPX3u3N0YCfokNOiI8KP3aqgzvv/1iwMAgHzebnPsK8MHnMfcinx/t++IMDE4VkQvKT2CwHjZKDT5jVfWD3chNWM6iRJEJH0LKVDqIEFlbXf6xUzlkiFDhgxHBC+phE6DBiW3RGu0rSFPRqfUc2pKIhvz3LatrtThuCJB6ECev7QSdrgntUV0k3Fm6oS4nb31xijKlFBrPo2JuRyqPv1fjTHkAFQGnZLv8KCU5ezpMhioiYBl8hwr5Su5PysF7OQOqdeNb7icLDgKFvUvxkDVin04ORPxJhLZJx/N470PRDIPtxF+OvuiwV3vejXBYB8NQnk7rZDxEtXJ1ve/aDBly3kWTozIzi1gRpxGo4UuRuguV8WF86NP1lIq4Zdl52HQWcwV2vzv3AsxwOxAxuvOhsYIo44LeVGxVWs7cw5YWtb40Y+lrVbWpR37BtyUj8VkDMoX13DvDumHmQ2q+VaELkfeG9OSm895aNF5oeUfrh6vaZjFVIIC1b9mzlmvRFhd252PbiahZ8iQIcMRwUsnoVsW8MY50aOdPTMKAIgj4PNbYni8cmueV+oN9wAiKZeZcf3MK0xEQZe5ZhChNydS5vIa083NduQP5R+DXTm8dUmYgi+9KVL+5Akbiu57xs3PtSwUC5TWTUEOSH/OOAucm5IVv5iP0IoZIERjZ97zYOeZdZ4igSHD1LrN45EmwUg0NP0Fo9ik19IoUqp4sCpi9vsfVVPJfDtps9OeMTkuzxjq89JM81ZbQE/pR4MDVHeWilL4k8MF9Bap1+cusJjzUpfPBstWb4RYWZffK2TmfP18D6YYNNVsiW5XWSqNuL3/QKJDZx7szCXtRcXmfq7WknZEkeE9SpI0cOZZ+HGa9Dz8+ENuI1WUetm243Ha7Tg6anbxGo5hW+RustBTwJ1p2SWsVvaZ0vIJMJJ0wQRFoZ3eT7PRtE6w20zDL92EPjHi4utfPwYAOHVGJnadWDh5VbZ4i4sysa9U2kQ9YeeuhQPNc2j0o/Evn8ulIfrRknT2sW4bPmeWwQFp7dfOD+LVi6JqyZXoieH7iOhTbJPK01VAuSz32NxiP80I9Dzo/KjGmbXl9Aka38IAYULeZ07aDSvB7Ip8AHG08eMDgMg0mvnJBnyqXwwveqlYREDPoI+vyUK6stRWfW332WoApH/G6xfEE6S/z0rfYRbGxNKoNTdGiO61cblzcRmhJ8bFV6Xvpk73oJ+++yYnZVdPOW1wnzON1i6ilrTF+rK0a++og1KX1KdWYRSy48FwjM3Oy71+q12Wl0XV0gm9afgMdzltkiqzMFtmgsKGgfHM1dVtGoKtYLjUbVip51mSUtUmuHlbjKbNWoeq1tRjn9Re6QKkgfEBmWumxoQC2HFcNJoiFBk6YMsBHGd3hchULhkyZMhwRPDSSehTkzkcOykWvmYoVhjfDzHI7Nl/5QuSMaiyEqPJaLWHCyJxz640wcT2yHM5dknm0130QNsJBkma9LM/NZbmIXTpG9o7nEOuTKmrblQWMYo5uTlHw2GMBDo2eQP3rv5PQttVETg1JRJlsZcJLKDhce22GHV5++E67t7aqMPolCCM2iSKDJeLgk8Jx0gQVa3w2bS4Zl69LGoqnTxF1dLx2yBzRw4PMxbAAbQjbZ/QxWy5HmB+aX/FVg2gm7uFt86LimRoXAaKVwzgFsnTY9QIRR+k7IFLVVu+ZCGXk+vGT5f53AgB83HaieE50WhRIl9aard/p1pwtzBtaqQzjXbbH6TA30N15uSJYnrObEq9ggufEnq8T3HSnYTDJulGd6kAm1Kuw3iLyAdWFjaqWpRSbVVQh6T+WEmf4ju/GUq1k12YHUWXp/Az78g8NXFCIriFmYtOHiYRh1bb7kK2QiahZ8iQIcMRwUsnoZe7LVgMJAjrjPZaWUOebnOX3hb9tue58Ol+WHlEEv87VSRKVuk+crmQJBGua8Os73nyb0xM5VLpukVJS1sxQpNbMjR6UQWba2PLpPtOYjh8nsdjE3vP4ZJKvPz3YI+F4+MibkYdvBZ+Ua64tyq7ivfeb6BR2ViKDZnhKSVE1EUqpaEptitbpOibMzX8yQ8k7G99Md7yOdthgDuIPPNT2pYCSPjvM7Jodj7GyvxGsXU/9MwnKZGPHWduT0ZUqcBDjoMkZv1blQY0s5fkyEDpN1upDjTH1GSO7aRjNV+QNmvFASIaUr2gHT27WQ+9myqae5/uuLv36LRnDPRLXfuGXcSa7Jv8RppOgkXaFjptBnsJ3VGYgSHpA6tkIYjlvV3dsnNo6Dw07I33ag2m1MUAXXwf1oFwBx2xnTuy1u2+7OPzv/hqEcfP0q3TMTsFJeMfgKXkwmYYp3PYTvHCT+jKWHzZLEsREDIHZaHICbNiIYplwlVUeeRKCmSvRa4oE0bvsV7E7KFCF1/AARCjHZ1ovBkixEjST4wJkpVKo0s1DYK25yLH8OIqE0xbFjAxJAPIcIo/WAr3dEZX7eJTcQS8Numht0fK0iCJlu1YWOXi996fi5pq4W64bVEMfW0StfndFQecw0jHpfs1VB4Yr4NtHobH1TBlBzjez99cOWm7FiIuJGs8PpqPEbfH/LbveFZYCugdJuUAufA1jWVe3kWhJB+fDhhJ7MeIWJG88YAJEsScoAOGcCdWApuG9jQhd6iQkCzq/ASzQVV83OGCuPoMnjyT1COe5IxRaUS4uULvmwOc5XM5ekapuG1o57eUBBoRx+Bm+um9woanstnrdR9WKAttX49IaquVBAvLG1cVxwK+/LpQab9zSo7Xri/i1n3xRAo5BtdZ9pVW25ussTFNMIC26mOqCxjtk34+R/qHE+cK0A7zr7KdLMuFReNtRG+cG3Pr7ZyiO0SmcsmQIUOGI4IXXkJPLQo83LwX4sa6rG5vMe9kIZdDg3kQA0pJlm6TbfkkZ/G1BvKyrEbMlmMI5JtxnKoo8twm6ySGQ6nUJr9CIQ+0XJHYPlsRQ2BBRXi9IKu/E9d4r8LgsKzIo8fESPZgafV5W2MDNAAu/rg4Kav6maliqlayKE3kPAtLlNgWb3PFT7YXdAOSktVqzBTTn4PtyDvy3AV1QaXuxk+z5aXRkxQhvnw+hxMTsoXyqcqwHBsh2/7BHHM+zj7uVrbXyLlAgX7nSbzRWBYkOt1e5IrMi5oHGiTTirlztxw7HTdmhwel0CTXTcNvk1WZZChnXhGD2MRQCfcfyFh671MZI7cqTyjspl1K3gW+9tVxAMBfelMMbSsLVfzg3TkAwB9fpoS5X1KxBgrs05P9Ro2hYbOAhkwrtixEap/97Tt0cfRhgBdrxDRIx1rKMvewjpXFjaRcOQ84dV764+Rb0o4DkzlcmhOHCuNxafhlFueqaTTz0go5i2ohchzg3eTumZjoRne/vNelilHbMZpNqvQskzQmhk1d77VVGYMfvr+OaJcJcTIJPUOGDBmOCF54CX2zsWjlThN/+sOHAIDBr08CAEqFPOyY0nKaMi1Jg2ksJdX0VJRyJwRGYqIbn99KYHMZdktM/KDtNFKyFpKc3/fw+UORct/7E0kWeHY0h1epL3edtpTiUKofGjQk9e2ozL0QN/Mu8IWLslt4i/o/30pS/hWH9c7nC3Bs0RmqbRTRncbGFgWY+TV51vnTZRQ86pWpNz5/rg+XL4soOc2Aoq2CfjrPnTspbfL2OwOImVgjYHBOy4pQpVH54Yy0cdjU+85v4rgKeeotHRg9MI3BGgj4mTQT6ccoAZbq4q7pQiSt4YINj/0dky8nSiLUqWBtUX/sOnZq03GYbzNfcPEqE2AMjUl//umfLeDOvNxrdLRB8tiGFRNDDrpPytir9HIX2V3CKw1xDrh+T76LB2v7o0zv94CfOC87lzPnZSfaUAk8TSpq0iGv+AmWN5VhrwLEOscHvYdx+qS0Z2+vh2qVqe+or66vBSndtUFPj4Ne2nSqgfRtUNIoTMlziuRqyvGbOtnsgsOdVmONO484QqksbaEK5GXJKVSb0gfG7bcABYs2lYBzTsm1seBLm/3pv6KjwYy/67H/wk/oBrrDNnnjxzKR/j85Gcinj9s40SeNMdJnqCejNFKU/FvwQwWPftU2HxiTkcpLEnTT/xyc0Ju+gh/KdR/ek06+druGR/S9bszKdnYIPWjQauJRRZMoGyGTAR8bled1F4Bqc+MK9TwDemJI4eQr8hG5AzTcNVpIlQbcZkfaQs2ctJ6+oCgF0GUX9x7IzF671GscUNDijyMnSnjjoqiV7v1wLa3PZs8brYE+GqgvXeQfvRaazEqU4+BerMa4OScT5PKj3XvNPCuUDcSmzJzIPa+d8PjhonyQV6YlfH9hRSNoyEL21mmZlIfOdSPgYhSzB+IoQsCPOc38pNq+5g1SKdgKyDOq+AQX5q8VNd5elnvrnJDW13ys1TgpMXr24pvd0JyAFkjX3N2TR65L2u9LF5mc+mGAWXLSz/MZcdRWJaS+7Fa738xIKfGz6MsDeQpAI0NyfHWyhKFJGQMtCj/1xSpitqNRsrTCBM3a/nvFu1xUrW6ZiCPLSh0qfLrX1BvN1AHCYGwsh17j3kKPtlYtQBjQuy2WuSakfzs8jQa/g6RMP3fHQYvvD6m6i+shGjVGgDLYoNybh0PHitBEXzs5XL8u6p2ZT9bTcu22xTKVS4YMGTIcEbw0EnrnUtWqyqr50fenAQB3h4qYek0k1Uuvyr6pYFlgalAMFZiRJ45hU8y0aDCN6e6YKzioOPKM2/dlJZ+fb6FeExHm6kfCEVN5VH+MRGK1EmKJWXpG6O9dqUaISeE5OCqr+itTObx3lfk4OxNmbFPtzi2X0eacGJTjmxeLKNFlLaSLopfTKBVMJK2U/c5CA5evcNvX2t2avzQn0sX6uo9etqNflfbJ91s4xx3Cyc+penmkU4maAaUY6Aa+9AYlWfLMrFUaqXqjwLyJD+/V8em78j6/vn+8N5sRhkCDeo2YzsgJzby6FcEOpW7L5P+4dT3AqAjS6CVRXBgmaJE9rJA3FMUKvV0mJoARpTkHAV1n1+rSjsW8B5cqwIiRp31jBfSP0ldZGRUg0GDbGwnTK7toUGWl0mxZAQZpLR/+yRG5PnSwtCyS39qa7CzXVmKEDbmnSKNwV5eDIDTiuvxW7pb3l7tyaUhruUsk1u6BIpaqIlmuLclzm80wHeA2Deiz8yFq1SfHPewVQl8apsZMRI1+CyEby2Hu0SAIH3t3LXaxGkqbDeZMjlKVRgSnZHX8poIkRo0Gbyb+ggcF25X5xLg723HajMiXDD+RjUUK4Qll6rmqxqcfyviKudvvdEveKTIJPUOGDBmOCHYsoSulbADvA3iotf4FpdQUgN8F0A/gQwB/U2t9oPnYNV3HVmcqqC6R8pb5DV1HokoB4CtfECny3DE3jfZr0r3RZrKGSujhh/9SXMauX5Fjq95CzMCakHm2VPq/9uq5tuxj4ZE8d5CUtWtr1dQ16cxpEedeea0Hn9wSg0fQ0VJbRZqltBKGPiQHXDon0vhbZCcs95XTHFbGyJMoF3ZRJLtl7mTefXcRM9NkBdyl+1qTknKzqTBM6TVgu4fNBAN0zfzaT0sdr35WxTIpSUeHpLyvnSlgcEzKbHIohutNFGhAasG4k8Xw623jamf99xuKEjlVmyANEKxEY6hP2vMnLkq9Tw1Zaaq8yQkZW6FOUpdH1xU7gWV5qFLyN0kYuoeKsH0Zq4YLKAwC+Eaipbuo7ai0DQLqavMFB8UcpXYG1zUaLXgF44Ir12ul4ObzLAMNsImH4/2yZR2PZBcbRW233BxtBm7BRqsp5wK6+xqeGQ0r5RlxaRgPQp3qgTWlY53odiDgovx2/Uqw693hs6C3S+oxziArVylE6XaYNNFbbP6m79TxRz+U7/6r74hBeWKklPLzaD6jQGNnXG3BoviumGdPKxuRiTA248dzYTvsC0+ODxY1/uiPJU2jreR56zUfD+5u9FXtzIG8U+xGQv9lANc6/v0bAP5HrfVZAKsAfml3r86QIUOGDHuJHUnoSqkJAP8mgP8WwN9WYrL/KwD+Q17y2wB+HcDf34cy7qB8QER91sL0YscPchjuFsnh7LFhxNSjVRsisZZLEkxw7UYDH/5/DwAA4VaZFDolxg42NkBcHm/dFp3dBHXElg7hWtwFcNkcGy/iJ78gUunteyL9zM3rNudKB4xQQXU4Lp138Tp52AfoIhn6MQKKkopilO9rTC9I3T74VHSaD6dbaXKBnWBDejhKzA9nE4wNMkUflYKNVpJ6qFx4TXTJr5zpxgqTXeSoS+7pLsGnN5Emj4lbCFGNZPjd+FQk1Qf3/D3x/tkt4linCbVNsmuz+1MKiEh1MH5KAk5OnUtS7xCHwVYqatM6BImcuzkd4c/fF0nMkIJcet3FGSa96O7l+EALbt4EmLS5343u3FAK1CIf+SJZIFneOG4n705otAibSTp+XIbj66iOmElOEvKY5HI2EhKWBx3Pa1GPH7MTEo6xKAxhmSQo3A0ErQRImJrQY0KVOMYKA/2u3pCxuLx8MBwEXV1SxyLD/GOdpC64FgPuerra7o10NIJfD/Hxjx4BADQ90b78k/0Y6pFvzdgntMvk1zZQKJogRsPB4iDBxraLlYfpGfn7/kPxQrr/oInbN8QmZ7KdaJ1Abw7Pe4ZvYKcql/8JwH8JwDCgDABY01obZ7gZAOO7f/3eQGukH5PqdLrSJlJSzhTyDlrMo5nwuhY/3Om7q+lEnm4xOw2WWzSu7vjx6rUVAMDUKWnSV8dzUOxo25aO6u528Zd/+jgA4M03ZKB/+tkKrt+mwZI+XoMDFnrL8pxiQV4yPl6AR1/pFg1oSRwhhuFckesXqiHefU8m8gdGzfIMs6NZrIzK6ZPLyxg7JhPasQFuXRFBc5NnDH2F7i6MlGTQ1xtSoXoQI6HlKMfOKJY0Prwi5Xz/XTGqtZoHZwjtRBAAqyv8e4zcGqmbbIJGXfrHhtQr15uD5odoIkBt24JHdciVO3L997+/iMX5+oZ3zc01cPWEqGTeuCTHV0+X0UOX2YgzTFD3oRlxalmmvwNE9HHPufL+7i4btYacq5J/ppBXcPi7IXwKdYKAej6feTSbTQtxYrIwmXyxOh1fFsdWrNt8SsZ/usXcnkGzhRYXoZgCTKSA+QX5e272YDMCFQomdFcOjXoLPlWfBYaP9vXmUWB4q8+yWxYQ03j66SeiFp17VMfImPQRNbMo5GR1GBp00G/Ytri6FwsWXE74lXV558P5Bj58X1Q5S0vyPQphmfEXRfu4B0LMU1UuSqlfALCgtf6g8/QWl25ZHKXUt5VS7yul3n/GMmbIkCFDhh1gJxL6VwD8olLq5wHkAXRDJPZepZRDKX0CwOxWN2utvwvguwCgTNLK/YAxIqbugDqVtPvIq+C4CpoqlxKpGGs0Jq7Xmo9t93cq2ColPA4AMM1chV+6dAx5RsnVa8ZQq2CT86V/WFQUx8Z78Oq0SARVk9G8P5dGnNYbIhEEvoaVmC26KV8Mh5LV3IK848OP6nhwr7mr8m+FzffOPqhi+r6op06fEAOoo/10p7O2IhJHPWjCYZ5Wv8H9bKyRoxHN4hZ/bh648plIsq007dzBqlo6ceu2tF9vr/TZyXGRmPM5C9QkQJGgNg41HKo5WhENxL5GQH6gq1fFJ21xvr5htwcA9VoT169K/8zPynWzlwZx9hV53jBTHZYcB16ebKJ0hc2H+TTgqYeBM83QwvtXZHsx+0h2PK9d6EVI/hJ6TcK1beTz5FfhVx/oBD7zn1pU7VnKS4NeLH4QTV6jbI0CDXuhInMiIjQDo8Yk5fGsjytXDH3uFo29j/DyUhGXAVA60MgVDK+MtJ0krtgok3aOu4jqwbmHFcyxj1SqAWir5rrpuplQnVkoWHDzdLJYEWl8fb2BwN8mQnaPx/tTJXSt9d/RWk9orScBfBPAv9Ra/w0APwDw7/GybwH4vb0tWoYMGTJk2A2eJ7DoVwH8rlLqvwHwEYDf3JsiPR86A9uNMbLA4AbbtlMDkuvJSl5blSWyuho8HvP8DHgwLdLXwwULUyfF5ODkRWpPohCVVZGiWnz/wGA3zp6Wgprf/DBBlZzajBeC0hZyDMCxjcElsbAmQi7e/1juvXOztaeLvpFMkkTj82sirUweF+Ps2ckiPNfoSMXgE7UCgG5cOYqCFixElMxXyQ3z8WcrWF7aKL4dlnSuFFBZZwDZNWnwAYaB9/Xn0uQUDuugowQWWfVyTJn3+Y11XL0qkvKtm229+WZbdKd0trYmku+P/uwRLl8WPesIg4lGBj2cZHqyMu0pzTpSeoquqkiCd+5X8QffF2bFdbKQ3psJ06C6ty6Ji+IrZ4sAddzlfvKN2BpaybixKdl6OQcxbR8mCM8zUUzKSvMNNFnFtarC/Rm5/iZ3OYsLCfxdsgTuFap1JiGhITTvuWkieMOs2tNno1jmeKw82VgrfUUjcMqhI3806wGa9V16aR+AwX9XE7rW+ocAfsi/7wD40t4XaQ9hAt5o/Y+VTsMXFY8VRnPW689uvOnsoEWqPn7/X8zgzbdFrXLpFZnYy54NRbJ9n+qThp+Dwwwlmvvz5bUaqjXZspkOyufaXWXKrpWDO5/L8+7eoQF0j/dwnXV7+EAm7f/7/70LAPiZrw3jtdPkZnEM/ayFlslDSoPcylKIh0ty79yszATT0/uUtuYZ0FnHuUcyEX38mUx0cIEh0sL298pk67geQnqg3L0vi9yfv7eKG9d5zzYSwVYfcxwnqDCy0RxvALjcK37JLqOaG40IBRJ6uVTnra0GaDY2tuX07TZNs1k0ph+WYJHreHREfOfL3W66CGgl15XyLirrUoZSNzMv0cOj2QpQp4F4+q70Y7UaYnmZ31D1kFbkDpj6Vqi+LA3asJhVKohMVKyNsXFGJ5Nob6t+2fOJ9wCaJ4sUzZAhQ4YjgpeHy+UZYNQFxt1PI4FLSdci8f7KukhYftjeIj7PypzQhenm9WxAju0AAAiKSURBVDU8ostaHAwDAL7y5RH0MbKyxe1aHATpu326/iU6aec65TbfctrZwJuGCfFRgGufi8SUpvzaI/enrWCiTOdn5Z1/+P2HuDste3ujoqivh2i0pJwtUhXPzwWorJmISbbz4Qtz2+LmDZE616oVDA6IBDw2KtLs4EAZdfqGf/geXdwedCZv2JvKra09voup13eXJGJmpr7hCADFkkjwXh6IKLWmLrZum+8nRzc/YxRuNTWadPsNgxezA5eZyGV6Rr7rkb6elAq5GchvXUUbU+OyS/nEZQKQAzbe7hcyCT1DhgwZjgiOtIRuaEtMgtec8tCiEQvkwqjXDf/E3r5bKaC6Jsv+v/qRRIX1jpRw+rhIBhFMktgQNbLuVSiReZaDvElBTk/PGAkercg9d8kGefNmgKWFTaLFAQpOK4shVmnYzDNQIwyTtqvaPiV1PgiY8bDwUGPhobT3DRpMC+XVdEfUqB5gFubngFLtbmjQXtSoP/l6oG34fNLzgMMzZHeiMwmOySB4e5ppKt/wUGCquCAw/VcCPO6QD8d2u284chP6Rn9SOS4sSUdGVg5JxKhMk4M02p8RqdEe9PNz8v5//vv38er5XgBAT4nRoz0WTGb0VlVmQi9vpYQ/ESMs5x75uHVLnrPAJBDBYdsVO2x/zUb7y3iMbOwQ/cufF511iULjERU99vuLXr+tyve07FI7IY17odChblyYJwX2eoJTzAJlDMkPV0J8dFk8kkwil/2MgbDTLGpUqe5TflcgU7lkyJAhw5HBkZPQDTpX3Mv0D544VcTkkBjx5kjGf//B2oZ7gD1aqfXjmoa56RaW58WIVuw2tKUq5ZqxuL56eZXSrRqesNXFBI3axv3hYUZWAthQx862eyGlt2fEkyTb7X5/WfA0V72XrW6d5V1dlg/nj34wg9nXZVecZ6Tozc/XcNu4mB7ADusgdzWZhJ4hQ4YMRwTqWZj4nvll+8nlst17uWydOFPGV3/2JADg1k2hNX33D+cBIGUD3Ndy7FCiftJ1m1WaL5kAlSHDwUMB+RJdfxm412qESA6WBHIv8IHW+p2nXfQXYkJvFwAYInVpY53Rbau78+vdizIAyGbjDBky7AY7mtAzlUuGDBkyHBEctIS+CKAOYOnAXvrsGMSLX86XoYxAVs69RlbOvcXLUM6TWuuhp110oBM6ACil3t/J1uGw8TKU82UoI5CVc6+RlXNv8bKUcyfIVC4ZMmTIcESQTegZMmTIcERwGBP6dw/hnc+Cl6GcL0MZgayce42snHuLl6WcT8WB69AzZMiQIcP+IFO5ZMiQIcMRwYFN6EqpryulriulbimlvnNQ730alFLHlVI/UEpdU0pdUUr9Ms//ulLqoVLqY/738y9AWaeVUpdZnvd5rl8p9X2l1E0e+w65jK90tNnHSql1pdSvvAjtqZT6LaXUglLqs45zW7afEvzPHK+fKqXePsQy/vdKqc9Zjn+mlOrl+UmlVLOjTf/BQZRxm3I+sY+VUn+HbXldKfVvHHI5/3FHGaeVUh/z/KG1555Ba73v/wGwAdwGcAqAB+ATAK8dxLt3ULYxAG/z7y5IOsfXAPw6gP/isMu3qazTAAY3nfvvAHyHf38HwG8cdjk39fsjACdfhPYE8FUAbwP47GntB+DnAfxzSGzvlwG8e4hl/FkADv/+jY4yTnZe9wK05ZZ9zO/pEwA5AFOcC+zDKuem3/8HAP/1YbfnXv13UBL6lwDc0lrf0VoHAH4XwDcO6N3bQms9p7X+kH9XAVwDMH64pdoVvgHgt/n3bwP4a4dYls34qwBua63vHXZBAEBr/ScAVjadflL7fQPA/64FPwbQq5QaO4wyaq3/QGtt2Ed+DGBiv8vxNDyhLZ+EbwD4Xa21r7W+C+AWDijB/HblVEopAP8BgH90EGU5CBzUhD4O4EHHv2fwAk6aSqlJAG8BeJen/ha3ub912KoMQgP4A6XUB0qpb/PciNZ6DpDFCcDwoZXucXwTGz+WF609gSe334s6Zv8TyM7BYEop9ZFS6o+VUj99WIXqwFZ9/KK25U8DmNda3+w496K1565wUBP6FrlPXix6KqVUGcA/AfArWut1AH8fwGkAbwKYg2zNDhtf0Vq/DeDnAPxnSqmvHnaBngSllAfgFwH8Xzz1IrbndnjhxqxS6tcARAB+h6fmAJzQWr8F4G8D+IdKqe7DKh+e3McvXFsSfx0bBY4XrT13jYOa0GcAHO/49wSA2QN691OhlHIhk/nvaK3/KQBoree11rHWOgHwv+KAtojbQWs9y+MCgH8GKdO8UQXwuHB4JdyAnwPwodZ6Hngx25N4Uvu9UGNWKfUtAL8A4G9oKnypwljm3x9AdNPnDquM2/TxC9WWAKCUcgD8OwD+sTn3orXns+CgJvT3AJxVSk1RcvsmgO8d0Lu3BfVovwngmtb673Wc79SX/tsAPtt870FCKVVSSnWZvyGGss8g7fgtXvYtAL93OCV8DBuknxetPTvwpPb7HoD/iN4uXwZQMaqZg4ZS6usAfhXAL2qtGx3nh5RSNv8+BeAsgDuHUUaW4Ul9/D0A31RK5ZRSU5By/vlBl28T/jUAn2utZ8yJF609nwkHZX2FeA3cgKx6v3bY1uCOcv0UZPv3KYCP+d/PA/g/AFzm+e8BGDvkcp6CeAp8AuCKaUMAAwD+CMBNHvtfgDYtAlgG0NNx7tDbE7LAzAEIIVLjLz2p/SBqgv+F4/UygHcOsYy3IDpoMz7/Aa/9dzkWPgHwIYB/65Db8ol9DODX2JbXAfzcYZaT5/83AP/ppmsPrT336r8sUjRDhgwZjgiySNEMGTJkOCLIJvQMGTJkOCLIJvQMGTJkOCLIJvQMGTJkOCLIJvQMGTJkOCLIJvQMGTJkOCLIJvQMGTJkOCLIJvQMGTJkOCL4/wGMW3oCm4ckuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7 0 6 2 5 4 3]\n"
     ]
    }
   ],
   "source": [
    "path = '/home/luo/文档/dataset/handwritting digital string/cvl-strings/train/'\n",
    "X_train, y_train, X_val, y_val = loda_cvl_string(path)\n",
    "\n",
    "# 输入数据前先对数据进行归一化、中心化\n",
    "# X_train = X_train\n",
    "# X_val = X_val \n",
    "# mean_image = np.mean(X_train, axis=0, dtype = np.uint8)\n",
    "X_train = (255 - X_train) / 255\n",
    "X_val = (255 - X_val) / 255\n",
    "mean_image = np.mean(X_train, axis=0)#, dtype = np.uint8)\n",
    "\n",
    "X_train -= mean_image\n",
    "X_val -= mean_image\n",
    "\n",
    "plt.imshow(X_train[0] + mean_image)\n",
    "plt.show()\n",
    "print(y_train[0])\n",
    "\n",
    "plt.imshow(X_val[0] + mean_image)\n",
    "plt.show()\n",
    "print(y_val[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 将dense转化为稀疏矩阵，用作CTC输入  \n",
    "## feed sparse注意事项  \n",
    "\n",
    "https://www.tensorflow.org/versions/r1.0/api_docs/python/tf/sparse_placeholder\n",
    "\n",
    "x = tf.sparse_placeholder(tf.float32)  \n",
    "y = tf.sparse_reduce_sum(x)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "  print(sess.run(y))  # ERROR: will fail because x was not fed.\n",
    "\n",
    "  indices = np.array([[3, 2, 0], [4, 5, 1]], dtype=np.int64)  \n",
    "  values = np.array([1.0, 2.0], dtype=np.float32)  \n",
    "  shape = np.array([7, 9, 2], dtype=np.int64)  \n",
    "  \n",
    "  print(sess.run(y, feed_dict={  \n",
    "    x: tf.SparseTensorValue(indices, values, shape)}))  # Will succeed.  \n",
    "  print(sess.run(y, feed_dict={  \n",
    "    x: (indices, values, shape)}))  # Will succeed.  \n",
    "  \n",
    "  sp = tf.SparseTensor(indices=indices, values=values, dense_shape=shape)  \n",
    "  sp_value = sp.eval(session)  \n",
    "  print(sess.run(y, feed_dict={x: sp_value}))  # Will succeed.  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dense_to_sparse(dense,dtype=np.int32):\n",
    "    '''\n",
    "    把密集矩阵转换为稀疏矩阵\n",
    "    \n",
    "    args:\n",
    "        dense：密集矩阵\n",
    "        dtype：稀疏矩阵值得类型\n",
    "    '''\n",
    "    indices = []\n",
    "    values = []\n",
    "    \n",
    "    #遍历每一行\n",
    "    for n,seq in enumerate(dense):        \n",
    "        #在已存在的列表中添加新的列表内容  添加密集矩阵每个索引位置\n",
    "        indices.extend(zip([n]*len(seq),range(len(seq))))\n",
    "        #添加密集矩阵每个元素的值\n",
    "        values.extend(seq)\n",
    "        \n",
    "        \n",
    "    indices = np.asarray(indices,dtype = np.int64)\n",
    "    values = np.asarray(values,dtype = dtype)    \n",
    "    shape = np.asarray([len(dense),indices.max(0)[1]+1],dtype = np.int64)            \n",
    "    return tf.SparseTensor(indices = indices,values = values,dense_shape = shape) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试稀疏矩阵是否正确"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if test_mode:\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        sparse_train = dense_to_sparse(y_train)\n",
    "        s = sess.run(sparse_train)\n",
    "        print(s.indices)\n",
    "        print(s.values)\n",
    "        print(s.dense_shape)\n",
    "        sparse_val = dense_to_sparse(y_val)\n",
    "        s = sess.run(sparse_val)\n",
    "        print(s.indices)\n",
    "        print(s.values)\n",
    "        print(s.dense_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sparse_tensor(sparse_tensor):\n",
    "    #print(\"sparse_tensor = \", sparse_tensor)\n",
    "    decoded_indexes = list()\n",
    "    current_i = 0\n",
    "    current_seq = []\n",
    "    for offset, i_and_index in enumerate(sparse_tensor[0]):\n",
    "        i = i_and_index[0]\n",
    "        if i != current_i:\n",
    "            decoded_indexes.append(current_seq)\n",
    "            current_i = i\n",
    "            current_seq = list()\n",
    "        current_seq.append(offset)\n",
    "    decoded_indexes.append(current_seq)\n",
    "    #print(\"decoded_indexes = \", decoded_indexes)\n",
    "    result = []\n",
    "    for index in decoded_indexes:\n",
    "        #print(\"index = \", index)\n",
    "        result.append(decode_a_seq(index, sparse_tensor))\n",
    "        #print(result)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def report_accuracy(decoded_list, test_targets):\n",
    "    original_list = decode_sparse_tensor(test_targets)\n",
    "    detected_list = decode_sparse_tensor(decoded_list)\n",
    "    true_numer = 0\n",
    "    # print(detected_list)\n",
    "    if len(original_list) != len(detected_list):\n",
    "        print(\"len(original_list)\", len(original_list), \"len(detected_list)\", len(detected_list),\n",
    "              \" test and detect length desn't match\")\n",
    "        return\n",
    "    print(\"T/F: original(length) <-------> detectcted(length)\")\n",
    "    for idx, number in enumerate(original_list):\n",
    "        detect_number = detected_list[idx]\n",
    "        hit = (number == detect_number)\n",
    "        print(hit, number, \"(\", len(number), \") <-------> \", detect_number, \"(\", len(detect_number), \")\")\n",
    "        if hit:\n",
    "            true_numer = true_numer + 1\n",
    "    print(\"Test Accuracy:\", true_numer * 1.0 / len(original_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义CNN网络，提取特征\n",
    "* image_batch->CNN->[batch_size,max_time_step,num_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "MAX_TIME_STEP = 32\n",
    "FEATURES = 128\n",
    "\n",
    "tf.reset_default_graph() # 清空之前的graph\n",
    "X = tf.placeholder(tf.float32, [None, 50, 200, 3])\n",
    "is_training = tf.placeholder(tf.bool)\n",
    "def conv_layers(X,is_training):\n",
    "    \n",
    "    #first conv layer 50*200*3->44*194*32\n",
    "    Wconv1 = tf.get_variable(\"Wconv1\", shape=[7, 7, 3, 32]) # conv1层参数\n",
    "    bconv1 = tf.get_variable(\"bconv1\", shape=[32])\n",
    "    out_conv_1 = tf.nn.conv2d(X, Wconv1, strides=[1,1,1,1], padding=\"VALID\") + bconv1 # conv1\n",
    "    out_relu_1 = tf.nn.relu(out_conv_1) \n",
    "    batch_norm_1 = tf.layers.batch_normalization(out_relu_1, axis = 1, training = is_training)\n",
    "    \n",
    "    #first pool layer 44*194*32->22*96*32\n",
    "    max_pool_1 = tf.nn.max_pool(batch_norm_1, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID')\n",
    "    \n",
    "    #second conv layer 22*96*32->18*92*64\n",
    "    Wconv2 = tf.get_variable(\"Wconv2\", shape=[5, 5, 32, 64]) # conv1层参数\n",
    "    bconv2 = tf.get_variable(\"bconv2\", shape=[64])\n",
    "    out_conv_2 = tf.nn.conv2d(max_pool_1, Wconv2, strides=[1,1,1,1], padding=\"VALID\") + bconv2\n",
    "    out_relu_2 = tf.nn.relu(out_conv_2)\n",
    "    batch_norm_2 = tf.layers.batch_normalization(out_relu_2, axis = 1, training = is_training)\n",
    "    \n",
    "    #second pool layer 18*92*64->9*46*64\n",
    "    max_pool_2 =tf.nn.max_pool(batch_norm_2, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID')\n",
    "\n",
    "    #third conv layer 9*46*64->7*44*128\n",
    "    Wconv3 = tf.get_variable(\"Wconv3\", shape=[3, 3, 64, 128]) # conv1层参数\n",
    "    bconv3 = tf.get_variable(\"bconv3\", shape=[128])\n",
    "    out_conv_3 = tf.nn.conv2d(max_pool_2, Wconv3, strides=[1,1,1,1], padding=\"VALID\") + bconv3\n",
    "    out_relu_3 = tf.nn.relu(out_conv_3)\n",
    "    batch_norm_3 = tf.layers.batch_normalization(out_relu_3, axis = 1, training = is_training)\n",
    "    \n",
    "    #second pool layer 7*44*128->3*22*128\n",
    "    max_pool_3 =tf.nn.max_pool(batch_norm_3, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID')\n",
    "    \n",
    "    #first fc layer\n",
    "    W1 = tf.get_variable(\"W1\", shape=[3*22*128, 32*128]) # fc-net1参数\n",
    "    b1 = tf.get_variable(\"b1\", shape=[32*128])\n",
    "    out_pool_flat = tf.reshape(max_pool_3, [-1, 3*22*128])\n",
    "    dropout_1 = tf.layers.dropout(out_pool_flat, rate = 0.5, training = is_training)\n",
    "    out_fc_1 = tf.matmul(dropout_1, W1) + b1\n",
    "    out_fc_relu_1 = tf.nn.relu(out_fc_1)    \n",
    "    batch_norm_4 = tf.layers.batch_normalization(out_fc_relu_1, axis = -1, training = is_training)\n",
    "   \n",
    "    #second fc layer\n",
    "    W2 = tf.get_variable(\"W2\", shape=[32*128, 32*128]) # fc-net1参数\n",
    "    b2 = tf.get_variable(\"b2\", shape=[32*128])\n",
    "    dropout_2 = tf.layers.dropout(batch_norm_4, rate = 0.5, training = is_training)\n",
    "    out_fc_2 = tf.matmul(dropout_2, W2) + b2\n",
    "    out_fc_relu_2 = tf.nn.relu(out_fc_2)    \n",
    "    \n",
    "    features = out_fc_relu_2\n",
    "    shape = tf.shape(features)\n",
    "    batch_features = tf.reshape(features, [shape[0], MAX_TIME_STEP, FEATURES])\n",
    "    \n",
    "    return batch_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义RNN网络\n",
    "* num_classes = 10位数字 + blank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_hidden = 128\n",
    "num_classes = 10 + 1\n",
    "num_layers = 1\n",
    "    \n",
    "#定义一个ctc loss需要的稀疏矩阵, 以及一维的batch_size长度\n",
    "targets = tf.sparse_placeholder(tf.int32)\n",
    "\n",
    "def lsbm_layer(inputs):\n",
    "\n",
    "    seq_len = np.ones(BATCH_SIZE) * MAX_TIME_STEP\n",
    "    #定义LSTM\n",
    "    cell = tf.contrib.rnn.LSTMCell(num_hidden, state_is_tuple=True)\n",
    "    stack = tf.contrib.rnn.MultiRNNCell([cell for _ in range(num_layers)],\n",
    "                                        state_is_tuple=True)\n",
    "    outputs, _ = tf.nn.dynamic_rnn(stack, inputs, seq_len, dtype=tf.float32)\n",
    "    \n",
    "    shape = tf.shape(inputs)\n",
    "    batch_s, max_time_steps = shape[0], shape[1]\n",
    "    \n",
    "    outputs = tf.reshape(outputs, [-1, num_hidden])\n",
    "    \n",
    "    W3 = tf.get_variable(\"W3\", shape=[num_hidden, num_classes]) # fc-net1参数\n",
    "    b3 = tf.get_variable(\"b3\", shape=[num_classes])\n",
    "    \n",
    "    logits = tf.matmul(outputs, W3) + b3\n",
    "    logits = tf.reshape(logits, [BATCH_SIZE, -1, num_classes])\n",
    "    logits = tf.transpose(logits, (1, 0, 2))\n",
    "    \n",
    "    return logits, seq_len\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试cnn_layer和rnn_layer是否正确  \n",
    "判断输出维度是否与期望维度一致"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if test_mode:\n",
    "    x = np.random.randn(32, 50, 200,3)\n",
    "    train_indicies = np.arange(X_train.shape[0])\n",
    "    np.random.shuffle(train_indicies) # 打乱顺序\n",
    "    idx = train_indicies[0:32] \n",
    "    y_label = y_train[idx]\n",
    "    sparse = dense_to_sparse(y_label)\n",
    "\n",
    "    test = conv_layers(X, is_training)\n",
    "    logitstest, seq_len_test = lsbm_layer(test)\n",
    "    total_loss = tf.nn.ctc_loss(targets, logitstest, seq_len_test)\n",
    "    need = [test, logitstest, total_loss]\n",
    "    with tf.Session() as sess:\n",
    "        with tf.device(\"/gpu:0\"): #\"/cpu:0\" or \"/gpu:0\"\n",
    "            tf.global_variables_initializer().run()\n",
    "            s = sess.run(sparse)\n",
    "            feed_dict = {X: X_train[idx,:],                        \n",
    "                targets: tf.SparseTensorValue(s.indices, s.values, s.dense_shape),                          \n",
    "                is_training: True }\n",
    "\n",
    "            ans1, ans2, ans3 = sess.run(need,feed_dict=feed_dict)\n",
    "            %timeit sess.run(need,feed_dict=feed_dict)\n",
    "\n",
    "            print(ans1.shape)\n",
    "            print(np.array_equal(ans1.shape, np.array([32, 32, 128])))\n",
    "            print(ans2.shape)\n",
    "            print(np.array_equal(ans2.shape, np.array([32, 32, 12])))\n",
    "            print(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model(session, out, loss_val, Xd, yd,\n",
    "              epochs=1, batch_size=64, print_every=100,\n",
    "              training=None, plot_losses=False):\n",
    "    \n",
    "    (logits, seq_len) = out\n",
    "#     decoded, log_prob = tf.nn.ctc_greedy_decoder(logits, seq_len)\n",
    "    decoded, log_prob = tf.nn.ctc_beam_search_decoder(logits, seq_len, merge_repeated=True)\n",
    "    # Inaccuracy: label error rate\n",
    "    ler = tf.reduce_mean(tf.edit_distance(tf.cast(decoded[0], tf.int32),targets))                                                           # tf.cast返回的是bool类型的tensor\n",
    "    \n",
    "    \n",
    "    # shuffle indicies\n",
    "    train_indicies = np.arange(Xd.shape[0])\n",
    "    np.random.shuffle(train_indicies) # 打乱顺序\n",
    "\n",
    "    training_now = training is not None # 判断是不是training\n",
    "    not_training = training is None\n",
    "    \n",
    "    if training_now:\n",
    "        variables = [decoded[0],mean_loss,ler,training] \n",
    "    else:\n",
    "        variables = [decoded[0],mean_loss,ler]\n",
    "    \n",
    "    # counter \n",
    "    iter_cnt = 0 # 步数\n",
    "    for e in range(epochs): \n",
    "        # keep track of losses and accuracy\n",
    "        incorrect = 0\n",
    "        losses = []\n",
    "        # make sure we iterate over the dataset once # 历遍整个数据集\n",
    "        maxloop = int(math.ceil(Xd.shape[0]/batch_size))\n",
    "        for i in range(maxloop): # 向上取整\n",
    "            # generate indicies for the batch\n",
    "            start_idx = (i*batch_size)%Xd.shape[0] # 防止向上取整以至于i*batch > Xd.shape[0]\n",
    "            idx = train_indicies[start_idx:start_idx+batch_size] \n",
    "            \n",
    "            y_label = yd[idx]\n",
    "            sparse = dense_to_sparse(y_label)\n",
    "            s = sess.run(sparse)\n",
    "            \n",
    "            # create a feed dictionary for this batch\n",
    "            feed_dict = {X: Xd[idx,:],                         #Xd size [N D]\n",
    "                         targets: tf.SparseTensorValue(s.indices, s.values, s.dense_shape),  #yd size [c,]\n",
    "                         is_training: training_now }\n",
    "            # get batch size\n",
    "            actual_batch_size = yd[idx].shape[0] # 防止向上取整以至于i*batch > Xd.shape[0]\n",
    "                                                    # [a,b] --> a <= x < b\n",
    "            \n",
    "            # have tensorflow compute loss and correct predictions\n",
    "            # and (if given) perform a training step\n",
    "            if training_now:\n",
    "                dd,loss, incorr,_  = session.run(variables,feed_dict=feed_dict) # 运行session以获得这些参数\n",
    "                                                                            # 返回 mean_loss, correct_prediction, accuracy\n",
    "            else:\n",
    "                dd,loss, incorr = session.run(variables,feed_dict=feed_dict) # 运行session以获得这些参数\n",
    "                                                                            # 返回 mean_loss, correct_prediction, accuracy\n",
    "            # aggregate performance stats\n",
    "            losses.append(loss*actual_batch_size) # 记录了每一步下来的loss\n",
    "            incorrect += incorr\n",
    "            \n",
    "            # print every now and then \n",
    "            if training_now and (iter_cnt % print_every) == 0:\n",
    "                print(\"Iteration {0}: with minibatch training loss = {1:.3g} and inaccuracy of {2:.2g}\"\\\n",
    "                      .format(iter_cnt,loss,incorr))\n",
    "            iter_cnt += 1\n",
    "            if (iter_cnt % print_every) == 0:\n",
    "                print(y_label[e])\n",
    "                dense = sess.run(tf.sparse_tensor_to_dense(dd))\n",
    "                print(dense[e])\n",
    "        \n",
    "        total_incorrect = incorrect/maxloop # 总的正确率\n",
    "        total_loss = np.sum(losses)/Xd.shape[0] # 总的loss\n",
    "        print(\"Epoch {2}, Overall loss = {0:.3g} and inaccuracy of {1:.3g}\"\\\n",
    "              .format(total_loss,total_incorrect,e+1))\n",
    "        if plot_losses:\n",
    "            plt.plot(losses)\n",
    "            plt.grid(True)\n",
    "            plt.title('Epoch {} Loss'.format(e+1))\n",
    "            plt.xlabel('minibatch number')\n",
    "            plt.ylabel('minibatch loss')\n",
    "            plt.show()\n",
    "    return total_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_lstm = conv_layers(X, is_training)\n",
    "logits, seq_len= lsbm_layer(input_lstm)\n",
    "out = (logits, seq_len)\n",
    "\n",
    "total_loss = tf.nn.ctc_loss(targets, logits, seq_len)\n",
    "mean_loss = tf.reduce_mean(total_loss)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)\n",
    "\n",
    "# batch normalization in tensorflow requires this extra dependency\n",
    "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(extra_update_ops):\n",
    "    train_step = optimizer.minimize(mean_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n",
      "Iteration 0: with minibatch training loss = 51.2 and inaccuracy of 2.1\n",
      "[6 0 9 2 5 1]\n",
      "[0]\n",
      "Iteration 10: with minibatch training loss = 17.5 and inaccuracy of 0.96\n",
      "[8 8 6 4 3 9]\n",
      "[5 0]\n",
      "Iteration 20: with minibatch training loss = 15 and inaccuracy of 0.82\n",
      "[1 3 6 0 7 5]\n",
      "[0 0]\n",
      "Iteration 30: with minibatch training loss = 11.7 and inaccuracy of 0.74\n",
      "Epoch 1, Overall loss = 16.9 and inaccuracy of 0.897\n",
      "[3 5 9 9 1 0]\n",
      "[1 9 0 0 0]\n",
      "Iteration 40: with minibatch training loss = 11.1 and inaccuracy of 0.67\n",
      "[3 5 9 9 1 0]\n",
      "[1 3 7 0 0]\n",
      "Iteration 50: with minibatch training loss = 8.04 and inaccuracy of 0.52\n",
      "[1 3 6 0 7 5]\n",
      "[5 0 0 0 0 0 0]\n",
      "Iteration 60: with minibatch training loss = 5.95 and inaccuracy of 0.34\n",
      "Epoch 2, Overall loss = 8.58 and inaccuracy of 0.512\n",
      "[5 8 4 1 0 7 7]\n",
      "[5 4 0 7 0 0 0]\n",
      "Iteration 70: with minibatch training loss = 5.47 and inaccuracy of 0.33\n",
      "[5 8 4 1 0 7 7]\n",
      "[5 8 4 7 0 0 0]\n",
      "Iteration 80: with minibatch training loss = 3.04 and inaccuracy of 0.2\n",
      "[2 5 0 0 0]\n",
      "[2 5 0 0 0 0 0]\n",
      "Iteration 90: with minibatch training loss = 2.75 and inaccuracy of 0.19\n",
      "[1 3 5 5 7 9]\n",
      "[1 3 5 7 9 0 0]\n",
      "Iteration 100: with minibatch training loss = 2.92 and inaccuracy of 0.22\n",
      "Epoch 3, Overall loss = 3.76 and inaccuracy of 0.236\n",
      "[1 3 6 0 7 5]\n",
      "[1 3 6 0 7 5 0]\n",
      "Iteration 110: with minibatch training loss = 2.44 and inaccuracy of 0.2\n",
      "[4 1 7 9 2 4 8]\n",
      "[4 1 7 9 2 4 8]\n",
      "Iteration 120: with minibatch training loss = 1.5 and inaccuracy of 0.17\n",
      "[2 5 0 0 0]\n",
      "[2 5 0 0 0 0 0]\n",
      "Iteration 130: with minibatch training loss = 1.05 and inaccuracy of 0.13\n",
      "Epoch 4, Overall loss = 1.49 and inaccuracy of 0.162\n",
      "[2 5 0 0 0]\n",
      "[2 5 0 0 0 0 0]\n",
      "Iteration 140: with minibatch training loss = 1.23 and inaccuracy of 0.16\n",
      "[7 0 6 2 5 4 3]\n",
      "[7 0 6 2 5 4 3]\n",
      "Iteration 150: with minibatch training loss = 0.315 and inaccuracy of 0.11\n",
      "[7 0 6 2 5 4 3]\n",
      "[7 0 6 2 5 4 3]\n",
      "Iteration 160: with minibatch training loss = 0.471 and inaccuracy of 0.11\n",
      "[3 5 9 9 1 0]\n",
      "[3 5 9 1 0 0 0]\n",
      "Epoch 5, Overall loss = 0.553 and inaccuracy of 0.134\n",
      "Iteration 170: with minibatch training loss = 0.269 and inaccuracy of 0.12\n",
      "[3 5 9 9 1 0]\n",
      "[3 5 9 1 0 0 0]\n",
      "Iteration 180: with minibatch training loss = 0.146 and inaccuracy of 0.12\n",
      "[7 0 6 2 5 4 3]\n",
      "[7 0 6 2 5 4 3]\n",
      "Iteration 190: with minibatch training loss = 0.165 and inaccuracy of 0.1\n",
      "[1 3 5 5 7 9]\n",
      "[1 3 5 7 9 0 0]\n",
      "Iteration 200: with minibatch training loss = 0.108 and inaccuracy of 0.14\n",
      "Epoch 6, Overall loss = 0.265 and inaccuracy of 0.126\n",
      "[1 3 5 5 7 9]\n",
      "[1 3 5 7 9 0 0]\n",
      "Iteration 210: with minibatch training loss = 0.0682 and inaccuracy of 0.13\n",
      "[2 5 0 0 0]\n",
      "[2 5 0 0 0 0 0]\n",
      "Iteration 220: with minibatch training loss = 0.147 and inaccuracy of 0.16\n",
      "[6 0 9 2 5 1]\n",
      "[6 0 9 2 5 1 0]\n",
      "Iteration 230: with minibatch training loss = 0.0724 and inaccuracy of 0.1\n",
      "Epoch 7, Overall loss = 0.137 and inaccuracy of 0.123\n",
      "[9 4 4 3 6 1]\n",
      "[9 4 3 6 1 0 0]\n",
      "Iteration 240: with minibatch training loss = 0.185 and inaccuracy of 0.14\n",
      "[9 4 4 3 6 1]\n",
      "[9 4 3 6 1 0 0]\n",
      "Iteration 250: with minibatch training loss = 0.0453 and inaccuracy of 0.17\n",
      "[5 8 4 1 0 7 7]\n",
      "[5 8 4 1 0 7 0]\n",
      "Iteration 260: with minibatch training loss = 0.0475 and inaccuracy of 0.1\n",
      "[5 8 4 1 0 7 7]\n",
      "[5 8 4 1 0 7 0]\n",
      "Iteration 270: with minibatch training loss = 0.118 and inaccuracy of 0.14\n",
      "Epoch 8, Overall loss = 0.0909 and inaccuracy of 0.122\n",
      "[8 8 6 4 3 9]\n",
      "[8 6 4 3 9 0 0]\n",
      "Iteration 280: with minibatch training loss = 0.046 and inaccuracy of 0.091\n",
      "[1 3 5 5 7 9]\n",
      "[1 3 5 7 9 0 0]\n",
      "Iteration 290: with minibatch training loss = 0.0335 and inaccuracy of 0.15\n",
      "[8 8 6 4 3 9]\n",
      "[8 6 4 3 9 0 0]\n",
      "Iteration 300: with minibatch training loss = 0.0332 and inaccuracy of 0.1\n",
      "Epoch 9, Overall loss = 0.0527 and inaccuracy of 0.122\n",
      "[9 4 4 3 6 1]\n",
      "[9 4 3 6 1 0 0]\n",
      "Iteration 310: with minibatch training loss = 0.099 and inaccuracy of 0.13\n",
      "[4 1 7 9 2 4 8]\n",
      "[4 1 7 9 2 4 8]\n",
      "Iteration 320: with minibatch training loss = 0.0386 and inaccuracy of 0.11\n",
      "[4 1 7 9 2 4 8]\n",
      "[4 1 7 9 2 4 8]\n",
      "Iteration 330: with minibatch training loss = 0.0329 and inaccuracy of 0.096\n",
      "[4 1 7 9 2 4 8]\n",
      "[4 1 7 9 2 4 8]\n",
      "Epoch 10, Overall loss = 0.0408 and inaccuracy of 0.121\n",
      "Validation\n",
      "Epoch 1, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "[5 8 4 1 0 7 7]\n",
      "[5 8 4 1 0 7 0]\n",
      "Epoch 2, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "Epoch 3, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "[7 0 6 2 5 4 3]\n",
      "[7 0 6 2 5 4 3]\n",
      "Epoch 4, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "Epoch 5, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "[8 8 6 4 3 9]\n",
      "[8 6 4 3 9 0 0]\n",
      "Epoch 6, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "Epoch 7, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "[4 1 7 9 2 4 8]\n",
      "[4 1 7 9 2 4 8]\n",
      "Epoch 8, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "Epoch 9, Overall loss = 1.08 and inaccuracy of 0.141\n",
      "[6 0 9 2 5 1]\n",
      "[6 0 9 2 5 1 0]\n",
      "Epoch 10, Overall loss = 1.08 and inaccuracy of 0.141\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "print('Training')\n",
    "run_model(sess,out,mean_loss,X_train,y_train,10,32,10,train_step)\n",
    "print('Validation')\n",
    "run_model(sess,out,mean_loss,X_val,y_val,10,32,10)\n",
    "\n",
    "sess.close()\n",
    "\n",
    "# with tf.Session() as sess:\n",
    "#     with tf.device(\"/gpu:0\"): #\"/cpu:0\" or \"/gpu:0\"\n",
    "#         tf.global_variables_initializer().run()\n",
    "#         run_model(sess,out,mean_loss,X_train,y_train,10,32,10,train_step)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}