2.gae.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Create Environments</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.multiprocessing_env import SubprocVecEnv\n",
    "\n",
    "num_envs = 16\n",
    "env_name = \"Pendulum-v0\"\n",
    "\n",
    "def make_env():\n",
    "    def _thunk():\n",
    "        env = gym.make(env_name)\n",
    "        return env\n",
    "\n",
    "    return _thunk\n",
    "\n",
    "envs = [make_env() for i in range(num_envs)]\n",
    "envs = SubprocVecEnv(envs)\n",
    "\n",
    "env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Neural Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.normal_(m.weight, mean=0., std=0.1)\n",
    "        nn.init.constant_(m.bias, 0.1)\n",
    "\n",
    "\n",
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0):\n",
    "        super(ActorCritic, self).__init__()\n",
    "        \n",
    "        self.critic = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, 1)\n",
    "        )\n",
    "        \n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, num_outputs),\n",
    "        )\n",
    "        self.log_std = nn.Parameter(torch.ones(1, num_outputs) * std)\n",
    "        \n",
    "        self.apply(init_weights)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        value = self.critic(x)\n",
    "        mu    = self.actor(x)\n",
    "        std   = self.log_std.exp().expand_as(mu)\n",
    "        dist  = Normal(mu, std)\n",
    "        return dist, value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()\n",
    "    \n",
    "def test_env(vis=False):\n",
    "    state = env.reset()\n",
    "    if vis: env.render()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        dist, _ = model(state)\n",
    "        next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0])\n",
    "        state = next_state\n",
    "        if vis: env.render()\n",
    "        total_reward += reward\n",
    "    return total_reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>High-Dimensional Continuous Control Using Generalized Advantage Estimation</h1>\n",
    "<h3><a href=\"https://arxiv.org/abs/1506.02438\">Arxiv</a></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_gae(next_value, rewards, masks, values, gamma=0.99, tau=0.95):\n",
    "    values = values + [next_value]\n",
    "    gae = 0\n",
    "    returns = []\n",
    "    for step in reversed(range(len(rewards))):\n",
    "        delta = rewards[step] + gamma * values[step + 1] * masks[step] - values[step]\n",
    "        gae = delta + gamma * tau * masks[step] * gae\n",
    "        returns.insert(0, gae + values[step])\n",
    "    return returns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_inputs  = envs.observation_space.shape[0]\n",
    "num_outputs = envs.action_space.shape[0]\n",
    "\n",
    "#Hyper params:\n",
    "hidden_size = 256\n",
    "lr          = 3e-2\n",
    "num_steps   = 20\n",
    "\n",
    "model = ActorCritic(num_inputs, num_outputs, hidden_size).to(device)\n",
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_frames   = 100000\n",
    "frame_idx    = 0\n",
    "test_rewards = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAE/CAYAAABLrsQiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXeYnFd5sH+f6WV7VVlJK8myZMtFtoWN6dgGm0AwoZMAdmgJJR8h/BICJCEkQPIF8kEKJZA4QCCAIYANMTY2GJtmbLlLtpplde1qd7ZOb+f3x1vmndmZnZndnd3Z3ee+rrl2521z5p2Z85ynK601giAIwurFtdQDEARBEJYWEQSCIAirHBEEgiAIqxwRBIIgCKscEQSCIAirHBEEgiAIqxwRBDWglNqulHpEKTWtlPo/Sz0eYelQSt2olPrFUo9DEBYSEQS18WfA3VrrVq31Py/1YEpRSn1RKXVAKZVXSt1YZv/7lFJDSqkppdRNSim/Y9+gUupupVRcKbVfKXXNYpy7WlFKfUopdchcVOxXSr25ZP9VSqmHzHt2RCn1Dse+tUqpW5VSp5VSWik1WMPrvVcp9bRSKqaUelIpda5j3x+Z+6aUUnuUUs9x7PuRUirqeKSVUo879v+tUupxpVRWKfXXJa+plFIfVkodN6/9TaVUm2P/l83rOa/vNvcNmu/Nue8v67h/bqXUx8x7NK2Uelgp1WHu8yulPm3uG1dKfU4p5XWc+zWl1BlzzAeVUm8rufbblFKHzTHdrpRa59jnV0p9QSk1rJQaU0r9QCm1vtZrLzlaa3lUeQB3AW+bZb97icf3buBqYA9wY8m+a4FhYCfQCfwM+HvH/l8D/w8IAq8CJoDeRp9b5/vzLNF9nfG5AjcCv5jHNT8K7MBYhF0BjAPPMvd5gUngDwAFPAOIAheb+/uBdwFXAhoYrPJabwMeA843r7cV6DL3XQHEgMvMfe8ERip9l83P768cz28AXgLcAvx1ybE3APuBDUCLecxXHPu/DHyswusMmu+t7Gc+2/0z938M+CmwyXxfFwABc99HgJ8DXUAvcB/wUce5OwG/+f8OYAi4zHz+AuCseYwP+Dxwj+PcPwMeNT+jAPBV4Lu1XLsZHks+gGZ/mF+qHJA0f5Tnml/kzwO3mT+ma4CXAg8DU8AJ54/D8eX+fXPfOPCH5g/9MYwJ9F9LXvctwJPmsXcAm2oY6y+YKQj+G/iE4/nVwJD5/7lACmh17P858IeNPLeG93Ej8Evg00AEc9KodE8wJod/Mf/3mp/JJ83nQfOzsybAb5s/wkngXmCn43XLfa7dwK3m53o/8LfMQxCUea+3Au83/+83vychx/4HgDeUnOOhiiDAmChPAFdX2P864H7H87B5zbVljh3E+A3MeD3ga8wUBN8B/tTx/FnmZxBy3Oc5CYIq968T4ze6tcKxe4DXOJ7/LnCiwrHbgTPAa83nnwI+69i/zhznVvP554F/cOx/KXCglms3w0NMQ1XQWl+FMcG9R2vdorU+aO76XeDjQCvGBBwD3gx0YHwJ3qmUekXJ5a4AtmH8CD8DfBhjstkJvFYp9XwApdT1wIeAV2KsXH4OfGOOb2EnxkrF4lGgXynVbe47orWeLtm/s8Hn1sIVwBGMyfHjVe7JPRgrNjCE6xDwPPP5lRg/yDHz+Y8wPoM+4CHg6yWvW/q5fhZjEluLIYje4jxYKfVDpdSf1/ieilBKBc3x7gPQWg+b7+n3TRPHlRgr27n4JAbMxwVKqROmCeijSinrN/8jwK2UusI0y7wFeATj3pXyZuDnWuuj9by9kv/9GPfd4l2mCeVBpdSrypx/TCl1Uin1n0qpnrIvUHL/gAuBLPBq0yR5UCn17irjGlBKtTuu+TmlVBxDozmDsSiodC4YGgfAfwDPVkqtU0qFgN/DuMfO8c527aVlqSXRcnhgqMVvczz/MvDVKud8Bvi0+f8gxuphvWN/BHid4/n/AH9s/v8j4K2OfS4gThWtgPIawVPAdY7nXnMsg8CbgPtKjv848OVGnlvD/b4ROF6yreI9obDq7wb+HENgnMQwS3wU+OcKr9Nhjqm93OcKuIEMsMOx7RMskEYAfAW4HVCObb+NYVLLmo+3lzmvFo3gWeYx/2u+z0HgoHU9jInsQ+b7ywKjwDMqXOtw6ffKsa+cRvA287UGgXaMVbsGrjT3X2p+Vh7gt4Bp4NnmvhZgt7mvH0O7uKOW+4chxDXGpBwELsIwd73I3P8xDE2zF1gD/IYyWpD5uT8H+AvAa267xrxHF5nX/jcgj6mtme/zm+b1shjWga4yY55x7WZ4iEYwd044n5grq7uVUiNKqUkM00/pSmbY8X+izPMW8/9NwD8ppSaUUhPAGMYPdz31EwXaHM+t/6fL7LP2W6v8Rp1bCydKnle8J1rrBIba/3wMTeAe4FfAs81t94DtSPx7pdRTSqkp4Kh5befn5HzdXowJybntWI3jx3QeWg7PD5Xs+yTGavK12pqZldqBMZm8GcMOvRP4M6XUS2t9TQcJ8+8/aK0ntLGa/zeMiRfgrRimSsvm/Ubgh04HqDmm52BMmt+p47VvwtBsfoaxWr/b3H4SQGv9kNY6orXOaq1vw9DKXmnui2qt95j7hoH3AC9WSrWWjGvG/XO857/RWie01o9h3E/rPX8cY4J+BOP78X0MQej8HaK1zmmtf4GhUb3T3HYXho/hfzC+N0cxvssnzdM+i6H1dGOY2b5LiUZQ6drNgAiCuaNLnv83xspng9a6HfgCxapkPZwA/kBr3eF4BLXWv5rDtfYBFzueXwwMa60j5r4tJT+yiymo2o06txZK72+1e3IPcBVwCYZd/R4Mh/XlGL4AMFaM12Os7toxVqxQ/Dk5X3cEY3W3wbFtY43jR2v9h9owJ7ZorT9hbVdKfRTD0fpirfWU45QLgINa6zu01nmt9QGMFf1Lan1NBweANMXvx/n/LuCHWuuD5mvdjmGueFbJdW7AcHpGa31h83of0VoPaq0HML4Lp8xH2VOo/FuxxmzPVbPcv8dKzin63xQO79Far9dab8HQyh/UWucrvLYHw8Funf9ZrfU2rXU/hkDwAHvN3bswtOExrXUK+Bfg8kpmrdJrLzlLrZIshwflTUMfKznmLHCD+f/l5vOvmc8HKXGAYawkXuB4/jXgL8z/fwfjC7ZTF9TO18wyPh9GpMIvgbeb/7vMfddh2H3PxzAR/JTiyJ/7MBxhAfN1nZE/DTu3yv2+kRLzS7V7ArwYw6H7E/P5TvP5Pscx78JYDbZhrNo+Z34u58zyuX4LY1UZMt/LydKx1fld+iBwCFhTZt9WDE3qKgpRPoeBdziOCVBw7G7HjIip8FpfBX6I4e8YwLBNv9XcdwOG+WaL+VovwjC1Oc1gQQyn+lVlru01x/LfGCaXAGbEEUZUzlbzuuebn5vzPbwaQ/t1mZ/bNOZvAcM3tN3c123e/7truX/m/nsxNB8/cB7G7/Bqc996DCevAp6Jsbh4sbmvD3i9OS43xiIiBrzccd8vMM/diDEnOIMh/hNDOLSb9+ZDwKlart0MjyUfwHJ4UJsgeDWG2WDa/PH9K3MUBObzNwGPU4hCuqnK+HTJw3ntP8FQf6fML6zfsW/QPD+BsYq8puTajTp3H/B7Fd7PjZSZbGe7J+aPLAN8xHyuMCaBz5ccc4v5GR3DMMFUEwS95udZNmoIQ/3/UB3fJY0RbRV1PD7k2P9ajInTMjv8X0yh7ji/6OHY9wXgC47nbRhCbNq8X39FwZ6ugL8Bjpv7nwTeVDLWN5j3SZV5H18uM5YbzX3nmt+HuHn+n5Sc+3MMATOFEUTw+pLXfBpjojyDIczW1HH/1mP4DaIYwQZ/4Nj3PAyTTtwc3+859vViaJET5rgex+GfwVjMPGaOawj4OxyhthhC6+sY37kJDH/d5bVcuxke1pdCEARBWKWIj0AQBGGVI4JAEARhlSOCQBAEYZUjgkAQBGGVI4JAEARhleNZ6gHMl56eHj04OLjUwxAEQWg6HnzwwVGtdW+145a9IBgcHGTPnj1LPQxBEISmQylVU0kUMQ0JgiCsckQQCIIgrHJEEAiCIKxyRBAIgiCsckQQCIIgrHJEEAiCIKxyRBAIgiCsckQQCIIgrHJEEAiCIKxyRBAIgiAsAacmEjx5Zqr6gYuACAJBEIQl4B/vOMDvfuk+Mrn8Ug9FBIEgCMJSEE1lGY9n+MWh0aUeiggCQRCEpSBtagI/ePT0Eo9EBIEgCMKSYJmE7tg3RDKTW9KxiCAQBEFYAtLZPEGvm1g6x0/3n13SsYggEARBqMLX7jvGf/366IJeM53T7B7spKfFv+TmIREEgiAIVfjWAyf4j188vaDXzGTz+D1uXnrhGn6y/yzTycyCXr8eGiYIlFKfVErtV0o9ppT6nlKqw7Hvg0qpw0qpA0qpax3brzO3HVZK/XmjxiYIglAPY7E0x8bixFLZBbtmOpfH73Hx8l3rSGfz3PnE8IJdu14aqRHcCVygtb4IOAh8EEApdT7wemAncB3wOaWUWynlBj4LvAQ4H3iDeawgCMKSEoml0BoODk8v2DUzuTxet+LSjZ2s7why6xKahxomCLTWP9ZaW+LzPmDA/P964Jta65TW+mngMHC5+TistT6itU4D3zSPFQRBWDLi6SzJjBHhs39o4QRBOpvH63ahlOK523p4/OTkgl27XhbLR/AW4Efm/+uBE459J81tlbbPQCn1DqXUHqXUnpGRkQYMVxCExeYff3yAXx5e+uSqUsZiafv//QtYEiKTy+PzGFPw2vYgkViaVHZpwkjnJQiUUncppfaWeVzvOObDQBb4+nwHa6G1/qLWerfWendvb+9CXVYQhFn40eNniERTDbm21pov3PMUtz1+piHXnw9FgmABNYKUqREArG0PAHB2qjH3txqe+Zystb5mtv1KqRuBlwFXa621ufkUsMFx2IC5jVm2C4KwhAxPJXnn1x/iA9ft4J0v2Lrg109kcmRymkR6aROryhExBcF5a9vYPzSN1hql1LyvmzGdxQBrTEFwZjLJhq7QvK9dL42MGroO+DPg5VrruGPXrcDrlVJ+pdRmYBtwP/AAsE0ptVkp5cNwKN/aqPEJglA7B8yV8Hg8XeXIuTGZMEInE0ucYVuOsajxnp+1tZvJRIahqeSCXDft0AgsQbBQ166XRvoI/hVoBe5USj2ilPoCgNZ6H3Az8ARwO/BurXXOdCy/B7gDeBK42TxWEIQlxoqWGY/NLgjmWknTEgTxJtQILNPQs8/pBmD/mYJ56Ev3HuFzPztc9zVzeU1eY/sIbEEwmZjvcOdEI6OGztFab9Ba7zIff+jY93Gt9Vat9Xat9Y8c22/TWp9r7vt4o8YmCEJ9WIJgIlE56enoaIwrPvETvnH/8bqvPxk3NYImFASRWBqvW3HZpi4AnhwyHMbRVJZP33WQWx6uP+zTEpiWRtDq9xDyuRmaXBofgWQWC8Iy44Bpp17U1xyOAoUJu5RkJse7vv4QY7E0RyOxuq9vCZh4ZuESthaK8ViazpCP9qCX9R1BWyO47bEzxNM5JhL1m8tSWUMQWBqBUoo17QGGplaYRiAIwsJz+Ow0137mXu4+sHhFyvJ5zeHh2X0EH/3BPp44M4XbpeaUfWv7COrQCG7fe4a9p2bG3t++d4iHj4/P2H7znhPcsW+o7rFFYmm6wj4AdqxpZb+pEdy8x4h2H49n6hbMlkbgcxeczmvbA5yZXHk+AkEQFpijo0bcxZNnFi6MsRqnJhLE0jl8bldZ09B3HzrJN+4/wTtfsJW17QHiqfrNO1NzEAR/8f19fOGep2Zs/+gP9vGvP51pt//MnQd5/82Pcna6vsl2LJaiu8UUBGtbeWokxpNnpthzbJy+Vj/pbL5uJ3c6W2waAuhvCzAsgkAQhGpYUSVPj9ZvfnGSyeX5y+/vrek6ln9g14YOJktWv5lcnr+6ZR+Xb+7i/S86l7DPQyw9d40gXuOEms9rxuPpGXH3+bzm7HSKk+PFJpZ0Ns/QVJJoKssnbz9Q19jGTNMQwI41beTymr//0X7cLsUbn7kJMLSCerA1Ak9hCl7bHmB4OkUuv7hmPxBBIAjLirOmIDg6T0Gw/8w0/3XfMb73cPVUnYOmf+AZmztJ5/JFkT1jsTTRVJaXX7wOj9tFyO8mNgeNoN6ooclEhlxeM1yyuo/E0uTympPj8SKBNTSZJK9hQ1eQbz94kkdPTNQ8tkgsTbdpGjpvbSsA9xwc4YXb+zi3vwWAiTrDastpBGvaAuTyumFJe7MhgkAQmhCtNb88PEq+ZHU4bK6A56sRHDBX+eVs7KUcHJ5mbXuADZ1GopPTPDRqTlo9pumkxT83jWDCXFGns/maVsRWktfwVLJowh82BWUsnStapZ8cN0xqf/nS8+lp8fPXP9hX1q4fS2WLfByZXJ7pZJausB+Awe6wvYp/7e4BOkxNYaJOjSBdRiNY0x4EWBI/gQgCQWggc60x/4vDo/zev/+Gew4V19KyTEORWLpiBE8tHDAdno+XEQQ/O3CWqGMyPDA0zbn9rY5Jr7D6jZjJVt0txkQZ8rnn5COYdAiXWuzt1qo5mckzlSyM1Wn/tyZ/gBPm/+etbePPrtvOw8cn+PKvjhYJ2lsfPc2Vf/cT/vQ7j9rbrLyJLlPQedwutvW10NPi44U7+ugIeY3j6tQIMjnjdX3uYtMQiCAQhBXFr54aZdff3MmhOZQu/vkho/jasZKV//BU0l5FPu0I0xyLpbnuM/dy78HaijBa4aAj0ynb3ASGyenG/3yAT96+HzASnw6PRNm+ptWe9JwCKBIzJmTLdBL2eYqESK04BUG8Bo3CWf/HOf5hh8/A6Sc4OZ7A7VKsbQ/w6ksHuHywi4/+4Amu/cy9fOuB4/zxNx/m/3zjYWLpHI85qoBamof1/gA++vKd/MsbLsXrdtm+g3p9BOnsTI2gvy1gvgcRBIKwYvjBo2fI5TUPHpsZyliNX5iCoNTpOTyV5NKNRo+np0ej9vb7jkTYPzTN+771SNHEmMzk+Ppvjs2YnA8OTbPRrGnj1Ap+9VQEgJv3nGQ8luZYJEY6m2dbX4tj9esQBKZG0NNqaARhv6emibyUKadGUIOfIOIQBM7J3zmJOjWCk+MJ1rQF8LhduFyKr7/9Cj79uotxuxQf+J/H+cFjZ/iTF53LO5+/lVMTCbsKqCVwrAkfYPdgF1duNbKMC8KxXo1gpo+gO+zD61aiEQjCSiGf19z1pNFx6ok6SxdHoin7HKcgSGUNu/flg124FDw9WpjoHjkxgdetiKdzvO/mR8jnNZOJDG++6X4+/L29fO2+Y/axk3GjXs4rdq1DqWJB8OsjEVr8HhKZHP913zE7Ymj7mlZ7MnQmUI1EU/jcLlr9Rv3KkN9oxl4vk4mMHatfi8N4rEgQFGsEPS1+2gIeTow5NYI4A51B+7nX7eJ3LhngR+99Ljf/wZXc/t7n8n+u3sY5fS1oDccjxr21NYKWgiBw4ve4CfncC6IRuFyKvtaAaASCsFJ47NQkI9Mp3C7FE6crC4LjkTg33HR/0erVWpX3tPg5OVHYboVKDnSGGOgMFTmMHzk+wc517fz1y8/nl4cj/P3t+3ndv/2ah4+P0x708psjEftYy1F8ycZOtva22A5jrTX3HYlw9Xl9XLWjj6/86iiPnpxEKTinr4X2oLH6nSjRCLpbfHY1zrDPQzqbr6vmkNaaiUSGNaZppFZBYFXudEYOnZ1K0t/mZ6AzNEMjGOicWdVTKcXlm7vY1m9EAw32hIGCM972EYTLCwKAjqC3bh9B2tYIiquYGkllBQF2YGiaJ89MNTyTXASBIDSAu54Yxu1S/PZFa3nyzNSM6B8wJsC/uGUv9xwc4Uv3HrG3//LwKK0BDy86v69II7BWin1tfjb3hG3TUDaX5/FTk+za0MFrd2/gZRet5Yv3HuH4WJybbnwGv3XhWvYcHbejcSxBcO6aVi5c387eU4agemokxsh0iiu3dPOO520hEkvzlV8dZUNniJDPQ8DrJuB1lTiLU0Wr5bCpGdTjMI6lc+TymnUdhiBI1uIsjqVZ2x6gNeApyiUYnk7S3xZgQ1fQvndWDoFTI6jE5u5iQRCJpVHKmOwr0RHy1R81ZGoEfk/xFLymPcCQwzT0r3cf5o3//hsanVoggkAQGsCdTwyze1Mnz9raQyyds6NWnNy+d4h7D47Q1+rn5j0n7WStnx8a5Vlbu9nYFWYinrHt+5YtfE17wBAEIzG01hwcjpLI5Ni1oQOlFJ945YW86Zmb+OY7nslzt/XyzC1dTKeytmZycGiaVr+Hde0Bdq5rY2gqych0il+bWsMzt3RzxeYuLh5oJ57Oca65WgboCBZPekaMvd9+Hva5AeoKIbUcxVYFzto0ghRdYZ+RjVtiGipoBAm01pyeSKA1NQmC9pCX7rDPrpc0FkvRHvTicVeeKjvD3rrzCMr5CMDIJRgyQ2JT2Rx37z/LNef143bNv//BbIggEIQF5ngkzoHhaV50fj/nrW0DmGEeiqayfPQHT3De2jZuuvEZJDI5vvHAcY6PxTk1keA55/TYE9cpc2VrhY72twbY0hsmls4xEk3xiJkctWuD4URuC3j521dcwEUDxvNnbjEcm7952pjoDwxPc+6aVpRSXLi+HTDyCe57KsLa9gCbukMopXjH84wGNNvXtNjj7gh5i/IIItE0PS0FQRAyNYJ66g1ZUUgF01D1cyPRNF1hP/1tflsQZHN5RqMp+loDDHQGSWRyRGJpWzOoteHLYE+YIyOWIEjPahaC+WkEMwRBe4BkJs9kIsOvnooQTWW59oL+uq49F0QQCMICc6fpJH7R+f1s628x/AQlDuN//skhhqaSfOwVF3DB+naefU43X/7lUX52wAj/fLZDEFi27rNm6GhHyMugZcIYifHIiXE6Q142dZef6PrbAgx2h7jvyJipQUzbq/ydpiB47OQk9x2JcOWWbtvef90Fa3j3C7fyyksH7Gt1hAqrX601I9GUnUwG0OK3NILaTUMFjcB4v7VEDY2Z2b79rQFbUxqNptHaeL9W8tvJ8YR9/2rRCAA294RtjSASTReFjpZjLj6CciUmoLhBzY/3DdHi9/CsrT11XXsuiCAQhBLyeT2vzN27nhhmW18Lm7rDBLxuzultKdIIjoxE+Y9fPM3rn7GByzZ1AvC252xhaCrJZ+46yFrT9DPgmMzAmBz62/wopdjscGo+cmKCi02zUCWu2NzNA0fHGJpKMhHPsGONIQha/B629IS55ZFTRGJpnmmGRQK4XYo/vXYHW3sdGoHDNBRNZUln80U+gpBvDhqBGYVkJVRVSyjT2qgz1NXio68twNlpw5RiaQb9bX4GugpC1MohsDSOamzuCTM8lSKWytakEXSGfEwmMmX9QJVImwllpRqBdQ9OTyS484lhXrC9l4DXXfN154oIAkEo4X8fP8PV//gzTk/UXxt+Mp7h/qNjvOj8gjp//rq2Io3gG/cfRwHvf/F2e9vzz+1la2+Y8XiGZ5/Tg1KKnhYffo/LXtEOTyXtyWxdRxCfx8XjpyY5dDZqm4UqccWWLiYTGW59xGii4rT7X7C+nSOm4LtyS3fZ8y06wwXTkJ1V7PARtMzFNGReb22NPoKpZJZMThsaQZufTE4zHs84BEHAFqInxgyNYG17YFY7vxNLyB6NxAyB43h/5egIeclrmE7W/p4rO4sNAfbDx84wGk1z7c41NV9zPoggEIQS9g9Nkddw6Gy0+sEl3H3gLLm85urzHIJgbRtnJpOMxdKks3m++9Aprjmvn97WwgTjcine8pzNADznHMMUoJRifWeQU6ZAGp5K0WcKArdLMdgd4rbHz6A1NQgCY4L/2m+MfAKrWBpg+wkGOoNV7ejtQR8T8TRa60JWcZFGYKxe62k5aQmC3lY/LlXdNGSVl7CcxWAIyeFpY3tfq58Wv4fOkNfWCGo1CwG22e2pkRjj8Qxd4coRQ4Aju7h281AlZ3Ffqx+l4IePnsHndvGC7b01X3M+eBblVQRhGXHcTEQ6FokB9f0Q73pymJ4WH5c4Jubz1xkO4yfPTDGdzBCJpXndMzbMOPe1uzcQ8Lj5rQvX2tuc0S/DU0mu2tFn7xvsDtuVQasJgvUdQQY6g5wYS9DT4rdrA4GhEUB1bQCM1W8mp4mnc4xaWcWOa1nho/WUmZhMZHC7FC1+DyGfp6oQGXPE9rcGjNcbnkpydiqJSxXqHln37uR4gudsq93OPthjCMPHTkyQy+uaNAIwBMEg4ZpeI53N43apGdFAXreLnhY/I9MpXri9l9bA7EJooRCNQBBKOG46Co+Ozgz5nI10Ns89B0e4akcfLscP3IocevLMFDfvOUl/m5/nlpmYvG4Xr7psoMiBONBpxMNHU1ni6Rz9bYVJaXOvMels7gnbBeFm44rNxkRv+QcsLhpoZ2tvmN++eF3Va3Sak95EIuOoPDpTENRTZmIykaE96EUpRcDrJlGlXaWV7dvT4qev1dAIzk6lGJ5K0tvqtyfXgc4gT41EGZ6uLYfAIuTzsKYtwB6zNEhVZ7GdcV175FAml5+RTGZhmf8WyywEIgiEFcS+05P85ff31pXVWo7jY4YAODZL793TEwn+v28/ypSjuugDR8eYTmaLzEJgrFzXtAX46f6z/OzAWV516UDN9uqBziBjsbTtvO53ODy3mLbsatqAxRVbjObrTv8AGJP3T97/Ap53bnXtpz1YqEBq+QicztSg6dispyfBRDxjZy2HfO6qpiGnRtBnCsbhqaSZQ1C4Pxu6LG0KO4qoVjb3hNl3etJ+ndmwhWMdpqFUNl9UedTJmvYALgXXnN/4sFELEQTCiuFnB0b4r/uO8c37j8/5GlPJjF03ZrYm7L84PMp3HjzJV3911N5215PD+Dyusqv989e18aunIuS1YQKqlfUdxkrWKlznnOg29xh2/osH2mu61rPP6cHrVuzaWJvgKEdHqFBmIhJN0RbwFGkwbpci6HXX7SxucwiCekxDfo+bzpCX4ekkw1NJW0OA4nDRejQCMHIJrFLRtUQNAYzH6tMISkNHLV5z2QDvvfrcIk2r0YggEFYMljnin35yaE6lkKFQbGxzT5j0C89rAAAgAElEQVQTY4mKTVIss8h//vIoiXQOrY0ic8/e2m2HUDo53zQPXb65y65nUwtW9IslCJwhkJds7OCPrjqH63etr+la6zuC/OIDV/HbF62tfnAFOh2NWEZj6bKTVdjvqSuPYCpR0AiCPnfV8NFINE3Y57bDKo3s4hRnp1NFprMiQVBjMpnFFsdnVE0QtAW9KFWfRpDO5mc4ii1evHMN771mW83XWghEEAgrhng6h0sZiUVfdNTuqQfLLPTcbT2kc/miAmBORqeNH30klubbD57g0NkoJ8YSFdX5nabDuB5tAGCDOZk9ZAqCPsdE53W7eP+Lt9NZZaJy0t8WmDXfoBq2RpBIE4mmKggC95x8BGCYlqprBCm7UQxAX1uAE2NxxmLpYtOQKUQ9LkV/a32r68E6BIHbpWgLeOv2EVTSCJaC5hmJIMyTRDpHT4ufl120li/de6SoLn+tHItYgqC36Hkpo9EUG7tCXLapk3+75wi37x0C4Ood5QXB1ef186nXXMwrdlV3yDrpafHj87g4PZmkNeApq20sJs4KpKNm5dFSQj5P3aahjjp8BJFYcWx/f6ufw2aor1MjWG8K0bUdtecQWFi5BE7NYzY6Q966SlFncrqiRrAUNM9IBGGexNM5Qj43f3rtdrL5PJ++61Dd1zg+FqMr7OOC9cYKvpKfYNQsrfCuFxiNTD7/s6e4YH2bXSKgFJ/Hxasvq91JbOFyKQZMP0GtmbGNJOB1E/S6TWdxqqwgaKmjgb3VN6FgGvJUNQ1Z5SUs+tsCZE0TXp/jHoV8HrrDPgY66jMLAWzsCuFSFGkes2HUG1oYZ/FS0DwjEYR5Ek/nCPo8bOoO84bLN3LznhN1rUzBMA1t7ArR3xrA73HNqhH0tPh54fY+tve3ksjkuOa8xkR5WCvb/iYQBGCYhyLRNOPxTFFWsYWRC1DbfY+ms+R1QdMIeaublUrLPji1gP7W4nv05isHedVlA9SLz+NioDNEVw1huWDck3oTyrxiGhKEhSeZydmZrTvXtZHL67rstmCYgjZ1h3C5FJu6QxytUHNoNJqmp9WPy6V4z1Xn4FJGkbZGMNBkgqA96OUp8770lLG9h/3ump31VuVRp7N4Nh+BkdFcrBE4tQCnUAB47zXbePUcBAHA656xoabcCjCc6JUqkD50fJy7D5wt2pbO5vE3kUYgmcXCiiGezto2dCsjczqZAWoLHUxn85yeSPA7lxhROJu6w2U1gmwuz3i8EDHz2xev41lbu4uydRcSK3KodJJbKjpDPrurWU8ZR2q4huxgC6u8RJszamiWc61Cd10lpiEwun111riCr4V3v/Ccmo81qrKWFwSfvvMgw1NJXri9kBWeyeXxe5tHEDTPSARhnhimIUMjsEoPTCVqNw2dnkiQ19hN3Qe7QxyNxGZUlRyLGeWOex3240YJAShoBJX8D4tNR8jLtLniL/e+w/7ancVW03orGinkdZPN64pJgWNlWkdaArKvNVCU0b2YdIZ8tpAq5dREYoZgTOfERyAIDSHhMA0VawS1ccwMHbUEwabuMKlsvqgnLhgN24FFS/jZZBZBs5LLlhpr0obyTd1DPqOBfS19di3TndM0BJWL1pVrJt/TYhRq61tCjcnKLp4sMUVqrTkzkZyh5cyWR7AUNM9IBGGeWFFDUNAI6ikNbNUYsiZeqwplac0hu9hanbHpc+XigXa+8pbLeYHDtLCUWGUmAHrKOIvDfg+5vCZVZnVcymSJILBMe5XMQ2N2WYvifIrusJ++Rfo8ytEeKpTecDIRz5DI5GZEQjWbs1h8BMKKIZnOEfRaPgJLENSuERwfi+P3uOwJxer4dSwS40pHw5bR6cXVCJRSPL+GOkCLhbX69boVbcGZU0jYsaqvFoM/UxBY55YX4JZpqLQQ3F++7Ly6y0gsJJ12BdLi75tVQjyRMTQkK5kvnWsuZ3HDR6KUer9SSiulesznSin1z0qpw0qpx5RSlzqOvUEpdch83NDosQkrB6018UyOoM/4SreZpqGpOjSCY5E4G7pCtp15XUcQr1txNFKqEViCYOEck8sJyzTUHfaXzVIO19GcZjKRwetWtgCwBEc9piGA63et57JNXTW+g4WnUk8Cq7mR1hRpSJlscyWUNVQjUEptAF4MOKuAvQTYZj6uAD4PXKGU6gI+AuwGNPCgUupWrfV4I8corAzSuTy5vLZNCwGvG5/bVZ9paCzOJkdNGrdLsaErNKMK6Wg0hd/jsrtxrTYs01A5/wA4BEENuQTOEtRQ0AiSFZLKxmIpAl7XkmdYl2IJx8kSjeDMZMG/lMwUNKT0Kisx8WngzzAmdovrga9qg/uADqXUWuBa4E6t9Zg5+d8JXNfg8QkrBMumHHSYIloDnqIy0bOhteb4WHxGh67B7nAZjSBtOiiXJkJlqbHMIJUipazJvJbsYmflUee5s2kE5ZLYlpqOKhoBFPdizqwWZ7FS6nrglNb60ZJd64ETjucnzW2Vtpe79juUUnuUUntGRkYWcNTCcsWaOKyJBAxBUKtGMBpNE0/nbL+AxaZuQyNwRsCMRlOL5ihuRqxJr1wOAdTXt3jS0YsAqkcN1dJMfikI+9x43aqijwCK31OqyTSCeelXSqm7gHLplB8GPoRhFlpwtNZfBL4IsHv37uoxasKKx/qRBYsEgbdmZ/HxMStiaKZGEE/nGJku9AsemU4tqWNyqbHMIJWEoWW2qaXMxGQiU2RisjS6Sl3KmlUQKKXK1hsq0gjM76jWRp6Er0KHsqVgXiJJa32N1vqC0gdwBNgMPKqUOgoMAA8ppdYApwBnLd4Bc1ul7YJQFcumXGoaqlUjOGH2KS7tZHVOn9H8xeoNDAXT0GqlM+SjO+xje0mnM4uwvz7TUEeRacgKHy0fehqJpqu2jlwqOstkF5+ZTNpRaNZ3NJvXaE1TaQQNGYnW+nGtdZ/WelBrPYhh5rlUaz0E3Aq82YweeiYwqbU+A9wBvFgp1amU6sTQJu5oxPiElUfBNFRQcg1BUJtGYEUC9ZUULbP6++4fmgIgl9eMxcrX4V8t+Dwu7vvQ1bzy0vINcepxFk/E0xVMQzPPNeoMla942gx0BH1FPoJMLs/wVJKtvcZiwvIRWFnTq8JHMAu3YWgMh4EvAe8C0FqPAX8LPGA+/sbcJghVsSYOp2moLeCtWSMYj6dxqUL+gUV3i5Go9OSZafu4vF69oaMWXrerorM87LN8BLNrBNFUlqlklv52Z+lo0zRUxkcQT+dIZvINLecxHwY6gzw9WvAnDU8lyeuCVmm9p0zW2N9MgmBRYrBMrcD6XwPvrnDcTcBNizEmYWWRKOssrkcQZOgI+crWqtmxts3WCOwcglXsLK5GwOvCpar7CKywXCuDG4zJ0etWxMuEj44ucmmPerl4QwffffgUZyaTrOsI2qGjW3uN92dpBKmc8XfFm4YEYbGpFDUUTWUr9h12MhFPF9XQcXLemlYODUfJ5PJ2i8pmnYyaAaUUYZ+nailqqz/0xpKQ3YC3fAVSq7RHs5qGdm3oAOCRExNAwVG8pbdEI8gZ30cpOicIC0yigrMYIFqDVjAey1QsYXze2jbSuTxPj8aaflXaLIT8buJVTENWkb/SSK1K7Soj1r1vwjwCML4nPo/LFgSnbEFQrBFYFUpFIxCEBSZRJny0UGaiusN4PJ62E6VK2bHWcBg/eWbKFgS9IghmJezzVHUWH4vE6Qr77EqxFiGfp4JpyCr215wagc/jYue6Nh45XtAIOkJee9EgzmJBWEBS2VxRog5UjhqC2iqQTpg+gnJs6WnB61bsH5pmJJrC53aVLbYmFKilJ8GxSGyGWQgMrS5RRohYGkEz5hFY7NrQweOnJsnm8pyZSLKuPYjf40IpoygiiEYgCAvC1+87zrWfvreoeUk8k8XnceF2OHvr6Ukwm0bg87jY2ttiaATTabpbfKu2vEStWD0JZuNYJM5g90xBEKrQrjISS9Ma8OD3zF7RdCnZtaGDRCbHgeFpTk0kWNcRQCllCDfLNGRrBM3zHRJBICw7hqaSRFPZoiYgCUcvAgtr1V5NI0ikc6Sy+YoaAcD5a9vYf2bablovzE6Lf/YG9ulsnjOTCTY6IoYsgj73jPr9YEQNNbtJ7pINnYDhMD49kWCd2Uwo6C0IN9EIBGEBsKJRpkoEQbCk9r2tEaRm1wgmEobtebZ+tzvWtjI0leTw2eiqzyGohZDfM2sewcnxOHlNUbVXi2DFqKHmTSaz2NAVpCvs45eHR5lKZm1BEHBoBJYmK1FDgjAPrCggp0Zg9CIoFQS19S0ejxnXqWQaAtixpg0wIkFEI6hO2Oee1UdQKWIIZjENRZuz8qgTpRS7NnRw936jGKatEfjcdokJWxCIRiAIc8eaYKqZhmrtUmYVCpvNNGRFDoEkk9VC2O+pWEEUHDkEZQRB0Ff+3Egs3bQRQ04sPwHAOjNr2qnlWKYhiRoShHkQLSMI4uksIW9xJI/f48bnqd6cxiod3BmurBH0tQZsk5BoBNUJ+9zE0tmKDeyPRmKEfO6yNn8jj6D4M8vm8ozHm18jgEJiGRRrBAVncfOVmGiekQhCjVT0EfhmRpO0BTxV21VahcJm8xFAwTwkPoLqhPwetKas0xcMjWBjV6hs9JUVYeMUImPxNHqZ1Hi62BQEbpeyK48a78nQBCyNwC+mIUGYO5ZpyDnBx8s4i6G2ngQF01BljQAKlUibPXKlGSj0LS4vCI6Nxcv6B8BYPedLevxGosuntEd70MuW3jBr2gJ4zFW/MzeiGRPKJCtGWHaUMw0lMjN9BGBoBLWYhkI+d9X4dGult34VN6WplbDdrjJLb4lPJZ832oJetaOv7LnOCqRWj9+IXWeo+QUBwBuv2FRUkrrINNSE4aMiCIRlhy0I4tVNQ60Bb9USE+PxdFFzlEq89MK1bOkNs6lM7LtQjJXhXa7MxPB0knQ2Xzar2DjX7EmQydFpbrNKezR7+KjFW56zuei5UUjPEAAZSSgThPmRzeVJmrbWYmdxeY2gli5ls5WXcOJyKXaua69zxKsTq29xueifo6OVQ0cBWwtwOoyXe7G/oLcQPpqW8FFBmB9Om7O10s/nNYlMjqBvpoJbS5ey8Xh61oghoX5CZrvKcqWorf7QgxU0q3LtKiOxNF63oi2wPI0YIV/BAW6Hj7qaZ/ptnpEIQg1EHatESyNIZmeWoLaopTlNrRqBUDtWl7JypaiPReJ4XIq17YEZ+8BhGnJqBNMpusP+ZVvjKehzk8trMjmjcb3Xrco2QVoqRBAIyworq9jvcdmCoFx3MovWgJGclM2Vb4YOsxecE+aG3cC+jI/g2Ficgc6gHVFTStDhI7CIxNLLxj9QjoK5K0c6m2+qiCEQQSAsMyxTw7qOoC0I4mV6EVhY9YYqdcvK5zWTicpNaYS5UehbXMY0FImXLTZnUa5vcWSZF/uztNVEJkcmp0UQCMJ8KAiCANNJow2lFZZXKXwUKlcgnUpm0Hr28hJC/Vg+glJnsdaaY5FY2WJzFkHvTEEwGl3eGkHQZ0y1iYxR6baZHMUggkBYZlgrzPVm6n40mS3br9jC0gicEUZO7PISYhpaUPweN2Gfm5HpVNH28XiGqWS2YsQQzDQNaa2Xfflvp3DL5PJNVXkURBAIywynaQiMCd5yKgbKOIuraQS1lpcQ6mdTd5ijkVjRNuv55p7ZTENW1JDxmcXMfhHLobxEJayItkTG8BGIRiAI88ByFjsFQaJMm0qLal3Kai0vIdTP5p4wR0dLBIH5fLakPGv1bGl6VovK5VBwrhKlGkEzJZOBCAJhmVFqGppMZGb1EVTrW1zoRbB8V5vNymBPiBPjiaKWokcjcVzKaOBSCbdL4fe4bAG/3LKKy1HsLBaNQBDmRTRl9Ca2JgXDNDRbHsHsPQnENNQ4Nve0kMtrTo4n7G3HIjHWdQSr1nVqDXg5OWGcN7qMCs5VotRZLFFDgjAPoqksrX4P7WZtoKlkpkoegWUaKq8RTMQzuFRBYAgLx+YewyHsNA8dHY3N6h+weNlFa/nxviHOTiWXVeXRSlj+q6Q4iwVh/sRSWcIOQVCkEZQRBD6Pi4DXxXSFPILxeJqOkK+psjxXClYJiaedgiBSufy0k99/9iDZvOarvz5mm4a6wstXawuJs1gQFo6oKQiCXjdetzKdxWbUUAVzQ2vAW9TExolRXkIcxY2gK+yjNeCxBcF4LM1kIlOxxpCTTd1hXnReP1//zTFOjSdoD3qbbvKsB0koE4QFxDINKaVoC3htZ3HQ6664qp+tAqlRXmL5rjSbGaWUETlkhoxaf2sRBABvfc5mxuMZbnn01LJ2FEOhG1ncLDEhpiFBmAeGRmCsrtqDxkq/Uglqi9l6EozHM5JM1kAGu8O2RmALghp8BACXb+7igvVtJDN5epZx6CgYJcwDXhdJM2rI22TaTXONRhCqEEvlaDEdwG1Br51HUM4/YDFbl7LJeJr24PJebTYzm3vCnJ5IkMrmODpaPXTUiVKKt5oNXnpal/9nZLSrNEtMiEYgCHNnOpmlpYxGUC501GK2ngSiETSWzT1h8hpOjMU5WmPoqJOXXriOwe4Q5/S1NnCUi4PRwN7KI2iu4ASJmROWFbFU1u5+1Rb0ciwSoz3km9U01FahJ0EykyORydG5jKNRmh3LDPT0aJyjkXjN/gELn8fFj9/3/KbLxJ0LVt/iVRc+qpT6I6XUfqXUPqXUPzi2f1ApdVgpdUApda1j+3XmtsNKqT9v5NiE5YdVaTRsCoL2oIfJRIZkFdNQa8BT1kcwYRack6ihxrHZDiGNcnQ0xmBP9dDRUnwe17JtSOMk6HOTbNJ+BA3TCJRSLwSuBy7WWqeUUn3m9vOB1wM7gXXAXUqpc83TPgu8CDgJPKCUulVr/USjxigsL6yCcy22IPAylcwSz2Tpay3f7QoMZ3EykzdrvBR+gJJV3HjaQ146Q14eOTFRc+joSiXodRNPG+GjzRYK28jRvBP4e611CkBrfdbcfj3wTa11Smv9NHAYuNx8HNZaH9Fap4FvmscKAlCoM+QUBLm8ZnQ6PatG0N9mRJzcsW+oaPu4FJxbFAZ7wvz80Cgwe7G5lU7A6yaeyZHONZ9G0MjRnAs8Vyn1G6XUPUqpZ5jb1wMnHMedNLdV2j4DpdQ7lFJ7lFJ7RkZGGjB0oRmxNIKwQxAAnJ1OzuosfsUl67l0Ywd/+u3H2Hd60t4+EZeCc4vB5p6w7aPZPAfT0Eoh6HXbQQsrSiNQSt2llNpb5nE9htmpC3gm8KfAzWqBDH1a6y9qrXdrrXf39vYuxCWFReD0RIJfPTU65/Nt05BZF6jNDCPN6/J1hiz8HjdfeNNldIS8vOOrDzIyneK2x8/wqTsO4HEp+lqXd4x6s2P5CZSCgc5VLAh8bjvDvdmcxfPyEWitr6m0Tyn1TuC7WmsN3K+UygM9wClgg+PQAXMbs2wXVgD/dNch7nhiiEf+6sVzOr+cachiNtMQQF9rgC+9eTev/sKveP4n7yaeznFOXwv/9qbL6F7GxcyWA1bk0Lr2YNnmQauFkM9td8prtiioRoql7wMvBDCdwT5gFLgVeL1Syq+U2gxsA+4HHgC2KaU2K6V8GA7lWxs4PmGROTA8zVQig7E2qB+rKY0zfNQi5K2+prlgfTuffu0utva28KnXXMwdf/w8rj6vf05jEWrHqjY6l4ihlUTA6yaTM777vjpyKRaDRuYR3ATcpJTaC6SBG0ztYJ9S6mbgCSALvFtrnQNQSr0HuANwAzdprfc1cHzCIqK15vDZKHkNyUy+6gq+HOWihixmMw05ecmFa3nJhWvrfm1h7lgawWqOGILifhnNphE0TBCYkT9vrLDv48DHy2y/DbitUWMSlo6hqaQ9kUdT2QURBE6NIDCH6wmLQ4vfwweu28Fzt/Us9VCWFKcgaDZnsWQWC4vCoeGo/b/RbL5+u3ysJGrIqEIKWkNoFduelwPvfMHWpR7CkuNc/DSbs7i5RiOsWA6dLQiCaIUmMdWIpnL4PC57NeVyKTtyqFbTkCAsFUWCoMk0guYajbBiOXx22v7f6ihWL9FUxjYLWVh+grmYmgRhMSn2ETTX1NtcoxFWLIeGowS8xtdtrhpBLJWzexFY2IJATENCk9PMPoLmGo2wItFac+hslAvXtwMFW3+9GCWoi8tBtAUNDcHqCSsIzYozoEE0AmHVMRJNMZnIsGtDBwDx1NxMQ0YJ6goagZiGhCanSCMQQSCsNg6bEUO7NnQC83EWZyv6CMRZLDQ7YhoSVjVWxNDFGwzTkBE+Wj+xVNYOHbVoE0EgLBNCvuZNKBNBIDScQ2enaQt4WN8RxOd2EZ2jaSiaytIaKBYEg91h2oNe8REITU+giTUC+fUIDefQcJRt/a0opQj53XPWCKKpLOGSCf+1uzfwWxeubbofliCUIgllwqrm8Nko5/a3ABD2eebkI8jlNfF0zi5BbeF2qaKaQ4LQrIiPQFi1RKIpIrE05/S1AhD2u+cUPhpLF9cZEoTlRkASyoTVymHTUbytz9QI/J45ZRaX1hkShOWG26VsTUA0AmFVYUUMbTNNQy3+uZmGSpvSCMJyxIoc8rgkakhYJWit+dmBs7QGPKxpCwDGD2EuCWXTSREEwvIn6HXjc7tYoK69C4YIAqFh3LznBHc9eZZ3vmCr/cUPz1kjMIRHqbNYEJYTQa+76cxCIIJAaBCHz0b561uf4Flbu/nD5xVq0Yd9njmFj0ZTGft8QViuBLzupksmA8kjEBaI/3nwJN+4/zjPO7eXF27v4wP/8xgBr4tPv24XLoc9NOz32Kv7erCS0EoTygRhORH0NadGIL8qYUG4+8BZHj4xwYPHx/l/dx4E4N/fvJt+0zdgEfa5SefypLP5un4QEjUkrARCPnfThY6CCAJhgZhMZLhwfTtffPNl/PTJs3jdLq45v3/GcdZEHk9n8Xl8NV8/agsCqSkkLF8CTeojEEEgLAiTiQxdYR99rQFef/nGisdZUT/RVJaOUO2C4OhojJ4WH36PCAJh+XL9rnWcmUgu9TBmIIJAWBAm4hk294SrHhcyV/T1JpXtPT3FznXtcxqbIDQLL7to3VIPoSzNp6MIy5LJRKammj9hh0ZQK6lsjkPD01ywvm3O4xMEoTIiCIR5k89rppIZOmoRBGb4Zz1JZQeHomTzWjQCQWgQIgiEeTOdzKJ1oUnMbFjOXqdGcO/BET5yy96K5+w9PQnABSIIBKEhiCAQ5s1EIg1Qk/PX1ggcSWU/2jvE135zHK112XP2npqkNeBhQ1dwAUYrCEIpIgiEeTOZMLJ+6/EROEtRj0ZT5PKaTK68INh3eoqd69qarj6LIKwURBAI88YSBB2h6oKgED5a8BGMTKcASGRm+g2yuTxPnpkSs5AgNBARBMK8mYjXrhEEvC5cqtg0ZAmCZBlB8NRIjFQ2zwXrRRAIQqMQQSDMG1sjqEEQKKWK2lVqrRmNmhpBmdyCfaajeOc6CR0VhEYhgmCVoLUmm8s35NqWIKglagiMpDIrfHQ6lSWVNcZVzjS099QUAa+LLb0tCzRaQRBKEUGwSviTmx/l97/8QEOuPZnIEPC6inqyzkbY7yFqmoZGTbMQlM823nt6kvPXtuFuso5OgrCSaJggUErtUkrdp5R6RCm1Ryl1ubldKaX+WSl1WCn1mFLqUsc5NyilDpmPGxo1ttXI/U+P8fNDozw9Glvwa0/E0zX5ByzCPg9x0zQ04hAEpT6CfF7zxOkp8Q8IQoNppEbwD8BHtda7gL8ynwO8BNhmPt4BfB5AKdUFfAS4Argc+IhSqrOB41s1xFJZTk0kAPjuQycX/Pq1lpewCPvddk+CkWhBEJT6CI6PxYmmsuIfEIQG00hBoAHrF9wOnDb/vx74qja4D+hQSq0FrgXu1FqPaa3HgTuB6xo4vlXDYbOBfNjn5rsPnSKfLx+vP1cmExk6grVXEnU2sHeahkp9BHttR7FoBILQSBopCP4Y+KRS6gTwKeCD5vb1wAnHcSfNbZW2C/PkkCkI3vbcLZyaSHDf05Gyx50Yi3NgaLru60/EMzU7igFCjnaVRRpBiSA4OBzFpeDc/ta6xyQIQu3MSxAope5SSu0t87geeCfwPq31BuB9wH8sxIDN132H6XfYMzIyslCXXbEcOjuN1614+/O20Or38J0Hy5uH/uqWvbzr6w/Wff2pRKamZDILo4G9MemPTqfxmR2bSn0E08kMYZ+nKRt5CMJKYl6/MK31NVrrC8o8bgFuAL5rHvptDLs/wClgg+MyA+a2StvLve4Xtda7tda7e3t75/MWVgWHh6Ns6Wmhxe/hZRev5fa9Q0UlHiz2nZ7i6dFY2cSu2Zio10fgcxdpBAOdRg2h0qiheCpn9y8QBKFxNHKpdRp4vvn/VcAh8/9bgTeb0UPPBCa11meAO4AXK6U6TSfxi81twjw5dDbKOf1GHP6rLh0gns7xo71DRceMx9KcnU6R19QVWZTO5omnczUlk1mE/R7i6Rz5vGZkOsV6UxCUOotj6axdpE4QhMbRSEHwduAflVKPAp/AiBACuA04AhwGvgS8C0BrPQb8LfCA+fgbc5swDxLpHCfG42zrMwTBZZs6GewOccsjxcrWfodvwPIp1IJdcK4u05DZpSyTYzSaor8tgN/jmqGJJNKiEQjCYtCw5ZbW+hfAZWW2a+DdFc65CbipUWNajTw1EkVr2NZnOFyVUrxgex/feuAE2Vwej2mfPzA0ZZ9zeLh2h3E9lUct7C5lySyj0RQ9LX5CPvcMZ3EsnSUkGoEgNBzxwq1wrNDRbf2FEg2XbOwgkclxwDHhHxiepjPkZXNPuE6NwOhFUI8gsCqQnppIkMlpelv9BL3uGaaheDpHyCcagSA0GhEEK5xDZ6fxuBSD3YXG8pduNPL0Hjo+YW/bPzTN9jWtbOtrmZNpqJamNBbWKv9YxPBF9Lb6CZuksVwAABzGSURBVJTTCFLiIxCExUAEwQrn0HCUwZ5wUQjmQGeQnhY/Dx8fB4xSDgeHptmxpo1t/S0cHY2RztZWoG5upiFjlX80Egegp8VH0Oue4SMQjUAQFgcRBCucw2ejtqPYQinFJRs7eMTUCE5NJIilc6ZG0Eo2rzkaqS1yqJ5eBBbhEo2gzzQNzQgfTedsf4IgCI1DBMEKJpnJcTQSmyEIwPATHBmNMR5L8+QZw1G8fU0r55jHHhquzTxkl6AO1D5hW5P7UTNMtbclQLCMaSiezopGIAiLgCy3VjBPj8bIazinTIkGy0/wyIkJu6zEuf2tuJVCKcO3AGurvsZEPEOr32NHH9WC0zTkc7toC3oIet1FlUjT2TyZnBZBIAiLgAiCFYzl9C2nEVw00I5LwcPHx3lqNMaGrqAdzbOhM1Szw3gqkakrhwAKGsFkIsO69gBKKYK+Yh+BlXks4aOC0HjkV7aCOTw8jUvB5p7wjH0hn4cda9p4+MQEZyaTbO8vlHre1tfC4RpNQ/WWlwCKIoF6Wv0ARvhokSAw/g9LQpkgNBzxEaxgDgxPs6k7XLFz2KWbOnj4+ARPj8bYsaZgPjqnv4Ujo9GaWltO1llwDsDtUgS8xlevt8UQBIGSPALRCARh8RBBsEKJprL8/NAolw92VTzmkg2dRFNZcnnNdocg2NbXSianOTYWr/o69TalsbDMUL2WRlDiLLYa14hGIAiNRwTBCuXWR04TT+d4/eUbKh5zycYO+/8dRYKg9sihiXiG9jqa0lhYK/2eloJpKJPTZEwtJGZqBEGvaASC0GhEEKxQvnH/cXasaWXXho6Kx2zuCdMR8uJzuxh0+BG2moLg8NnZaw5prQ1n8Rw0gnCJRmBFB1kO47hoBIKwaIggWIHsPTXJ46cmef0zNqCUqnicUoort3Rz4UA7Xkf4Z4vfw/qOYNXIoUQmRzqXn5sgMCf+HoePwLomGJVJQXwEgrAYyK9sBfKN+4/j97j4nUsGqh77yddcTC43s4fx1r4Wu2BdJQp1huavEQRNQZBMG6ahuNk4RzQCQWg8ohGsMOLpLLc8cpqXXri2pvj+Fr+n7HG9LX67fEQl5lJewvm6UOwshoJGEEuLRiAIi4UIgibkwWPjHKqjJ4CTHz52hmgqyxuu2DivMbQGPEwnZxcEtkYwB0EQsk1DhqPZ0gissFFLI5DMYkFoPCIImpAPf+9xPvXjA3WfNzyV5F9+eohz+lrYvalzXmNo8XuIprIYfYTKY2kEbXMQBF1hH61+j60ZlPoIYukcPreryHchCEJjEL27CYmmsozHZl+NlzIaTfG7X7qPsWiar73tilmdxLXQEvCQ18bEXMk8MzUPH8Hbn7eF37pwrT3O0qihRDorbSoFYZEQQdCEJDN5pqqYZZxMxNO88d9/w6mJBF/5/cu5ZOP8tAEo2PCjycrtIufSi8Cip8VvRwyBw0eQtvIIctKURhAWCdG7m5BUJmdPsrXw4e/v5chojC+9eTdXbOlekDG0mmWlp5LZisdMJNK4XcoWGvMhWBo+KiWoBWHRkCVXE5LI5MjPYpsv5cDQNFdt7+O523oXbAyWIIimKgsCq7zEfM1QUMZHkJLuZIKwWIhG0GRkc3myeU0snbPLLVRjLJamu6X+Mg+z0eI3zD3RWTSC4amUHfUzXwqmITNqKF3ZJCUIwsIigqDJSDp6BU/PMglb5PKa8Xia7vBCCwJLI6hsojoeibOxa2aJ67lgm4ashLJ0TpLJBGGREEHQZDhLMdfiJ5iIp9HaCMdcSCzTUCVhpLXm+FicTd2hBXk9t0vh87gcPoLK0UqCICwsIgiaDGeXrqkaBEEklgag2xGBsxBU8xGMTKdIZHILJgjA0AqSto8gKxqBICwSIgiajFS2Po0gEjUFwQJrBFYtoEoagdWrYGPXwgoCSyOKp3NSgloQFgkRBE1GMlPwEdSSSzBmagRdC+ws9rpdBLyuihrBsYghCDZ1L4yPAArNabTWxNOiEQjCYiGCoMlIFJmGqjuLx2IpYOF9BGBEDlXSCI5HYrgUrO8ILtjrBcy+xalsnryWgnOCsFiIIGgynD6CmkxDpkbQGVp4QdAa8FTWCMbirOsI4vMs3Fco6HWRSOeISQlqQVhURBA0GfWahiLRNB0hb0OKs7X4PUQrjOFYZOEihiws01BcSlALwqIigqDJSNSpEYzF0g0xC4FVirqCaWhs4XIILIJej6ERpKUEtSAsJiIImgzLNOTzuGoMH00teMSQhVWKupTpZIaxWLohGkGySCMQQSAIi4EIgiYjZQqCvlb/kmsELRU0AjtiaAFDR8H0EWRyjsb1YhoShMVgXoJAKfUapdQ+pVReKbW7ZN8HlVKHlVIHlFLXOrZfZ247rJT6c8f2zUqp35jbv6WUaszs1uRYpqH+tsCslT8tDEGwsMlkFq0VNILjVg7BQmsEZtSQmIYEYXGZr0awF3glcK9zo1LqfOD1wE7gOuBzSim3UsoNfBZ4CXA+8AbzWID/C3xaa30OMA68dZ5jW5ZYzuL+Nn9V01A+rxmLpRes8FspLYHyXcqORmLAwuYQAAR8buLpnN2uUvoRCMLiMC9BoLV+Umtdrqfi9cA3tdYprfXTwGHgcvNxWGt9RGudBr4JXK+MOsZXAd8xz/8K8Ir5jG25kszk8LgUnSFfVUEwkciQb0CdIYvWgJdcXhc5sMEoNtcd9i1IHwInQa+bdDZvVzwVjUAQFodG+QjWAyccz0+a2ypt7wYmtNbZku2rjkQmR9Drpj3oZTKRmbVncCOTyaC4S5mTY5H4gpuFoDDxW7kRIfERCMKiUFUQKKXuUkrtLfO4fjEGWGFM71BK7VFK7RkZGVmqYTSEZCaP3+umLeglW2Y17qRQZ6hBPgKrAmmJn+D4WHzBHcVQKEVtvS/ruSAIjaXqkktrfc0crnsK2OB4PmBuo8L2CNChlPKYWoHz+HJj+iLwRYDdu3fX3sprGZDK5Ah4XXYf4KlE5QYtdp2hRdQIUtkcpycTbOweWPDXs7qURWIpgl43btf8O58JglCdRpmGbgVer5TyK6U2A9uA+4EHgG1mhJAPw6F8qzbsH3cDrzbPvwG4pUFja2os01BbwBAEs4WQjpqCoGHOYv/MUtQnxxNovfCho1DoUjYaTUt5CUFYROYbPvo7SqmTwJXA/yql7gDQWu8DbgaeAG4H3q21zpmr/fcAdwBPAjebxwJ8APgTpdRhDJ/Bf8xnbMuVZCZHwPQRwOxlJsZME0pnA53FYCSQWRy3q4420jSUsoWCIAiNZ17eOK3194DvVdj3ceDjZbbfBtxWZvsRjKiiVU0ykyfgddEWND6ayfgsgiCWoi3gaUidISjfpeyYGTraCGexLQhiada0BRb8+oIglEcyi5uMZLZ2jSASSy94ZzIn5UxDx8bihHxuehvwupYWMBHPSOioICwiIgiajETaEAS1+Agi0caVl4BCiQens/jEWIINnSGM1I+FxWkOkvISgrB4iCBoMlLZPAGv2zbLzNacZiyWbljBOTAK3/k9xV3KTk0kWN+5cM1onDjDRUUjEITFQwTBEhJNZYsa0YDpLPa48LhdtPg9s2sEsTTdDYoYsmgNeItqHp0ajy9oVzInxYJANAJBWCxEECwhv/ul+/i7254s2pbI5GwTSVvAU9FHkM9rxuONNQ1BcZey6WSGqWS2YRpBwCcagSAsBbLsWiKSmRx7T03SU+J0tcJHAdrMMhPlmEpmyOV1wyqPWji7lJ2aSAAL26fYiVMjEB+BICweohEsEU+Pxsjr4hh9rbURPmr2AW4LeisWnhu1y0s0ViNwNqc5NW4KggZpBF63C6/bcEKLRiAIi4cIgiXi0NkoUByjn8oaJagtE0n7LBqBVV6i0T4CZ3MaSyMYaJBGAIUyE1KCWhAWDxEES8Th4WmgWBBYjuOAx/IReCv2DG505VELZ9/iU+MJfG7XDHPWQmKZhySzWBAWDxEES0RBIyis+K2mNNaquH0W05BVqrlRlUctnF3KTk4kWNcRwNXAYnCWAJBaQ4KweIggWCIOmhqBswOYVXI66LN8BB6mU1ly+ZkFVgt1hrwNHaezS9mp8cblEFhYGoGEjwrC4iGCYAlIZ/McjcQJeF3kNcTThgAoNQ1ZZSamy4SQRmJpWgMe/J7Grpxb/EaXsmQmbySTNdA/AOIjEISlQATBEnA0EiOX11w00AEU/AS2IPAWfAT8/+3de2yd9X3H8ffXPvbx5Rxfkji2YycQiJPghHs2blGpgCDoBcrWaVs7wdZuSNuqlg5poqrUdVInrdK0C9KEROk2YB1dS2kJrSgFhlTUABkUCLmQOoZAnPgSJ3Yc3+2c3/54nsfnHPsc5+bzHHaez0uy7PM85/j5PfzC8/Xv+7uRe5mJYwWeVRxI+DOcB0enOHpyiraGpV9sLtNci0CpIZHQKBAUQVe/1z9w5ZogEHgP+iA1FK/wqiVzc5r5jo9NFbyjGLxJbQBdA14qq9CpoWDYqIaPioRHgaAIugZOYgZXrvYDgd8ZO+V3FldnTCiD3C2C/pEpmpKF7SiG9Aqk+3r9QFDo1FClUkMiYVMgKIKugVFWN9bMPcjzpob8PQnmLzPhnKNnaJzVjYVN00A6EOzv8wJBe2idxWoRiIRFgaAIDvSPsr45QSKe3Rk8OZsdCOrztAiOjU0zOZMq+EMZ0n0E+/u8VkxzgTeMCQKBlpgQCY8CQchmT6V4b3CUdSuTc0tNB+v9T0zPSw1VBX0E2YGgx1/qoT2EFkHSD1bdR0dpTlZRGSvsP5maynLKDOIFvo6IpOnPrpB9cHycmVOOjpWJBVtBplND3kOwprKcWJktaBH0DHn7BrcvK3yLICjjbMoVvKMY4LNXt7NmeWE2vhGR3BQIQhaMGOpoTlBbGcMsf2rIzLyF5yZztwgK3XEL2SmaMK7X0ZykozlZ8OuISJra3yE74A/DvLgpQVmZkaiMzY0amvQnlmWmRbxlJrKHj/YMjdNQU0GyqrCziiG9SxkUfuioiBSHAkHIugZGaWuonvtLO3NRt8nZFFUVZVlpkaZEnCP+qp+BnqGJUDqKA0F6KIwWgYiEL5KBYHRqlk8++DL/+eoHoV+7q3+UdSsTc6+9ZZ791FDGpjSB9S0J9vefnFuPCPxAUOAZvpmCIaRqEYiUpkgGgtrKcvpHJnn70HDo1+4fmcx6oCarKuZW95yYPpW1SxfAxpY6Tk7OcuTEJJCeQxBui8BLQRVyHwIRKZ5IBgIz45LWOvYcGQn1us45hidmaKhO5/YXpobmBwKv4/TdXq+sg6PhzSEIqEUgUtoiGQgAOlfV0TVwkml/V7AwBEtKN9ak1whKxGNZw0fnj59fHwQCf2ZvMHR09bIQU0NVMRprKrQ0tEiJiuz/2ZtW1TNzynFgYJTOVXWhXHN4zOsLqK/JbBFUZAWC+S2CuqoK2hqq55Z4CHMyWeBTl7WyeVV9aNcTkXBFNhB0tnoP/z1HToQXCCb8zWQyWgR18zqL5/cRgJcemh8IwkzT3HlFW2jXEpHwRTY1tHZFLdUV5eztDa+fYGjce+A3ZLQIEvEYU7MppmdTTM6k5mYVZ9rQkqT76CjTsyl6hsZprKmYy9uLiJyvyAaC8jJjY2uSvSF2GA+PBy2C7M5i8Ia0TuRIDQFsbK1jNuXoPjrqzyEILy0kIqUvsoEAvPTQ3t6RrDH6Z8M5x33ff5On3zp8Ru8fnmsRpFNDwdDMk5Mzi6aGwFsBNOyhoyJS+qIdCFZ5Y/SDvPvZeu394/zkrSM88/aRM3p/EAjqM4aPJjIWnpucSRHPEQjWrqilotzY1zsS+qxiESl9kQ4Em/yRMOc6n+DxV7yZycHQztMZGp8mGY9RUZ7+z565AunUzKmcfQQV5WWsW5nkV92DTM2mlBoSkSV1XoHAzH7PzPaYWcrMtmQc32Zmb5jZO/73mzLOXe0fP2BmD5q/sI6ZLTOz582sy//eeD5lOxMbmpOUGew9cuKsP9t3YpLn9vSRrIrRMzQxNzt4McPj01lDRyG958DJyRkm8qSGwEsP7T7sBSy1CERkKZ1vi2A38DvAL+cdHwQ+7Zy7FLgHeDzj3EPAnwEd/tdt/vEHgBedcx3Ai/7rgqquLOfipsQ5jRz6r50fcso57rtlPZDeynExwxMzWUNHIT1rd3h8htmUy9lZDN7IoYBaBCKylM4rEDjn9jnn9uc4/qZzLkic7wGqzSxuZq1AnXPuVef10D4GfMZ/353Ao/7Pj2YcL6jOVXVnPXJoejbFEzs/5OPrm7i1sxk4s0AwND6TNXQU0qmho6NTADlTQ5DuMAYt9SAiSyuMPoLfBX7tnJsC2oCejHM9/jGAZudcr/9zH9AcQtnobK3jyIlJhsam+eDYGPf/4G1efe/Yop95bk8fR09Ocff1F3pLSleWs7/v9MHkxPh01oghSHcWHz3pBYL8qSFv0pvmEIjIUjvtE8XMXgBacpz6unPu6dN8dhPwbeDWsymUc86ZWd4xnWZ2L3AvwJo1a87mVy8QdBh/Y/sefrGnj6nZFLOpFNdetDzvZx5/5QMuWF7DjR1NlJUZ61uSZ9RhPDQ+kzWHACAeK6cyVjbXIsg1agiguS5OfXVFqGsMiUg0nDYQOOduOZdfbGbtwI+Bu51z3f7hw0B7xtva/WMA/WbW6pzr9VNIA4uU6WHgYYAtW7ac2yQA3yWtXsrlmbePcPvmFo6PTbOrJ3/n8bHRKXYePM7929ZTVuZtILOxJcmzu/twzuXda/dUyjEymb3yaKCuKjbXIsjXR2Bm3HVlG8tqK3OeFxE5VwVJDZlZA/Az4AHn3K+C437qZ8TMrvVHC90NBK2K7Xgdy/jfF21tLJXliTh/8+lOHrl7Cw/90dV8bH0T7w+OLdgwPvCKnzba2rFi7tiG5iTD4zMM+A/zXEYmZnCOBakh8CaVDZ4mNQTwzTs28eWbO87ovkREztT5Dh+9y8x6gOuAn5nZc/6pLwHrgG+Y2Vv+10r/3F8AjwAHgG7gWf/43wPbzKwLuMV/HYo/uWEtt/idvpe2eami3Ydztwp2dB8jGY/NvQ9gg5+/Xyw9NDyxcJ2hQCKe2SKI9NQOESmC8+p1dM79GC/9M//4t4Bv5fnM68DmHMePATefT3mWwmXt3gP+7Z5hbli3YsH5HQcGueaiZcQyJoWll4AY4cb1TTl/79D4wpVHA8mq9Ab2+VJDIiKFoj8/52moqeSC5TXsOrSwRXB4eIKDx8a57uLsANFYW8nKZHzxFoEfCHK1CIIhpABVMQUCEQmXAkEOl7bV806O1NCOA4MA3LBu4YiiDS1JftO/WCBYuOBcIBFPB4fqSlWJiIRLT50cLm9v4PDwBIOj2Z2/O7qPsby2kg3NyQWf2diSpKt/lFOp3IOYgr0I5g8fhewWQVwtAhEJmQJBDkE/wa6e4bljzjl2dA9y3cXLcw4R3dBSx9RsioPHxnL+zhPj05ill53OVJeZGlIfgYiETIEgh01t9ZiRNZ+g++gY/SNTOTuQIXvPgFyGxmeor66gvGxhEElkBILqSgUCEQmXAkEOiXiMdU2JrECwo9vrH7j+4twzjtetTFBm8G6eBeyGJ3JPJoPsVkJVTFUiIuHSUyePy9ob2NUzPLd72Y4Dx2hrqGZNniUeqirKuXx1A8/s6iWVo59gOMc6Q4GgjyBWZlnDUkVEwqCnTh6XtdczODrN4eEJntj5IS/tH2DruhV5l5AA+MINa3l/cIwX9vUvODc0Pp2zoxjSS1EvNqtYRKRQFAjyCDqMP/ed1/jaU+9w1ZpGvrpt/aKfuX1zC20N1Tzy8vsLzg2PzyzSIvACRL4F50RECkmBII9LWuuIx8o4PjbN3921me/96TW01Fct+plYeRlf2LqWnQeP89ah4axzwzn2IggEo4a0vISIFIOePHlUVZTzoz+/nhf+6kY+f80FcyuNns7v/9ZqklUxvvPye3PHZk6lGJ2apaE6d4sgGDWk1JCIFIMCwSI2t9WfthUwXyIe43PXrOHZd3o5dHwcSM8qbqxdfNSQ5hCISDEoEBTAH19/IWVmPPbKQQBOTHjrDNXnGT5aW1mOmVJDIlIcevIUQGt9NTeub5rbrCa9vETu1JCZkYjH1CIQkaJQICiQWzc10zM0wb7ekwyN5V+COlBXVaFAICJFoUBQIDdtbMYMnt/bv+imNIFtnc1szbN8hYhIIZ3XxjSSX1MyztVrGvnF3j7uvGIVsHgg+OYdm8IqmohIFrUICujWTc3sOTLCniMjxMpsbgaxiMhHiQJBAW3rbAHg57v7aKipWHR5ChGRYlEgKKC1K2rpWJlgajaVd+ioiEixKRAU2K2bmoHFRwyJiBSTAkGBBemhfAvOiYgUmwJBgV3WVs8Fy2vy7mMgIlJsGsZSYGVlxvYvbdXyESLykaVAEAJ1FIvIR5n+TBURiTgFAhGRiFMgEBGJOAUCEZGIUyAQEYk4BQIRkYhTIBARiTgFAhGRiFMgEBGJOAUCEZGIM+dcsctwXszsKPDBOX58BTC4hMX5qNP9lr6o3bPud3EXOOeaTvem//eB4HyY2evOuS3FLkdYdL+lL2r3rPtdGkoNiYhEnAKBiEjERT0QPFzsAoRM91v6onbPut8lEOk+AhERUYtARCTyIhkIzOw2M9tvZgfM7IFil2epmdlqM3vJzPaa2R4z+4p/fJmZPW9mXf73xmKXdSmZWbmZvWlmP/VfrzWz1/x6/m8zqyx2GZeSmTWY2ZNm9q6Z7TOz60q5js3sq/6/591m9oSZVZVaHZvZv5nZgJntzjiWs07N86B/77vM7KpzvW7kAoGZlQP/CtwOdAJ/aGadxS3VkpsF7nfOdQLXAn/p3+MDwIvOuQ7gRf91KfkKsC/j9beBf3LOrQOGgC8WpVSF8y/Az51zG4HL8e69JOvYzNqALwNbnHObgXLgDyi9Ov4P4LZ5x/LV6e1Ah/91L/DQuV40coEA+G3ggHPuPefcNPB94M4il2lJOed6nXO/9n8+ifeAaMO7z0f9tz0KfKY4JVx6ZtYOfBJ4xH9twE3Ak/5bSu1+64GPAd8FcM5NO+eGKeE6xttjvdrMYkAN0EuJ1bFz7pfA8XmH89XpncBjzvMq0GBmredy3SgGgjbgUMbrHv9YSTKzC4ErgdeAZudcr3+qD2guUrEK4Z+BvwZS/uvlwLBzbtZ/XWr1vBY4Cvy7nw57xMxqKdE6ds4dBv4B+BAvAJwA3qC06ziQr06X7FkWxUAQGWaWAH4E3OecG8k857zhYiUxZMzMPgUMOOfeKHZZQhQDrgIecs5dCYwxLw1UYnXciPcX8FpgFVDLwhRKyStUnUYxEBwGVme8bvePlRQzq8ALAt9zzj3lH+4Pmo7+94FilW+J3QDcYWYH8VJ9N+Hlzxv8NAKUXj33AD3Oudf810/iBYZSreNbgPedc0edczPAU3j1Xsp1HMhXp0v2LItiIPhfoMMfbVCJ1+G0vchlWlJ+fvy7wD7n3D9mnNoO3OP/fA/wdNhlKwTn3Necc+3OuQvx6vN/nHOfB14CPuu/rWTuF8A51wccMrMN/qGbgb2UaB3jpYSuNbMa/993cL8lW8cZ8tXpduBuf/TQtcCJjBTS2XHORe4L+ATwG6Ab+Hqxy1OA+9uK13zcBbzlf30CL2/+ItAFvAAsK3ZZC3DvHwd+6v98EbATOAD8EIgXu3xLfK9XAK/79fwToLGU6xj4W+BdYDfwOBAvtToGnsDrA5nBa/V9MV+dAoY3ArIbeAdvRNU5XVczi0VEIi6KqSEREcmgQCAiEnEKBCIiEadAICIScQoEIiIRp0AgIhJxCgQiIhGnQCAiEnH/B5GPjRl6sxsTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "state = envs.reset()\n",
    "\n",
    "while frame_idx < max_frames:\n",
    "\n",
    "    log_probs = []\n",
    "    values    = []\n",
    "    rewards   = []\n",
    "    masks     = []\n",
    "    entropy = 0\n",
    "\n",
    "    for _ in range(num_steps):\n",
    "        state = torch.FloatTensor(state).to(device)\n",
    "        dist, value = model(state)\n",
    "\n",
    "        action = dist.sample()\n",
    "        next_state, reward, done, _ = envs.step(action.cpu().numpy())\n",
    "\n",
    "        log_prob = dist.log_prob(action)\n",
    "        entropy += dist.entropy().mean()\n",
    "        \n",
    "        log_probs.append(log_prob)\n",
    "        values.append(value)\n",
    "        rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(device))\n",
    "        masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device))\n",
    "        \n",
    "        state = next_state\n",
    "        frame_idx += 1\n",
    "        \n",
    "        if frame_idx % 1000 == 0:\n",
    "            test_rewards.append(np.mean([test_env() for _ in range(10)]))\n",
    "            plot(frame_idx, test_rewards)\n",
    "            \n",
    "    next_state = torch.FloatTensor(next_state).to(device)\n",
    "    _, next_value = model(next_state)\n",
    "    returns = compute_gae(next_value, rewards, masks, values)\n",
    "    \n",
    "    log_probs = torch.cat(log_probs)\n",
    "    returns   = torch.cat(returns).detach()\n",
    "    values    = torch.cat(values)\n",
    "\n",
    "    advantage = returns - values\n",
    "\n",
    "    actor_loss  = -(log_probs * advantage.detach()).mean()\n",
    "    critic_loss = advantage.pow(2).mean()\n",
    "\n",
    "    loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-283.0576102217745"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_env(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}