IMU_Data_Preprocessing_Paper_Implementation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "IMU Data Preprocessing Paper Implementation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/artykov1511/MyDLProjects/blob/IMU-Preprocessing-with-RNN-paper-implementation/IMU_Data_Preprocessing_Paper_Implementation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eBXSMf97abyx",
        "outputId": "245c3da6-abf3-4862-ba75-f3f61abff642"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "%cd /content/drive/MyDrive/Deep_Learning_Applications/VO\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "#import data_loader\n",
        "#import torch_se3\n",
        "import time\n",
        "#from params import par\n",
        "from torch.autograd import Variable\n",
        "from torch.nn.init import kaiming_normal_, orthogonal_\n",
        "import torch_se3\n",
        "import os\n",
        "\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "!pip install transformations\n",
        "import transformations\n",
        "import time"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "/content/drive/MyDrive/Deep_Learning_Applications/VO\n",
            "Collecting transformations\n",
            "  Downloading transformations-2021.6.6.tar.gz (45 kB)\n",
            "\u001b[K     |████████████████████████████████| 45 kB 1.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.15.1 in /usr/local/lib/python3.7/dist-packages (from transformations) (1.19.5)\n",
            "Building wheels for collected packages: transformations\n",
            "  Building wheel for transformations (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformations: filename=transformations-2021.6.6-cp37-cp37m-linux_x86_64.whl size=113512 sha256=dd2b78fbcf5f7487fa5d5cddae9f33804f9ea0c1c7808391dd3f3974b7a8a2b8\n",
            "  Stored in directory: /root/.cache/pip/wheels/f0/3c/16/a67f60f017d0003f27b7156162df6f3aa45e4b2cee82232e3a\n",
            "Successfully built transformations\n",
            "Installing collected packages: transformations\n",
            "Successfully installed transformations-2021.6.6\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pEuLsfYu2KW6",
        "outputId": "633a6ff1-4f5b-4c21-c27c-b09e96025026"
      },
      "source": [
        "gt_timestamps = []\n",
        "gt_data = []\n",
        "base_dir = '/content/drive/My Drive/Deep_Learning_Applications/VO/BB/clover_0p5/csv'\n",
        "\n",
        "imu_timestamps = []\n",
        "imu_data = []\n",
        "imu_csv = open(os.path.join(base_dir, 'groundTruthPoses_2.csv'), 'r')\n",
        "\n",
        "imu_csv.readline()  # skip header\n",
        "i = 0\n",
        "for line in imu_csv:\n",
        "    line = line.split(\",\")\n",
        "    timestamp = int(line[0])\n",
        "    data = [float(line[i]) for i in range(1, 11)]\n",
        "    #data1 = [float(line[i]) for i in range(1, 11)]\n",
        "    #gyro_data = [float(line[i]) for i in range(22, 25)]\n",
        "    #accel_data = [float(line[i]) for i in range(35, 38)]\n",
        "    #data = gyro_data + accel_data\n",
        "    #print(timestamp)\n",
        "    #print(line[37])\n",
        "    imu_timestamps.append(timestamp)\n",
        "    imu_data.append(data)\n",
        "    i += 1\n",
        "    if i == 3:\n",
        "      break\n",
        "\n",
        "ref_time = np.datetime64(int(imu_timestamps[0]), \"ns\") # nanosecs\n",
        "print((np.datetime64(imu_timestamps[2], \"ns\") - ref_time) / np.timedelta64(1, \"s\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.005556\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qBcrtX8evVqc"
      },
      "source": [
        "\n",
        "class IMU_Dataset(Dataset):\n",
        "  def __init__(self, seq_len, sequence):\n",
        "    self.base_path = os.path.join('BB', sequence, 'csv')\n",
        "    self.imu = self.read_imu()\n",
        "    self.gt = self.read_gt()\n",
        "    self.seq_len = seq_len\n",
        "    \n",
        "    \n",
        "    \n",
        "    \n",
        "  def read_imu(self):\n",
        "    imu_timestamps = []\n",
        "    imu_data = []\n",
        "    imu_csv = open(os.path.join(self.base_path, 'blackbird_slash_imu.csv'), 'r')\n",
        "\n",
        "    imu_csv.readline()  # skip header\n",
        "    for line in imu_csv:\n",
        "        line = line.split(\",\")\n",
        "        timestamp = int(line[0])\n",
        "        gyro_data = [float(line[i]) for i in range(22, 25)]\n",
        "        accel_data = [float(line[i]) for i in range(35, 38)]\n",
        "        data = gyro_data + accel_data\n",
        "        imu_timestamps.append(timestamp)\n",
        "        imu_data.append(data)\n",
        "    \n",
        "    imu_time = []\n",
        "    ref_time = np.datetime64(int(imu_timestamps[0]), \"ns\") # nanosecs \n",
        "    for i in range(0, len(imu_timestamps)):\n",
        "      ts = (np.datetime64(imu_timestamps[i], \"ns\") - ref_time) / np.timedelta64(1, \"s\")\n",
        "      imu_time.append(ts)\n",
        "      imu_data[i].insert(0, ts) \n",
        "\n",
        "    return (imu_timestamps, imu_data, imu_time)\n",
        "\n",
        "  def read_gt(self):\n",
        "    gt_timestamps = []\n",
        "    gt_data = []\n",
        "    gt_csv = open(os.path.join(self.base_path, 'groundTruthPoses_2.csv'), 'r')\n",
        "\n",
        "    gt_csv.readline()  # skip header\n",
        "    for line in gt_csv:\n",
        "      line = line.split(',')\n",
        "      timestamp = int(line[0])\n",
        "      data = [float(line[i]) for i in range(1, 11)]\n",
        "      gt_timestamps.append(timestamp)\n",
        "      gt_data.append(data)\n",
        "    \n",
        "    # Preprocess gt according to imu (imu freq. = 100Hz, gt freq. = 360Hz)\n",
        "    imu_timestamps, imu_data, imu_time = self.read_imu()\n",
        "    #imu_data = np.array(imu_data)\n",
        "    gt_data = np.array(gt_data)\n",
        "\n",
        "    assert np.all(np.diff(gt_timestamps) > 0), \"nonmonotonic timestamp\"\n",
        "    assert np.all(np.diff(imu_timestamps) > 0), \"nonmonotonic timestamp\"\n",
        "\n",
        "    gt_imu_aligned_timestamps = []\n",
        "    gt_imu_aligned_data = []\n",
        "    for i in imu_timestamps:\n",
        "      gt_idx = (np.abs(np.array(gt_timestamps)  - i)).argmin()\n",
        "      gt_imu_aligned_timestamps.append(gt_timestamps[gt_idx])\n",
        "      gt_imu_aligned_data.append(gt_data[gt_idx])\n",
        "\n",
        "    gt_imu_aligned_data = np.array(gt_imu_aligned_data)\n",
        "\n",
        "    \"\"\"\n",
        "    gt_gamma = []\n",
        "    gt_beta = []\n",
        "    gt_C = []\n",
        "    for i in range(1, gt_imu_aligned_data.shape[0]):\n",
        "      dt = imu_time[i] - imu_time[i-1]\n",
        "      T_k = transformations.quaternion_matrix(gt_imu_aligned_data[i-1][3:7])      # gt pose of body\n",
        "      C_b_w = np.transpose(T_k[0:3, 0:3])   \n",
        "      gamma = C_b_w.dot(gt_imu_aligned_data[i][0:3] - gt_imu_aligned_data[i-1][0:3] - gt_imu_aligned_data[i-1][7:10] * dt - 0.5*np.array([0,0,9.82])*(dt**2)) \n",
        "      gt_gamma.append(gamma)\n",
        "      beta = C_b_w.dot(gt_imu_aligned_data[i][7:10] - gt_imu_aligned_data[i-1][7:10] - np.array([0,0,9.82])*dt)  \n",
        "      gt_beta.append(beta)\n",
        "      T_k1 = transformations.quaternion_matrix(gt_imu_aligned_data[i][3:7])\n",
        "      C_k1 = T_k1[0:3, 0:3]\n",
        "      gt_C.append(C_b_w.dot(C_k1))\n",
        "    \"\"\"\n",
        "\n",
        "    return (gt_imu_aligned_timestamps, gt_imu_aligned_data)  # (gt_gamma, gt_beta, gt_C)\n",
        "\n",
        "\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.imu[1]) - self.seq_len * 1 + 1\n",
        "  \n",
        "  def __getitem__(self, index):\n",
        "    \n",
        "    imu_ts = self.imu[0]\n",
        "    imu_data = self.imu[1]\n",
        "    imu_time = self.imu[2]\n",
        "\n",
        "    gt_ts = self.gt[0]\n",
        "    gt_data = self.gt[1]\n",
        "\n",
        "    \n",
        "    imu_stacked = imu_data[index:index + self.seq_len]\n",
        "    #gt_stacked = gt_data[index:index + self.seq_len]\n",
        "    \n",
        "    dt = imu_time[index + self.seq_len - 1] - imu_time[index]\n",
        "    T_k = transformations.quaternion_matrix(gt_data[index][3:7])      # gt pose of body\n",
        "    C_b_w = np.transpose(T_k[0:3, 0:3])   \n",
        "    gamma = C_b_w.dot(gt_data[index + self.seq_len - 1][0:3] - gt_data[index][0:3] - gt_data[index][7:10] * dt - 0.5*np.array([0,0,9.82])*(dt**2))\n",
        "    beta = C_b_w.dot(gt_data[index + self.seq_len - 1][7:10] - gt_data[index][7:10] - np.array([0,0,9.82])*dt)\n",
        "    T_k1 = transformations.quaternion_matrix(gt_data[index + self.seq_len - 1][3:7])\n",
        "    C_k1 = T_k1[0:3, 0:3]\n",
        "    C = C_b_w.dot(C_k1)\n",
        "\n",
        "    # Convert tensors into torch style\n",
        "    # DO WE NEED TO TRANSFORM TENSORS FROM NUMPY TO TORCH ????\n",
        "\n",
        "    return np.array(imu_stacked), np.expand_dims(gamma, 1), np.expand_dims(beta, 1), C\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "tr_seq = 'clover_0p5'\n",
        "val_seq = 'clover_1p0'\n",
        "train_dataset = IMU_Dataset(param['seq_len'], tr_seq)\n",
        "val_dataset = IMU_Dataset(param['seq_len'], val_seq)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "  tr_dataset = IMU_Dataset(8, 'clover_0p5')\n",
        "  tr_loader = DataLoader(\n",
        "        tr_dataset,\n",
        "        batch_size=32, #params[\"batch_size\"],\n",
        "        #sampler = train_sampler,\n",
        "        shuffle=False,                                                          # Shuffle might be True !\n",
        "        num_workers= 4, #params[\"num_workers\"],\n",
        "        pin_memory=True )\n",
        "  print(len(tr_dataset))\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iZCWYoTKbTcC"
      },
      "source": [
        "param  = {\n",
        "    'device': 'cuda',\n",
        "    'input_size': 6,\n",
        "    'hidden_size': 128,\n",
        "    'num_layers': 2,\n",
        "    'num_classes': 6,\n",
        "    'batch_size': 32,\n",
        "    'lr': 0.0001,\n",
        "    'epoch': 20,\n",
        "    'seq_len': 10,\n",
        "    'num_workers': 4 \n",
        "}\n",
        "\n",
        "\n",
        "\n",
        "class Huber_Loss(nn.Module):\n",
        "  def __init__(self, delta=1.0, rotation=False):\n",
        "    super(Huber_Loss, self).__init__()\n",
        "    self.rotation = rotation  \n",
        "    self.delta = delta\n",
        "\n",
        "  def forward(self, inputs, targets):\n",
        "    total_error = 0\n",
        "    total_points = 0\n",
        "  \n",
        "    for i, n in enumerate(inputs[0]):\n",
        "      total_points += 1\n",
        "      if self.rotation == False:\n",
        "        error = torch.linalg.norm(inputs[i] - targets[i])\n",
        "      else:\n",
        "        mm = torch.matmul\n",
        "        #print(inputs[0].dtype, targets[0].dtype)\n",
        "        #a = torch_se3.log_SO3(mm(inputs[i].transpose(-2,-1), targets[i]))\n",
        "        #print(a.dtype)\n",
        "        error = torch.linalg.norm(torch_se3.log_SO3(mm(inputs[i].transpose(-2,-1), targets[i])))\n",
        "      \n",
        "      if error <= self.delta:\n",
        "        huber_error = 0.5 * (error**2)\n",
        "      else:\n",
        "        huber_error = self.delta * (error -  0.5*self.delta)\n",
        "      \n",
        "      total_error += huber_error\n",
        "    \n",
        "    huber = total_error / total_points\n",
        "    return huber\n",
        "\n",
        "L_rot = Huber_Loss(delta=1.0, rotation=True).to(param['device'])\n",
        "L_v = Huber_Loss(delta=1.0).to(param['device'])\n",
        "L_p = Huber_Loss(delta=1.0).to(param['device'])\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mSupUbQlapy0"
      },
      "source": [
        "\n",
        "\n",
        "\n",
        "class Bi_LSTM(nn.Module):\n",
        "  def __init__(self, param):\n",
        "    super(Bi_LSTM, self).__init__()\n",
        "\n",
        "    self.input_size = param['input_size']\n",
        "    self.hidden_size = param['hidden_size']\n",
        "    self.num_layers = param['num_layers']\n",
        "    self.num_classes = param['num_classes']\n",
        "    self.param = param\n",
        "    self.LSTM = nn.LSTM(self.input_size, self.hidden_size, self.num_layers, batch_first=True, bidirectional=True)\n",
        "    self.fc = nn.Linear(self.hidden_size*2, self.num_classes)\n",
        "\n",
        "    self.reset_hidden_states()\n",
        "\n",
        "  def reset_hidden_states(self, batch_size=32, zeros=True):\n",
        "    if zeros:\n",
        "      self.h = Variable(torch.zeros(self.num_layers*2, batch_size, self.hidden_size)).to(self.param['device'])\n",
        "      self.c = Variable(torch.zeros(self.num_layers*2, batch_size, self.hidden_size)).to(self.param['device'])\n",
        "      #print(self.h.size(), self.c.size())\n",
        "\n",
        "    else:\n",
        "      self.h = Variable(self.h.data).to(self.param['device'])\n",
        "      #print(self.h.dtype)\n",
        "      self.c = Variable(self.c.data).to(self.param['device'])\n",
        "      #print(self.c.dtype)  \n",
        "  \n",
        "  def forward(self, x):\n",
        "    # Extract imu time data\n",
        "    batch_size = x.size(0)\n",
        "    seq_len = x.size(1)\n",
        "    t = x[..., 0].view(batch_size, seq_len, 1)\n",
        "    w = x[..., 1:4].view(batch_size, seq_len, 3)\n",
        "    a = x[..., 4:7].view(batch_size, seq_len, 3)\n",
        "    x = torch.cat((w, a), 2)\n",
        "    #print(x.size())\n",
        "    #print(w.size())\n",
        "    #print(t.size())\n",
        "    #print(self.h.device, self.c.device)\n",
        "    out, (self.h, self.c) = self.LSTM(x, (self.h, self.c))\n",
        "    #print(self.h.dtype, self.c.dtype)\n",
        "    out = self.fc(out) #self.fc(out[:, -1, :])  \n",
        "    out = torch.cat((t, out), 2)\n",
        "    return out\n",
        "\n",
        "\n",
        "\n",
        "class IMU_Preprocess(nn.Module):\n",
        "  def __init__(self, param):\n",
        "    super(IMU_Preprocess, self).__init__()\n",
        "\n",
        "    # RNN block\n",
        "    self.bi_lstm = Bi_LSTM(param)\n",
        "    \n",
        "  \n",
        "  def forward(self, x):\n",
        "    # Propagate through RNN block\n",
        "    self.bi_lstm.reset_hidden_states(zeros=False)                               # Reset LSTM hidden states --> zeros=False ???\n",
        "    out = self.bi_lstm(x)\n",
        "    # Propagate through integration block\n",
        "    gamma, beta, C = self.integration_block(out)\n",
        "\n",
        "    return gamma, beta, C\n",
        "\n",
        "\n",
        "  \n",
        "  def integration_block(self, imu_cl):\n",
        "    seq_len = imu_cl.size(1)\n",
        "    num_batches = imu_cl.size(0)\n",
        "\n",
        "    C_accum = torch.eye(3, 3, device=imu_cl.device).repeat(num_batches, 1, 1)\n",
        "    gamma_accum = torch.zeros(num_batches, 3, 1, device=imu_cl.device)\n",
        "    beta_accum = torch.zeros(num_batches, 3, 1, device=imu_cl.device)\n",
        "\n",
        "    for i in range(0, seq_len-1):\n",
        "      t = imu_cl[:, i, 0].view(-1, 1, 1)\n",
        "      t1 = imu_cl[:, i+1, 0].view(-1, 1, 1)\n",
        "      w = imu_cl[:, i, 1:4].view(-1, 3, 1)      # gyro meas\n",
        "      a = imu_cl[:, i, 4:7].view(-1, 3, 1)      # accel meas\n",
        "      \n",
        "      dt = t1 - t\n",
        "      dt2 = dt * dt\n",
        "      exp_int_w = torch_se3.exp_SO3_b(dt * w)\n",
        "\n",
        "      gamma_accum = gamma_accum +  beta_accum*dt + 0.5*torch.matmul(C_accum, (dt2*a))\n",
        "      beta_accum = beta_accum + torch.matmul(C_accum, (dt * a))\n",
        "      C_accum = torch.matmul(C_accum, exp_int_w)\n",
        "      \n",
        "    #print(gamma_accum.size(), beta_accum.size(), C_accum.size())\n",
        "    return (gamma_accum, beta_accum, C_accum)  \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "  input_size = 6\n",
        "  hidden_size = 6 # NOT GIVEN IN THE PAPER\n",
        "  num_layers = 2  # NOT GIVEN IN THE PAPER\n",
        "  num_classes = 6\n",
        "  batch_size = 32\n",
        "  seq_len = 8\n",
        "  x = torch.rand((32, 8, 7))        # (batch_size, seq_len, feature #)\n",
        "  #print(x.dtype)\n",
        "  #bi_lstm = Bi_LSTM(input_size, hidden_size, num_layers, num_classes)\n",
        "  model = IMU_Preprocess(param)\n",
        "  model.to(param['device'])\n",
        "  #x = x.permute(1,0,2)               # (seq_len, batch_size, feature #)\n",
        "  #for i in range(seq_len):\n",
        "  #  out = bi_lstm(x[i])\n",
        "  #  print(out.shape)\n",
        "  gamma, beta, C = model(x.to(param['device']))                  # (batch_size, feature #)\n",
        "  #print(gamma.shape, beta.shape, C.shape)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mEfwFn5psLi1"
      },
      "source": [
        "train_loader = DataLoader(\n",
        "        train_dataset,\n",
        "        batch_size=32, #params[\"batch_size\"],\n",
        "        #sampler = train_sampler,\n",
        "        shuffle=False,                                                          # Shuffle might be True !\n",
        "        num_workers= 4, #params[\"num_workers\"],\n",
        "        drop_last=True,\n",
        "        pin_memory=True )\n",
        "i = 0\n",
        "for i, (imu, gamma, beta, C) in enumerate(train_loader):\n",
        "  imu = imu.type(torch.float32)\n",
        "  #print(imu.dtype, gamma.dtype, beta.dtype, C.dtype)\n",
        "  #imu_torch = torch.from_numpy(imu)\n",
        "  #gamma_torch = torch.from_numpy(gamma)\n",
        "  #beta_torch = torch.from_numpy(beta)\n",
        "  #C_torch = torch.from_numpy(C)\n",
        "  #print(imu_torch.size(), gamma_torch.size(), beta_torch.size(), C_torch.size())\n",
        "  print(imu.size())\n",
        "  i += 1\n",
        "  if imu.size(0) != 32:\n",
        "    print(len(tr_loader.dataset))\n",
        "    break\n",
        "\n",
        "print(i)\n",
        "print(len(train_dataset))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "boKQ5xOwtGSo"
      },
      "source": [
        "def train(train_loader, model, optimizer, epoch, param):\n",
        "  model.train()\n",
        "  running_loss = 0.0\n",
        "  start_time = time.time()\n",
        "\n",
        "  for i, (imu_data, gt_gamma, gt_beta, gt_C) in enumerate(train_loader):\n",
        "    imu_data = imu_data.type(torch.float32)\n",
        "    gt_gamma = gt_gamma.type(torch.float32)\n",
        "    gt_beta = gt_beta.type(torch.float32)\n",
        "    gt_C = gt_C.type(torch.float32)\n",
        "    \n",
        "    imu_data = imu_data.to(param['device'], non_blocking=True)\n",
        "    gt_gamma = gt_gamma.to(param['device'], non_blocking=True)\n",
        "    gt_beta = gt_beta.to(param['device'], non_blocking=True)\n",
        "    gt_C = gt_C.to(param['device'], non_blocking=True)\n",
        "\n",
        "    #model.reset_hidden_states(zeros=False)\n",
        "    gamma, beta, C = model(imu_data)\n",
        "\n",
        "    loss = L_p(gamma, gt_gamma) + L_v(beta, gt_beta) + L_rot(C, gt_C)\n",
        "    running_loss += float(loss.item()) * imu_data.size(0)\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "  \n",
        "  tr_time = time.time() - start_time\n",
        "  return (running_loss/len(train_loader.dataset)), tr_time\n",
        "\n",
        "\n",
        "\n",
        "def validate(val_loader, model, epoch, param):\n",
        "  model.eval()\n",
        "  running_loss = 0.0\n",
        "  start_time = time.time()\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for i, (imu_data, gt_gamma, gt_beta, gt_C) in enumerate(val_loader):\n",
        "      imu_data = imu_data.type(torch.float32)\n",
        "      gt_gamma = gt_gamma.type(torch.float32)\n",
        "      gt_beta = gt_beta.type(torch.float32)\n",
        "      gt_C = gt_C.type(torch.float32)  \n",
        "\n",
        "      imu_data = imu_data.to(param['device'], non_blocking=True)\n",
        "      gt_gamma = gt_gamma.to(param['device'], non_blocking=True)\n",
        "      gt_beta = gt_beta.to(param['device'], non_blocking=True)\n",
        "      gt_C = gt_C.to(param['device'], non_blocking=True)\n",
        "\n",
        "      #model.reset_hidden_states(zeros=False)\n",
        "      gamma, beta, C = model(imu_data)\n",
        "\n",
        "      loss = L_p(gamma, gt_gamma) + L_v(beta, gt_beta) + L_rot(C, gt_C)\n",
        "      running_loss += float(loss.item()) * imu_data.size(0)\n",
        "\n",
        "  val_time = time.time() - start_time \n",
        "  return (running_loss/len(val_loader.dataset)), val_time\n",
        "\n",
        "\n",
        "\n",
        "def train_and_validate(model, train_dataset, val_dataset, param):\n",
        "  train_loader = DataLoader(\n",
        "        train_dataset,\n",
        "        batch_size=param[\"batch_size\"],\n",
        "        #sampler = train_sampler,\n",
        "        shuffle=True,\n",
        "        num_workers=param[\"num_workers\"],\n",
        "        drop_last = True,\n",
        "        pin_memory=True,\n",
        "    )\n",
        "  val_loader = DataLoader(\n",
        "        val_dataset,\n",
        "        batch_size=param[\"batch_size\"],\n",
        "        #sampler = valid_sampler,\n",
        "        shuffle=False,\n",
        "        num_workers=param[\"num_workers\"],\n",
        "        drop_last = True,\n",
        "        pin_memory=True,\n",
        "    )\n",
        "\n",
        "\n",
        "  optimizer = torch.optim.Adam(model.parameters(), lr=param['lr'])\n",
        "  # scheduler  = torch.optim.lr_scheduler.StepLR(optimizer, step_size=8, gamma=0.1)\n",
        "\n",
        "  tr_epoch_loss = []\n",
        "  val_epoch_loss = []\n",
        "  for epoch in range(param['epoch']):\n",
        "    tr_loss, tr_time = train(train_loader, model, optimizer, epoch, param)\n",
        "    val_loss, val_time = validate(val_loader, model, epoch, param)\n",
        "\n",
        "    tr_epoch_loss.append(tr_loss)\n",
        "    val_epoch_loss.append(val_loss)\n",
        "\n",
        "    f = open('records.txt', 'a')\n",
        "    f.write('Epoch: {}, Train_Loss: {}, Train_Time: {}, Val_Loss: {}, Val_Time: {}'.format(epoch, tr_loss, tr_time, val_loss, val_time))\n",
        "    f.close()\n",
        "    # Save model and optimizer state_dicts\n",
        "    torch.save(model.state_dict(), os.path.join('Training_Records', 'model_state.weights'))\n",
        "    torch.save(optimizer.state_dict(), os.path.join('Training_Records', 'optimizer_state.weights'))\n",
        "    print('Epoch: {}, Train_Loss: {}, Tr_Time: {}, Val_Loss: {}, Val_Time: {}'.format(epoch, tr_loss, tr_time, val_loss, val_time))\n",
        "\n",
        "  stat = {'tr_loss':tr_epoch_loss, 'val_loss':val_epoch_loss}\n",
        "\n",
        "  return model, stat\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SQa6A19apRYh",
        "outputId": "aae18c3a-d7a4-48c3-b30a-be7bf3fc87bb"
      },
      "source": [
        "\n",
        " \n",
        "model = IMU_Preprocess(param).to(param['device'])\n",
        "model, stat = train_and_validate(model, train_dataset, val_dataset, param)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0, Train_Loss: 0.030610643587491635, Tr_Time: 44.065940141677856, Val_Loss: 0.003319738485115385, Val_Time: 9.927104711532593\n",
            "Epoch: 1, Train_Loss: 0.0010761007224944815, Tr_Time: 44.539416790008545, Val_Loss: 0.0033371461338793217, Val_Time: 10.01361083984375\n",
            "Epoch: 2, Train_Loss: 0.0013983134853724586, Tr_Time: 44.414087772369385, Val_Loss: 0.00327751205588287, Val_Time: 10.00849986076355\n",
            "Epoch: 3, Train_Loss: 0.001740368436796018, Tr_Time: 44.50664758682251, Val_Loss: 0.003289818018285339, Val_Time: 10.018478870391846\n",
            "Epoch: 4, Train_Loss: 0.001123611399826621, Tr_Time: 45.00788402557373, Val_Loss: 0.0031979518833471637, Val_Time: 10.062824249267578\n",
            "Epoch: 5, Train_Loss: 0.0014107910265430432, Tr_Time: 44.31614875793457, Val_Loss: 0.002839904077275528, Val_Time: 10.100136756896973\n",
            "Epoch: 6, Train_Loss: 0.0011325756767094591, Tr_Time: 44.477285623550415, Val_Loss: 0.0027886053943007075, Val_Time: 10.071435689926147\n",
            "Epoch: 7, Train_Loss: 0.0009373341835488481, Tr_Time: 45.00946521759033, Val_Loss: 0.0027666591961655043, Val_Time: 10.065989255905151\n",
            "Epoch: 8, Train_Loss: 0.0008325806573717746, Tr_Time: 44.390215158462524, Val_Loss: 0.0026899680017331237, Val_Time: 10.131028890609741\n",
            "Epoch: 9, Train_Loss: 0.0012619112490435988, Tr_Time: 44.50616264343262, Val_Loss: 0.0026654180947345315, Val_Time: 10.1017427444458\n",
            "Epoch: 10, Train_Loss: 0.0008729815874888655, Tr_Time: 44.30663585662842, Val_Loss: 0.002645321684102292, Val_Time: 9.853741645812988\n",
            "Epoch: 11, Train_Loss: 0.000785055218826928, Tr_Time: 44.65993356704712, Val_Loss: 0.0026278851849826645, Val_Time: 9.931585311889648\n",
            "Epoch: 12, Train_Loss: 0.0009332579184650922, Tr_Time: 44.12377095222473, Val_Loss: 0.002620898121391434, Val_Time: 9.826382637023926\n",
            "Epoch: 13, Train_Loss: 0.0014720114608339158, Tr_Time: 44.16645407676697, Val_Loss: 0.0025930537575009054, Val_Time: 10.030859231948853\n",
            "Epoch: 14, Train_Loss: 0.0008611204552492906, Tr_Time: 45.031731605529785, Val_Loss: 0.0025401528645228925, Val_Time: 10.22313928604126\n",
            "Epoch: 15, Train_Loss: 0.0012194761634571542, Tr_Time: 44.65553379058838, Val_Loss: 0.0024471688072813737, Val_Time: 10.026320457458496\n",
            "Epoch: 16, Train_Loss: 0.000768254940174758, Tr_Time: 45.00239944458008, Val_Loss: 0.002486408360003303, Val_Time: 10.068777084350586\n",
            "Epoch: 17, Train_Loss: 0.0007752110238867202, Tr_Time: 45.045032262802124, Val_Loss: 0.002523945059568089, Val_Time: 10.04457950592041\n",
            "Epoch: 18, Train_Loss: 0.001056283427207816, Tr_Time: 45.16067576408386, Val_Loss: 0.0023943696557440952, Val_Time: 9.92343807220459\n",
            "Epoch: 19, Train_Loss: 0.0008102834312839799, Tr_Time: 44.94035720825195, Val_Loss: 0.0025259477684265747, Val_Time: 10.26227593421936\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "5VjlOXh8gkkZ",
        "outputId": "d23f0617-e03e-49d3-c4cc-14005cedf638"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "tr_loss = stat['tr_loss']\n",
        "val_loss = stat['val_loss']\n",
        "plt.plot(val_loss, label='Validation')\n",
        "plt.plot(tr_loss, label='Training')\n",
        "plt.legend()\n",
        "plt.title('Loss vs Epoch Graph')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xcdZ3/8dcnk1szSW9JufQCaW0LtlZ6CUW5CaIuIFLFIq3sAgsPEZRlxVUXXUWW1cdvUXaXRcHfgijIioVlF6xr+eFyW1AUKFguBbqEEpYU6J3ekyaZz++P75lkMp2kM8lMJs28n4/Hecw53/M9Z77nZDKf+X6/53yPuTsiIiLZKit2AURE5MCiwCEiIjlR4BARkZwocIiISE4UOEREJCcKHCIikhMFDpEDlJldYGa/LXY5+mNmV5vZvxa7HJJfChwybJhZi5l9pNjlGAgzO8nMEma2M236YLHLlsrMlpjZk2a2y8w2RPNfMDMrdtnkwKHAIZI/b7l7bdr0+2IXKsnM/gr4Z+D7wCHAwcAlwHFAZR/bxIasgHLAUOCQYc/MqszsejN7K5quN7OqaF2Dmf2nmb1rZlvM7HEzK4vW/bWZrTOzHWa2xsxOybDvY8zsndQvSDP7lJk9H80vNLOVZrbdzNab2T8O8BgeNbP/Y2ZPRfv6pZmNT1l/ppmtjo7jUTN7b8q6KWb2H2a20cw2m9kP0/Z9nZltNbPXzey0Pt5/DHAN8AV3v8fdd3jwR3c/193bo3y3mdmPzGyFme0CTjazj5vZH6Nyv2lmV6fst9HM3Mwujv42b5vZV9LevtLMfhb9HVabWdNAzqEMHwocciD4G+ADwFzgKGAh8M1o3V8BrcAEwi/obwBuZkcAlwFHu3sd8CdAS/qO3f1JYBfw4ZTkzwJ3RvP/DPyzu48G3gPcPYjjOA+4EDgU6ARuADCzmcAvgC9Fx7EC+JWZVUYB7T+BN4BGYBKwLGWfxwBrgAbge8CtfTQ7fRCoAn6ZRTk/C3wXqAN+Szg/5wFjgY8Dl5rZJ9O2ORmYAXwM+Ou0JsczozKPBZYDP0QOaAocciA4F7jG3Te4+0bgb4E/i9Z1EL6ID3f3Dnd/3MMAbF2EL8pZZlbh7i3u/lof+/8FsBTAzOqA06O05P6nm1mDu+909z/0U86JUY0hdYqnrL/D3V90913At4DPRIHhHODX7v5f7t4BXAeMAo4lBMmJwFfdfZe7t7l7aof4G+5+i7t3AbdH5+LgDGVrADa5e2cywcyeiMq4x8xOTMn7S3f/nbsnovd71N1fiJafj87Nh9L2/7dR+V4Afpo8n5HfuvuKqIx3EIK/HMAUOORAMJHwizvpjSgNQnt9M/AbM1trZlcCuHsz4Rf81cAGM1tmZhPJ7E7grKj56yzgWXdPvt9FwEzgFTN72szO6Kecb7n72LRpV8r6N9OOoYLwhd7r+Nw9EeWdBEwhBIdOMnsnZbvd0WxthnybgQYzK0/Jf6y7j43WpX4XpJYz2Zz3SNRUto3QL9KQtv/0Y0s91++kzO8GqlPLIQceBQ45ELwFHJ6yfFiURtRW/1fuPo3QJPLlZF+Gu9/p7sdH2zpwbaadu/tLhC+70+jdTIW7v+ruS4GDou3vSatF5GJK2jF0AJvSjy9qapoCrCN8IR+Why/a3wPtwKIs8qYPmX0noYlpiruPAf4vkN4cln5sbw2wnHIAUOCQ4abCzKpTpnJC08g3zWyCmTUAVwH/CmBmZ5jZ9OjLdhuhiSphZkeY2YejWkQbsAdI9PO+dwJ/CZwI/Fsy0cz+1MwmRLWAd6Pk/vbTnz81s1lmVkPoqL4nar65G/i4mZ1iZhWEfpt24AngKeBt4O/NLB6dk+NyfWN3f5fQxHeTmS02szozKzOzucD+AmEdsMXd28xsISG4pvuWmdWY2Wzgz4G7ci2jHDgUOGS4WUH4kk9OVwPfAVYCzwMvAM9GaRA6ZB8EdhJ+Vd/k7o8Q+jf+nvCL/h1CjeHr/bxvst3+YXfflJJ+KrDazHYSOsqXuPuePvYx0fa9j+PTKevvAG6LylMNXA7g7muAPwV+EJX3E8An3H1vFFg+AUwH/pdwIcA5/RxHn9z9e8CXga8B66PpX4C/JgSpvnwBuMbMdhCCdqYLBP6b0GT4EHCdu/9mIGWUA4PpQU4ihWdmjwL/6u4/LnZZ8snMGoHXgYp++mFkhFGNQ0REcqLAISIiOVFTlYiI5EQ1DhERyUlJ3ITT0NDgjY2NxS6GiMgB5Zlnntnk7hPS00sicDQ2NrJy5cpiF0NE5IBiZm9kSldTlYiI5ESBQ0REcqLAISIiOSmJPg4RGTk6OjpobW2lra2t2EUZMaqrq5k8eTIVFRVZ5VfgEJEDSmtrK3V1dTQ2NpL5mVWSC3dn8+bNtLa2MnXq1Ky2UVOViBxQ2traqK+vV9DIEzOjvr4+pxqcAoeIHHAUNPIr1/OpwNGfJ2+GF+4pdilERIYVBY7+PHs7vPjvxS6FiAwjJ598Mg888ECvtOuvv55LL700Y/6TTjqp+wbk008/nXfffXefPFdffTXXXXddv+9733338dJLL3UvX3XVVTz44IO5Fj8vCho4zOxUM1tjZs3JZ0Gnra8ys7ui9U9GY/tjZgvNbFU0PWdmn8p2n3lVUw+7Nu0/n4iUjKVLl7Js2bJeacuWLWPp0qX73XbFihWMHTt2QO+bHjiuueYaPvKRjwxoX4NVsMBhZjHgRsJznGcBS81sVlq2i4Ct7j4d+Cd6ngn9ItDk7nMJT2D7FzMrz3Kf+RNvgN0KHCLSY/Hixfz6179m7969ALS0tPDWW2/xi1/8gqamJmbPns23v/3tjNs2NjayaVP4Tvnud7/LzJkzOf7441mzZk13nltuuYWjjz6ao446ik9/+tPs3r2bJ554guXLl/PVr36VuXPn8tprr3HBBRdwzz2hKf2hhx5i3rx5zJkzhwsvvJD29vbu9/v2t7/N/PnzmTNnDq+88kpezkEhL8ddCDS7+1oAM1sGLAJeSsmziPBoUIB7gB+ambn77pQ81UBy7Pds9pk/NQ2wa3NBdi0ig/e3v1rNS29tz+s+Z00czbc/MbvP9ePHj2fhwoXcf//9LFq0iGXLlvGZz3yGb3zjG4wfP56uri5OOeUUnn/+ed7//vdn3MczzzzDsmXLWLVqFZ2dncyfP58FCxYAcNZZZ/G5z30OgG9+85vceuut/MVf/AVnnnkmZ5xxBosXL+61r7a2Ni644AIeeughZs6cyXnnncePfvQjvvSlLwHQ0NDAs88+y0033cR1113Hj388+IdQFrKpahLwZspya5SWMU/02MltQD2AmR1jZqsJz5i+JFqfzT6Jtr/YzFaa2cqNGzcO7AjiDdC+DTr3Dmx7ERmRUpurks1Ud999N/Pnz2fevHmsXr26V7NSuscff5xPfepT1NTUMHr0aM4888zudS+++CInnHACc+bM4ec//zmrV6/utyxr1qxh6tSpzJw5E4Dzzz+fxx57rHv9WWedBcCCBQtoaWkZ6CH3MmxvAHT3J4HZZvZe4HYzuz/H7W8GbgZoamoa2NOqaurD6+7NMPrQAe1CRAqnv5pBIS1atIgrrriCZ599lt27dzN+/Hiuu+46nn76acaNG8cFF1ww4DvbL7jgAu677z6OOuoobrvtNh599NFBlbWqqgqAWCxGZ2d+HgtfyBrHOmBKyvLkKC1jHjMrB8YAvdqG3P1lYCfwviz3mT/xhvCqfg4RSVFbW8vJJ5/MhRdeyNKlS9m+fTvxeJwxY8awfv167r+//9+5J554Ivfddx979uxhx44d/OpXv+pet2PHDg499FA6Ojr4+c9/3p1eV1fHjh079tnXEUccQUtLC83NzQDccccdfOhDH8rTkWZWyMDxNDDDzKaaWSWwBFielmc5cH40vxh42N092qYcwMwOB44EWrLcZ/7URIFDV1aJSJqlS5fy3HPPsXTpUo466ijmzZvHkUceyWc/+1mOO+64fredP38+55xzDkcddRSnnXYaRx99dPe6v/u7v+OYY47huOOO48gjj+xOX7JkCd///veZN28er732Wnd6dXU1P/3pTzn77LOZM2cOZWVlXHLJJfk/4BQFfea4mZ0OXA/EgJ+4+3fN7BpgpbsvN7Nq4A5gHrAFWOLua83sz4ArgQ4gAVzj7vf1tc/9laOpqckH9CCnDa/ATcfAp2+FOYv3n19ECu7ll1/mve99b7GLMeJkOq9m9oy7N6XnLWgfh7uvAFakpV2VMt8GnJ1huzsIASWrfRZMXDUOEZF0unO8P6PGAaY+DhGRFAoc/SmLQc141ThERFIocOxPje4eFxFJpcCxP3HdPS4ikkqBY39q6lXjEBFJocCxP/EG9XGISLfNmzczd+5c5s6dyyGHHMKkSZO6l5MDH/Zl5cqVXH755ft9j2OPPTZfxS2IYTvkyLBR0wB7tkKiK3SWi0hJq6+vZ9WqVUB4jkZtbS1f+cpXutd3dnZSXp75q7WpqYmmpn1ui9jHE088kZ/CFohqHPsTbwAcdm8pdklEZJi64IILuOSSSzjmmGP42te+xlNPPcUHP/hB5s2bx7HHHts9bPqjjz7KGWecAYSgc+GFF3LSSScxbdo0brjhhu791dbWduc/6aSTWLx4MUceeSTnnnsuyZu2V6xYwZFHHsmCBQu4/PLLu/c7FFTj2J/ugQ43Qe2E4pZFRHq7/0p454X87vOQOXDa3+e8WWtrK0888QSxWIzt27fz+OOPU15ezoMPPsg3vvEN/v3f932a6CuvvMIjjzzCjh07OOKII7j00kupqKjoleePf/wjq1evZuLEiRx33HH87ne/o6mpic9//vM89thjTJ06NauHSOWTAsf+6O5xEcnC2WefTSwWmrO3bdvG+eefz6uvvoqZ0dHRkXGbj3/841RVVVFVVcVBBx3E+vXrmTx5cq88Cxcu7E6bO3cuLS0t1NbWMm3aNKZOnQqEcbNuvvnmAh5dbwoc+1OjEXJFhq0B1AwKJR6Pd89/61vf4uSTT+bee++lpaWFk046KeM2ySHPoe9hz7PJM9TUx7E/qnGISI62bdvGpEnhGXO33XZb3vd/xBFHsHbt2u4HM9111115f4/+KHDsT3cfhzrHRSQ7X/va1/j617/OvHnzClJDGDVqFDfddBOnnnoqCxYsoK6ujjFjxuT9ffpS0GHVh4sBD6ue9PeHwfvPgdO/n79CiciAaFj1YOfOndTW1uLufPGLX2TGjBlcccUVA95fLsOqq8aRjRrdBCgiw8stt9zC3LlzmT17Ntu2bePzn//8kL23OsezEddAhyIyvFxxxRWDqmEMhmoc2ajRQIciw0kpNLEPpVzPpwJHNuIa6FBkuKiurmbz5s0KHnni7mzevJnq6uqst1FTVTZqGmD3ZnAHs2KXRqSkTZ48mdbWVjZu3FjsoowY1dXV+9x42B8FjmzEGyDRCW3vRo+TFZFiqaio6L5jWopDTVXZSN49rn4OEREFjqzEUwY6FBEpcQoc2ajRsCMiIkkKHNmIa6BDEZGkggYOMzvVzNaYWbOZXZlhfZWZ3RWtf9LMGqP0j5rZM2b2QvT64ZRtHo32uSqaDirkMQCqcYiIpCjYVVVmFgNuBD4KtAJPm9lyd38pJdtFwFZ3n25mS4BrgXOATcAn3P0tM3sf8AAwKWW7c919EINP5aiiGiprwyW5IiIlrpA1joVAs7uvdfe9wDJgUVqeRcDt0fw9wClmZu7+R3d/K0pfDYwysyqKqWa8ahwiIhQ2cEwC3kxZbqV3raFXHnfvBLYB9Wl5Pg086+7tKWk/jZqpvmWW+Y48M7vYzFaa2cq83ChUo/GqRERgmHeOm9lsQvNV6rCP57r7HOCEaPqzTNu6+83u3uTuTRMm5OFZ4XGNkCsiAoUNHOuAKSnLk6O0jHnMrBwYA2yOlicD9wLnuftryQ3cfV30ugO4k9AkVnjJYUdEREpcIQPH08AMM5tqZpXAEmB5Wp7lwPnR/GLgYXd3MxsL/Bq40t1/l8xsZuVm1hDNVwBnAC8W8Bh6xOtDjUMDq4lIiStY4Ij6LC4jXBH1MnC3u682s2vM7Mwo261AvZk1A18GkpfsXgZMB65Ku+y2CnjAzJ4HVhFqLLcU6hh6qWmArnbYu3NI3k5EZLgq6CCH7r4CWJGWdlXKfBtwdobtvgN8p4/dLshnGbMWT7mXo6quKEUQERkOhnXn+LCSvAlQ/RwiUuIUOLIV193jIiKgwJG9Go2QKyICChzZU41DRARQ4MheZS3EqlTjEJGSp8CRLbNQ69i9pdglEREpKgWOXNTUq6lKREqeAkcu4hroUEREgSMXNRroUEREgSMXcQ10KCKiwJGLmvowVlVHW7FLIiJSNAocuUjey6F+DhEpYQocuajRTYAiIgocuVCNQ0REgSMn3TUOdZCLSOlS4MhFXAMdiogocOSieiyUlauPQ0RKmgJHLszCJbmqcYhICVPgyFVNg/o4RKSkKXDkKq4ah4iUNgWOXGm8KhEpcQocuVIfh4iUOAWOXMUboG0bdHUUuyQiIkWhwJGrmuS9HOogF5HSVNDAYWanmtkaM2s2syszrK8ys7ui9U+aWWOU/lEze8bMXoheP5yyzYIovdnMbjAzK+Qx7COu8apEpLQVLHCYWQy4ETgNmAUsNbNZadkuAra6+3Tgn4Bro/RNwCfcfQ5wPnBHyjY/Aj4HzIimUwt1DBnVaLwqESlthaxxLASa3X2tu+8FlgGL0vIsAm6P5u8BTjEzc/c/uvtbUfpqYFRUOzkUGO3uf3B3B34GfLKAx7Av1ThEpMQVMnBMAt5MWW6N0jLmcfdOYBtQn5bn08Cz7t4e5W/dzz4Lq7vGoT4OESlN5cUuQH/MbDah+epjA9j2YuBigMMOOyx/haoZD5hqHCJSsgpZ41gHTElZnhylZcxjZuXAGGBztDwZuBc4z91fS8k/eT/7BMDdb3b3JndvmjBhwiAPJUVZDEaNUx+HiJSsQgaOp4EZZjbVzCqBJcDytDzLCZ3fAIuBh93dzWws8GvgSnf/XTKzu78NbDezD0RXU50H/LKAx5BZXHePi0jpKljgiPosLgMeAF4G7nb31WZ2jZmdGWW7Fag3s2bgy0Dykt3LgOnAVWa2KpoOitZ9Afgx0Ay8BtxfqGPoU00D7N4y5G8rIjIcFLSPw91XACvS0q5KmW8Dzs6w3XeA7/Sxz5XA+/Jb0hzF62HTq0UtgohIsejO8YHQQIciUsIUOAYi3gB7tkAiUeySiIgMOQWOgahpAE/Anq3FLomIyJBT4BiIuIYdEZHSpcAxEMkRctXPISIlSIFjIFTjEJESpsAxEDUa6FBESpcCx0DoYU4iUsIUOAaivBKqxqjGISIlSYFjoOL16uMQkZKkwDFQuntcREqUAsdAxRvUxyEiJUmBY6Bq6lXjEJGSpMAxUMkah3uxSyIiMqQUOAaqpgESHdC2rdglEREZUlkFDjOLm1lZND/TzM40s4rCFm2Y6757XP0cIlJasq1xPAZUm9kk4DfAnwG3FapQBwSNVyUiJSrbwGHuvhs4C7jJ3c8GZheuWAeA7rvHFThEpLRkHTjM7IPAucCvo7RYYYp0gIhrvCoRKU3ZBo4vAV8H7nX31WY2DXikcMU6ANRohFwRKU3l2WRy9/8G/hsg6iTf5O6XF7Jgw15lDVTUwC51jotIacn2qqo7zWy0mcWBF4GXzOyrhS3aAaCmQTUOESk52TZVzXL37cAngfuBqYQrq0pbXHePi0jpyTZwVET3bXwSWO7uHYBumVaNQ0RKULaB41+AFiAOPGZmhwPb97eRmZ1qZmvMrNnMrsywvsrM7orWP2lmjVF6vZk9YmY7zeyHads8Gu1zVTQdlOUx5F+8AXZvKdrbi4gUQ7ad4zcAN6QkvWFmJ/e3jZnFgBuBjwKtwNNmttzdX0rJdhGw1d2nm9kS4FrgHKAN+BbwvmhKd667r8ym7AWlgQ5FpARl2zk+xsz+0cxWRtM/EGof/VkINLv7WnffCywDFqXlWQTcHs3fA5xiZubuu9z9t4QAMnzFG6BzD+zdVeySiIgMmWybqn4C7AA+E03bgZ/uZ5tJwJspy61RWsY87t4JbAPqsyjPT6Nmqm+ZmWWRvzBqdBOgiJSebAPHe9z921HtYa27/y0wrZAF68e57j4HOCGaMl7dZWYXJ2tIGzduLExJ4roJUERKT7aBY4+ZHZ9cMLPjgD372WYdMCVleXKUljGPmZUDY4B+76hz93XR6w7gTkKTWKZ8N7t7k7s3TZgwYT9FHaDuGoduAhSR0pFV5zhwCfAzMxsTLW8Fzt/PNk8DM8xsKiFALAE+m5ZnebSf3wOLgYfd+34yUhRcxrr7pujy4DOAB7M8hvyLa6BDESk92V5V9RxwlJmNjpa3m9mXgOf72abTzC4DHiAMiPiTaJyra4CV7r4cuBW4w8yagS2E4AKAmbUAo4FKM/sk8DHgDeCBKGjECEHjlhyPOX/UxyEiJSjbGgcQAkbK4peB6/eTfwWwIi3tqpT5NuDsPrZt7GO3C7Ip65CoqoNYpWocIlJSBvPo2OJdzTRcmIVah/o4RKSEDCZwaMgRCP0cqnGISAnpt6nKzHaQOUAYMKogJTrQ1DSoj0NESkq/gcPd64aqIAeseANsfb3YpRARGTKDaaoSUB+HiJQcBY7BitfD3h3Q2V7skoiIDAkFjsHSvRwiUmIUOAZL41WJSIlR4Bgs1ThEpMQocAxWd41DHeQiUhoUOAarJhroUDUOESkRChyDVT0WLKY+DhEpGQocg1VWBjXjVeMQkZKhwJEPNQ3q4xCRkqHAkQ9xjVclIqVDgSMfajRCroiUDgWOfFCNQ0RKiAJHPtQ0QNu70NVZ7JKIiBScAkc+JG8C3LOluOUQERkCChz5oJsARaSEKHDkgwY6FJESosCRDxroUERKiAJHPmigQxEpIQoc+TBqfHhVjUNESoACRz7EymHUOPVxiEhJKGjgMLNTzWyNmTWb2ZUZ1leZ2V3R+ifNrDFKrzezR8xsp5n9MG2bBWb2QrTNDWZmhTyGrNXoJkARKQ0FCxxmFgNuBE4DZgFLzWxWWraLgK3uPh34J+DaKL0N+BbwlQy7/hHwOWBGNJ2a/9IPQFwDHYpIaShkjWMh0Ozua919L7AMWJSWZxFwezR/D3CKmZm773L33xICSDczOxQY7e5/cHcHfgZ8soDHkL2aetU4RKQkFDJwTALeTFlujdIy5nH3TmAbUL+ffbbuZ58AmNnFZrbSzFZu3Lgxx6IPQLxBfRwiUhJGbOe4u9/s7k3u3jRhwoTCv2FNA+zeAolE4d9LRKSIChk41gFTUpYnR2kZ85hZOTAG6K+jYF20n/72WRzxBvCuMNihiMgIVsjA8TQww8ymmlklsARYnpZnOXB+NL8YeDjqu8jI3d8GtpvZB6Krqc4Dfpn/og+A7h4XkRJRXqgdu3unmV0GPADEgJ+4+2ozuwZY6e7LgVuBO8ysGdhCCC4AmFkLMBqoNLNPAh9z95eALwC3AaOA+6Op+OJR18zuTcDMohZFRKSQChY4ANx9BbAiLe2qlPk24Ow+tm3sI30l8L78lTJPVOMQkRIxYjvHh5xGyBWREqHAkS/dz+TQTYAiMrIpcORLeRVUjVaNQ0RGPAWOfNLd4yJSAhQ48kl3j4tICVDgyKeaBvVxiMiIp8CRTzX1qnGIyIinwJFP8aiPo++b30VEDngKHPlU0wCJDmjfXuySiIgUjAJHPnXfBKh+DhEZuRQ48ql72BEFDhEZuRQ48qnXQIciIiOTAkc+aaBDESkBChz5pIEORaQEKHDkU2UcykepxiEiI5oCR77FG3RVlYiMaAoc+aaBDkVkhFPgyDcNdCgiI5wCR75poEMRGeEUOPJNNQ4RGeEUOPKtph46dsPe3cUuiYhIQShw5Jvu5RCREU6BI99097iIjHAKHPmmEXJFZIQraOAws1PNbI2ZNZvZlRnWV5nZXdH6J82sMWXd16P0NWb2JynpLWb2gpmtMrOVhSz/gNREAx2qxiEiI1R5oXZsZjHgRuCjQCvwtJktd/eXUrJdBGx19+lmtgS4FjjHzGYBS4DZwETgQTOb6e5d0XYnu/vw/GZWH4eIjHCFrHEsBJrdfa277wWWAYvS8iwCbo/m7wFOMTOL0pe5e7u7vw40R/sb/qpGQ1mFahwiMmIVMnBMAt5MWW6N0jLmcfdOYBtQv59tHfiNmT1jZhf39eZmdrGZrTSzlRs3bhzUgeTETPdyiMiIdiB2jh/v7vOB04AvmtmJmTK5+83u3uTuTRMmTBjaEurucREZwQoZONYBU1KWJ0dpGfOYWTkwBtjc37bunnzdANzLcGzCiterxiEiI1YhA8fTwAwzm2pmlYTO7uVpeZYD50fzi4GH3d2j9CXRVVdTgRnAU2YWN7M6ADOLAx8DXizgMQxMTYP6OERkxCrYVVXu3mlmlwEPADHgJ+6+2syuAVa6+3LgVuAOM2sGthCCC1G+u4GXgE7gi+7eZWYHA/eG/nPKgTvd/f8V6hgGTM/kEJERrGCBA8DdVwAr0tKuSplvA87uY9vvAt9NS1sLHJX/kuZZTQO0b4fOdiivKnZpRETy6kDsHB/+4tFNgKp1iMgIpMBRCLp7XERGMAWOQqjReFUiMnIpcBSCBjoUkRGsoJ3jB7qv/ttztG7dQ8Idd8Ir4TXh4O5hPhHSiNbVdm3nP4Af/Or33PebBipiZVSVl1ERC1NlefLVqNwnraw7raLcKDPDHZxQBo/KkixHcr4nPVm2MF9eZoyqiFFdEWNUZYzq8vA6qiJGVUUZoyp6lquT+SpiVMSM6Oo1EZFeFDj60dGVoCvhmEGszCi38EVuBmVmlEWv1p0WlmPESTSXMaOunSPrR7O3M0FHV5j2dibYvadrn7Tk696uBB1dTlfCcyqrGRh0l88ICV2J3PcF4Xiry8uor61iakOcqQ1xpk2Id89PHDOKsjIFFpFSpMDRj+uXzBv4xt8bz6mN5Zz6ifkD2rwr4XR0JUi4Y0TBIAoIycBlRGn91AzcnY4up62zi7a9Xezp6KKtI8Geji727O2irSNMezp61rVF6/Z0dLFhRzuvbx14KcUAABFSSURBVNrJypYt7Nrb1b3fyvIyptbHaWyoYWpDLdMa4kyNAkt9vFK1FZERTIGjUAY50GGszIiVxQZdDDMLTWLlZYyurhjwftydjTvaWbtpF69H09qNu2jesJOHX9lAR1dPraauujwEkoY4E+qqKI+VUVFmVMTKwnzMupvtymOhua48ZpSXhea78rJks55F56GnppecL+uu9fWxLpovjxlVUfOfgplIfihwFMoIG+jQzDhodDUHja7mA9Pqe63r7Eqw7t09Iahs3EXL5hBYnm7ZypZde+lMJHoFlmKpKg99TdVR/05Veaw7rao8mRbmq6P15TEjZiF4mRmxsp6AFQJVCFKxZBCL0nqCWO9A1tO8SU7rrY/X1Lyp2ySbTknWVumpmXbXVImWo3ja82q98pSV0X0OkseaPL7kOVBQLi0KHIUSr4cNLxe7FEOiPFbG4fVxDq+Pc/IRmfO4O50Jp7PL2duVoDPqy+noStAZNct1RGmdXYkoT+ifSV6M0JXw6IIE6HLvvjihK5G8UGDffB1dCdo7E7R3dIXXzgTtnV20d6TMdyZo70iwZdfeKL0nb0dngq60iyC6ogsRpIdZCC69AwvdNcxQm+yZryjvqYWmzidroN3z5dEFHMmLO3rNl/VaTuarrkxe4NH7otFEwulIhH7LzoTT1RW9Rp+/7vSE05kIn7/U5Uzb9UrvXp+gy6ErkQAgXlVObVU5ddXl1FZVEK+KUVdVQW11SK8sH9jFrYmEs2tvJzvbO9nZ1smO6DV9+fJTpuc9sCtwFIoGOuzFzKImKhjF4Jvgii01SCW6A1hISyR8n2CTemVewvcNdH2t70r0vFf3K2n7SKReZZe6rudqv2Sg67k6L7kcXZHXfWA9eYDuIJ1I9ATxruTxJXqXPzU9GcyTX8DJHwKpPxCS83v2dIUfEFFaat72ztAfN9ALPCpi1v2FPlyDfWWsrDuIdE/RckWsjF3tIRiEQNDBzvZOdrV3sbO9M6v9f+7EqdRU5verXoGjUOINsGcrJLogD30VMryYGbGoSUoKr6Mr0X0RR9veRPfFHJku8Eim7enooqMrXJJeXmbEypJ9aaFGVF5mxGJl3cuhT62s9/qyUEOKlYX+sozpyeXYvukJ9+hLvoOd7V1RjaCDHW2dvQLCrtTaQnsnG3a0sXZjJ3s7E91BZMyoCiaPHdUrsNRVl2dY7qnNxCtjlMfyf7ueAkeh1DQADru3QO0QP0hKZIRJXkxRN4gLPIollLm62MXIK905XijdAx2quUpERhbVOAolOV5VMfo53EMz2Za1UFkLE47ouWRGRGSQFDgKpXu8qgIFDvcQlLasTZteC69t21LKchA0Hg9TT4CpH4Lx0xRIRGTAFDgKJR81DnfYuWHfoLBlLWx5PTwsKsnKYMyUEBTetxjq3wPjpobA9frj0PI4rP6PkLduYhREToTGE2Dc4QMvY3/27gYcKuOF2b8MT+7hM/rmU9C+Aw7/IBw0G8rUMj5SKHAUSs348Lq/EXI72uDd/4WtLZmnjl09eS0WvuTHT4Mpx8D494T58dNg7GFQXpn5PeafF/6ZNzfD64+FINL8EDx/V1g/9jBoPDEEkqknwOiJ+z8+d9i1Eba9Cdta4d3oddubPWnJYx81DkZPhjGTYcyk8Jq6XHcoxA68Ts/96miDRAdU1RW7JIXVvhPeejYEitanw5T+uR81Dg4/LvqxcjxMeK8CyQFMgaNQYhVQPTbUOHZu6DswbH8LSLnAvKIGxjWGadqHQq2hPgoOY6YM/AvWDBpmhOnoi8IX/4aXQxB5/TF45T9h1b+GvOPf01MjqWmIAkIrbPvf3kGiq733e1TEYeyUUM6J88M8wLZ1sH1dCCj/+3toezetbGVQe0hKUJkU9pEMKjX1oemvsnZ4NrG5w7tvwPqXYP1q2LA6zG9uBu8KNbwJM2HCkdAwM/Q5NRwRjmk4Hk9/krWJ1qejQPFUOGYPN7vRMBNmngZTjobJC6GqFlp+By2/hZbocwbhb9p4fKjxNp4w+H64RFf4Abb5tXDek1PHbjhoFhzyPjjk/WG+qnbw56HEmQ/Xu2LyqKmpyVeuXDn0b/yDBeHDm65uYk9wGNcI46f2zMcnFOfLJJGA9S/0NGu1/A727uidp/bg6At9ck+ASC6PmRx+VWZT9vadUSCJAlLqfHK5s23f7WJVURCpD681DT1BpaY+ZT65PD7/99DseRc2RAEiOW14ufe5Gns4HPw+OHgWVIyCjf8Dm9aE19Qa5Khx+waTCTNDbWy4/BrfuwvWPRsCxJvJ2kTU/FpZB5MXhAAxZSFMWtBT0+7L1jeiIPJ4+Kxtbw3pyX64xuPDD5b66ft+ltxh5/regWHza2Ha+jp07e3JWzUmNNdWjAp/o+4fKxbSD5kT/kaHvD/M1x1SvCDuHmpoqf8D294M/wftO8L/Xd0h4UdU6mvtwQWvqZvZM+7etE+6AkcBvXAPtK6MAkMUHMYeBhUHwDXdXZ3wznPhg5sMDuVVQ/Pe3f9Ib4ba2q5NYXn3pjD+V/d8lJ7a19OLQdVoqKwJNbnKmlAr6l6O97ymzqfmbdsW1SCiWkTyiw6gekz48jloFhw8O0wHvbfvpin38GWw8ZXewWTjK7BnS0++inioGU44Ipz7iupQpvLq8EVYMQrKR4X08lEpaVG+ZHoy+HS0heNo2xbOVdu7Pctt29PWpaXvXB9qTQD1M0KAmHx0eJ1w5OACs3uodbc8HoLJ64/DjrfCutpDQhAZ1xiCQjJI7N3Zs32sKgSB+veEQFM/PdSW66f3rs25hy/jd16A9S/CO8+H+a0tPfuqaQgB5JCUYFI/A2J5aJTp2BNq3ckm3G2t4XOUGijSfyiVV4ead1VdaBLe8U7P36GbhePcJ6gc2nu59qABB0UFjmIEDhkanXt7gsnuzSmBZnP48tu7K0wdu1Ned4df/3t3h+WO3X3vv6w81AYOjgLEQVGQGD0xf79Sd22CjWt6gsmmNWF5x9s9zUC5ikV9Xqm/xDMpKw9BsHpMCLTJ+erRoXY8+WiY3LT/2sRgJZvBkrWRlsfDl+bYw3oCQ/30nkAx2JpZ2/bwg+CdF3qCyYaXe5pgY1Xhvc16xmghOV5LcswWT0lLrqdnfcfuDP2cFtXeJ/cxTQk15tTPViIRPt873g5BpK/XnRvo1fQN8PXWAfezKXAocEh/Egno3LNvQKkYFX559nXhQaG5Q1dHKFtHNHW2RcGuLUpvi9KT87ujPHsATwkEY9MCQxQcKmqGZ1+LOyQ6h/bCia5O2PxqTzB5982QbgZY7/nuc2Z9r49VRn13KU26dRML93nq6gjBIxlMdq4PfZoDpMChwCEikpO+AkdBe+DM7FQzW2NmzWZ2ZYb1VWZ2V7T+STNrTFn39Sh9jZn9Sbb7FBGRwipY4DCzGHAjcBowC1hqZrPSsl0EbHX36cA/AddG284ClgCzgVOBm8wsluU+RUSkgApZ41gINLv7WnffCywDFqXlWQTcHs3fA5xi4Ykji4Bl7t7u7q8DzdH+stmniIgUUCEDxyTgzZTl1igtYx537wS2AfX9bJvNPgEws4vNbKWZrdy4ceMgDkNERFINk7uM8s/db3b3JndvmjBBz8MQEcmXQgaOdcCUlOXJUVrGPGZWDowBNvezbTb7FBGRAipk4HgamGFmU82sktDZvTwtz3Lg/Gh+MfCwh+uDlwNLoquupgIzgKey3KeIiBRQwQY5dPdOM7sMeACIAT9x99Vmdg2w0t2XA7cCd5hZM7CFEAiI8t0NvAR0Al90D/fbZ9pnoY5BRET2VRI3AJrZRuCNAW7eAAzn57+qfIOj8g2Oyjc4w718h7v7Pp3EJRE4BsPMVma6c3K4UPkGR+UbHJVvcIZ7+foyYq+qEhGRwlDgEBGRnChw7N/NxS7Afqh8g6PyDY7KNzjDvXwZqY9DRERyohqHiIjkRIFDRERyosARGcyzQ4agbFPM7BEze8nMVpvZX2bIc5KZbTOzVdF01VCVL3r/FjN7IXrvfZ6aZcEN0fl73szmD2HZjkg5L6vMbLuZfSktz5CePzP7iZltMLMXU9LGm9l/mdmr0eu4PrY9P8rzqpmdnylPgcr3fTN7Jfr73WtmY/vYtt/PQgHLd7WZrUv5G57ex7YFf6ZPH+W7K6VsLWa2qo9tC37+Bs3dS34i3IX+GjANqASeA2al5fkC8H+j+SXAXUNYvkOB+dF8HfA/Gcp3EvCfRTyHLUBDP+tPB+4nPF/zA8CTRfxbv0O4salo5w84EZgPvJiS9j3gymj+SuDaDNuNB9ZGr+Oi+XFDVL6PAeXR/LWZypfNZ6GA5bsa+EoWf/9+/9cLVb609f8AXFWs8zfYSTWOYDDPDik4d3/b3Z+N5ncAL9PHcPLD2CLgZx78ARhrZocWoRynAK+5+0BHEsgLd3+MMMxOqtTP2O3AJzNs+ifAf7n7FnffCvwX4WFnBS+fu//Gw+MPAP5AGGS0KPo4f9kYkmf69Fe+6HvjM8Av8v2+Q0WBIxjMs0OGVNRENg94MsPqD5rZc2Z2v5nNHtKCgQO/MbNnzOziDOuzfpZKgS2h73/YYp4/gIPd/e1o/h3g4Ax5hst5vJBQg8xkf5+FQrosakr7SR9NfcPh/J0ArHf3V/tYX8zzlxUFjgOImdUC/w58yd23p61+ltD8chTwA+C+IS7e8e4+n/BY3y+a2YlD/P77FY2ofCbwbxlWF/v89eKhzWJYXitvZn9DGHz0531kKdZn4UfAe4C5wNuE5qDhaCn91zaG/f+SAkcwmGeHDAkzqyAEjZ+7+3+kr3f37e6+M5pfAVSYWcNQlc/d10WvG4B7CU0CqYbDs1ROA5519/XpK4p9/iLrk8130euGDHmKeh7N7ALgDODcKLjtI4vPQkG4+3p373L3BHBLH+9b7PNXDpwF3NVXnmKdv1wocASDeXZIwUVtorcCL7v7P/aR55Bkn4uZLST8bYcksJlZ3MzqkvOETtQX07ItB86Lrq76ALAtpVlmqPT5S6+Y5y9F6mfsfOCXGfI8AHzMzMZFTTEfi9IKzsxOBb4GnOnuu/vIk81noVDlS+0z+1Qf71vsZ/p8BHjF3VszrSzm+ctJsXvnh8tEuOrnfwhXXPxNlHYN4Z8EoJrQxNFMeKjUtCEs2/GEZovngVXRdDpwCXBJlOcyYDXhKpE/AMcOYfmmRe/7XFSG5PlLLZ8BN0bn9wWgaYj/vnFCIBiTkla080cIYG8DHYR29osIfWYPAa8CDwLjo7xNwI9Ttr0w+hw2A38+hOVrJvQPJD+DyasMJwIr+vssDFH57og+W88TgsGh6eWLlvf5Xx+K8kXptyU/cyl5h/z8DXbSkCMiIpITNVWJiEhOFDhERCQnChwiIpITBQ4REcmJAoeIiOREgUMkD8ysK20E3ryNumpmjamjrIoUW3mxCyAyQuxx97nFLoTIUFCNQ6SAomcrfC96vsJTZjY9Sm80s4ejAfkeMrPDovSDo2ddPBdNx0a7ipnZLRaex/IbMxtVtIOSkqfAIZIfo9Kaqs5JWbfN3ecAPwSuj9J+ANzu7u8nDBZ4Q5R+A/DfHgZbnE+4exhgBnCju88G3gU+XeDjEemT7hwXyQMz2+nutRnSW4APu/vaaKDKd9y93sw2EYbE6IjS33b3BjPbCEx29/aUfTQSnsExI1r+a6DC3b9T+CMT2ZdqHCKF533M56I9Zb4L9U9KESlwiBTeOSmvv4/mnyCMzApwLvB4NP8QcCmAmcXMbMxQFVIkW/rVIpIfo8xsVcry/3P35CW548zseUKtYWmU9hfAT83sq8BG4M+j9L8Ebjaziwg1i0sJo6yKDBvq4xApoKiPo8ndNxW7LCL5oqYqERHJiWocIiKSE9U4REQkJwocIiKSEwUOERHJiQKHiIjkRIFDRERy8v8B2dWR+TmOrTsAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}