Mentor_Copy_AIM_Project.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8bi5uYCri24m"
      },
      "source": [
        "# Install libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "tZLytqMwVJhn",
        "outputId": "68a7a6c4-237c-4649-e96c-0f2c1576e563"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torchtext==0.11 in /usr/local/lib/python3.7/dist-packages (0.11.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchtext==0.11) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchtext==0.11) (1.21.6)\n",
            "Requirement already satisfied: torch==1.10.0 in /usr/local/lib/python3.7/dist-packages (from torchtext==0.11) (1.10.0+cu111)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from torchtext==0.11) (4.64.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.10.0->torchtext==0.11) (4.1.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.11) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.11) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.11) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.11) (2.10)\n",
            "Requirement already satisfied: nlpaug==1.1.10 in /usr/local/lib/python3.7/dist-packages (1.1.10)\n",
            "Requirement already satisfied: requests>=2.22.0 in /usr/local/lib/python3.7/dist-packages (from nlpaug==1.1.10) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.16.2 in /usr/local/lib/python3.7/dist-packages (from nlpaug==1.1.10) (1.21.6)\n",
            "Requirement already satisfied: pandas>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from nlpaug==1.1.10) (1.3.5)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.2.0->nlpaug==1.1.10) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.2.0->nlpaug==1.1.10) (2022.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=1.2.0->nlpaug==1.1.10) (1.15.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.22.0->nlpaug==1.1.10) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.22.0->nlpaug==1.1.10) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.22.0->nlpaug==1.1.10) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.22.0->nlpaug==1.1.10) (1.24.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk) (1.15.0)\n",
            "Collecting pytorch-lightning==1.6.1\n",
            "  Using cached pytorch_lightning-1.6.1-py3-none-any.whl (582 kB)\n",
            "Collecting pyDeprecate<0.4.0,>=0.3.1\n",
            "  Using cached pyDeprecate-0.3.2-py3-none-any.whl (10 kB)\n",
            "Collecting fsspec[http]!=2021.06.0,>=2021.05.0\n",
            "  Using cached fsspec-2022.3.0-py3-none-any.whl (136 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.6.1) (4.1.1)\n",
            "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.6.1) (1.21.6)\n",
            "Requirement already satisfied: torch>=1.8.* in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.6.1) (1.10.0+cu111)\n",
            "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.6.1) (2.8.0)\n",
            "Collecting torchmetrics>=0.4.1\n",
            "  Using cached torchmetrics-0.8.0-py3-none-any.whl (408 kB)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.6.1) (21.3)\n",
            "Collecting PyYAML>=5.4\n",
            "  Using cached PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "Requirement already satisfied: tqdm>=4.41.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning==1.6.1) (4.64.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.6.1) (2.23.0)\n",
            "Collecting aiohttp\n",
            "  Using cached aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=17.0->pytorch-lightning==1.6.1) (3.0.8)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (1.0.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (1.8.1)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (1.44.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (0.37.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (3.3.6)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (0.4.6)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (3.17.3)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (1.35.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (1.0.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (57.4.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning==1.6.1) (0.6.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py>=0.4->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (1.15.0)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (4.2.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (4.8)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (4.11.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (3.8.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.6.1) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.6.1) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.6.1) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.6.1) (3.0.4)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch-lightning==1.6.1) (3.2.0)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Using cached multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 14.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.6.1) (2.0.12)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting asynctest==0.13.0\n",
            "  Using cached asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 66.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.6.1) (21.4.0)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Using cached async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, pyDeprecate, fsspec, aiohttp, torchmetrics, PyYAML, pytorch-lightning\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-6.0 aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 frozenlist-1.3.0 fsspec-2022.3.0 multidict-6.0.2 pyDeprecate-0.3.2 pytorch-lightning-1.6.1 torchmetrics-0.8.0 yarl-1.7.2\n",
            "\u001b[31mERROR: Could not find a version that satisfies the requirement panda==1.1.5 (from versions: 0.1.5.macosx-10.9-x86_64, 0.1.1, 0.1.2, 0.1.3, 0.1.4, 0.1.5, 0.3.1)\u001b[0m\n",
            "\u001b[31mERROR: No matching distribution found for panda==1.1.5\u001b[0m\n",
            "Collecting protobuf==3.19.4\n",
            "  Downloading protobuf-3.19.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 14.7 MB/s \n",
            "\u001b[?25hInstalling collected packages: protobuf\n",
            "  Attempting uninstall: protobuf\n",
            "    Found existing installation: protobuf 3.17.3\n",
            "    Uninstalling protobuf-3.17.3:\n",
            "      Successfully uninstalled protobuf-3.17.3\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.8.0 requires tf-estimator-nightly==2.8.0.dev2021122109, which is not installed.\u001b[0m\n",
            "Successfully installed protobuf-3.19.4\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "google"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[31mERROR: Could not find a version that satisfies the requirement sklearn==0.22.2.post1 (from versions: 0.0)\u001b[0m\n",
            "\u001b[31mERROR: No matching distribution found for sklearn==0.22.2.post1\u001b[0m\n",
            "Collecting wandb==0.12.14\n",
            "  Downloading wandb-0.12.14-py2.py3-none-any.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 15.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (2.23.0)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (6.0)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (7.1.2)\n",
            "Collecting shortuuid>=0.5.0\n",
            "  Downloading shortuuid-1.0.8-py3-none-any.whl (9.5 kB)\n",
            "Collecting GitPython>=1.0.0\n",
            "  Downloading GitPython-3.1.27-py3-none-any.whl (181 kB)\n",
            "\u001b[K     |████████████████████████████████| 181 kB 71.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (5.4.8)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (3.19.4)\n",
            "Collecting setproctitle\n",
            "  Downloading setproctitle-1.2.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (2.8.2)\n",
            "Collecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.5.10-py2.py3-none-any.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 71.9 MB/s \n",
            "\u001b[?25hCollecting pathtools\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (1.15.0)\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.12.14) (2.3)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "  Downloading gitdb-4.0.9-py3-none-any.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 2.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb==0.12.14) (4.1.1)\n",
            "Collecting smmap<6,>=3.0.1\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb==0.12.14) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb==0.12.14) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb==0.12.14) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb==0.12.14) (3.0.4)\n",
            "Building wheels for collected packages: pathtools\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8806 sha256=e31e904db152cb2f112aa0035313acc3123221f8cda40301fdefffb3bf8e5af3\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/31/09/fa59cef12cdcfecc627b3d24273699f390e71828921b2cbba2\n",
            "Successfully built pathtools\n",
            "Installing collected packages: smmap, gitdb, shortuuid, setproctitle, sentry-sdk, pathtools, GitPython, docker-pycreds, wandb\n",
            "Successfully installed GitPython-3.1.27 docker-pycreds-0.4.0 gitdb-4.0.9 pathtools-0.1.2 sentry-sdk-1.5.10 setproctitle-1.2.3 shortuuid-1.0.8 smmap-5.0.0 wandb-0.12.14\n",
            "Requirement already satisfied: seaborn==0.11.2 in /usr/local/lib/python3.7/dist-packages (0.11.2)\n",
            "Requirement already satisfied: pandas>=0.23 in /usr/local/lib/python3.7/dist-packages (from seaborn==0.11.2) (1.3.5)\n",
            "Requirement already satisfied: matplotlib>=2.2 in /usr/local/lib/python3.7/dist-packages (from seaborn==0.11.2) (3.2.2)\n",
            "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.7/dist-packages (from seaborn==0.11.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from seaborn==0.11.2) (1.21.6)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn==0.11.2) (1.4.2)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn==0.11.2) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn==0.11.2) (3.0.8)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn==0.11.2) (0.11.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib>=2.2->seaborn==0.11.2) (4.1.1)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.23->seaborn==0.11.2) (2022.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib>=2.2->seaborn==0.11.2) (1.15.0)\n",
            "Collecting h5py==3.6.0\n",
            "  Downloading h5py-3.6.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (4.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.1 MB 14.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.14.5 in /usr/local/lib/python3.7/dist-packages (from h5py==3.6.0) (1.21.6)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py==3.6.0) (1.5.2)\n",
            "Installing collected packages: h5py\n",
            "  Attempting uninstall: h5py\n",
            "    Found existing installation: h5py 3.1.0\n",
            "    Uninstalling h5py-3.1.0:\n",
            "      Successfully uninstalled h5py-3.1.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.8.0 requires tf-estimator-nightly==2.8.0.dev2021122109, which is not installed.\u001b[0m\n",
            "Successfully installed h5py-3.6.0\n",
            "Collecting rich==11.2.0\n",
            "  Downloading rich-11.2.0-py3-none-any.whl (217 kB)\n",
            "\u001b[K     |████████████████████████████████| 217 kB 16.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.7/dist-packages (from rich==11.2.0) (2.6.1)\n",
            "Collecting commonmark<0.10.0,>=0.9.0\n",
            "  Downloading commonmark-0.9.1-py2.py3-none-any.whl (51 kB)\n",
            "\u001b[K     |████████████████████████████████| 51 kB 9.8 MB/s \n",
            "\u001b[?25hCollecting colorama<0.5.0,>=0.4.0\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: typing-extensions<5.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from rich==11.2.0) (4.1.1)\n",
            "Installing collected packages: commonmark, colorama, rich\n",
            "Successfully installed colorama-0.4.4 commonmark-0.9.1 rich-11.2.0\n",
            "Collecting transformers==4.17.0\n",
            "  Downloading transformers-4.17.0-py3-none-any.whl (3.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 14.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (3.6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (21.3)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 51.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (6.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (4.64.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (4.11.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (2019.12.20)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 8.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.17.0) (2.23.0)\n",
            "Collecting tokenizers!=0.11.3,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 58.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.17.0) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers==4.17.0) (3.0.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.17.0) (3.8.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.17.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.17.0) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.17.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.17.0) (1.24.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.17.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.17.0) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.17.0) (1.15.0)\n",
            "Installing collected packages: tokenizers, sacremoses, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.5.1 sacremoses-0.0.49 tokenizers-0.12.1 transformers-4.17.0\n",
            "Collecting tokenizers==0.11.5\n",
            "  Downloading tokenizers-0.11.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.8 MB 15.9 MB/s \n",
            "\u001b[?25hInstalling collected packages: tokenizers\n",
            "  Attempting uninstall: tokenizers\n",
            "    Found existing installation: tokenizers 0.12.1\n",
            "    Uninstalling tokenizers-0.12.1:\n",
            "      Successfully uninstalled tokenizers-0.12.1\n",
            "Successfully installed tokenizers-0.11.5\n",
            "Found existing installation: tensorflow 2.8.0\n",
            "Uninstalling tensorflow-2.8.0:\n",
            "  Successfully uninstalled tensorflow-2.8.0\n",
            "\u001b[31mERROR: Could not find a version that satisfies the requirement torchmetrics==1.7.0 (from versions: 0.2.0, 0.3.0, 0.3.1, 0.3.2, 0.4.0, 0.4.1, 0.5.0, 0.5.1, 0.6.0rc0, 0.6.0rc1, 0.6.0, 0.6.1, 0.6.2, 0.7.0rc0, 0.7.0rc1, 0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.8.0rc0, 0.8.0)\u001b[0m\n",
            "\u001b[31mERROR: No matching distribution found for torchmetrics==1.7.0\u001b[0m\n",
            "Looking in links: https://download.pytorch.org/whl/cu113/torch_stable.html\n",
            "Collecting torch==1.10.1+cu113\n",
            "  Downloading https://download.pytorch.org/whl/cu113/torch-1.10.1%2Bcu113-cp37-cp37m-linux_x86_64.whl (1821.5 MB)\n",
            "\u001b[K     |██████████████▋                 | 834.1 MB 1.4 MB/s eta 0:11:59tcmalloc: large alloc 1147494400 bytes == 0x56499bb2e000 @  0x7f9225d89615 0x564999a0817c 0x564999ae847a 0x564999a0af9d 0x564999afcd4d 0x564999a7eec8 0x564999a79a2e 0x564999a0c88a 0x564999a7ed30 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999a0cce9 0x564999a50579 0x564999a0b902 0x564999a7ec4d 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e 0x564999a0c88a 0x564999a7a8f6 0x564999a0c7aa 0x564999a7ab4f 0x564999a79a2e\n",
            "\u001b[K     |██████████████████▌             | 1055.7 MB 1.5 MB/s eta 0:08:43tcmalloc: large alloc 1434370048 bytes == 0x5649e0184000 @  0x7f9225d89615 0x564999a0817c 0x564999ae847a 0x564999a0af9d 0x564999afcd4d 0x564999a7eec8 0x564999a79a2e 0x564999a0c88a 0x564999a7ed30 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999a0cce9 0x564999a50579 0x564999a0b902 0x564999a7ec4d 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e 0x564999a0c88a 0x564999a7a8f6 0x564999a0c7aa 0x564999a7ab4f 0x564999a79a2e\n",
            "\u001b[K     |███████████████████████▌        | 1336.2 MB 586 kB/s eta 0:13:48tcmalloc: large alloc 1792966656 bytes == 0x564a35970000 @  0x7f9225d89615 0x564999a0817c 0x564999ae847a 0x564999a0af9d 0x564999afcd4d 0x564999a7eec8 0x564999a79a2e 0x564999a0c88a 0x564999a7ed30 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999a0cce9 0x564999a50579 0x564999a0b902 0x564999a7ec4d 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e 0x564999a0c88a 0x564999a7a8f6 0x564999a0c7aa 0x564999a7ab4f 0x564999a79a2e\n",
            "\u001b[K     |█████████████████████████████▊  | 1691.1 MB 1.2 MB/s eta 0:01:45tcmalloc: large alloc 2241208320 bytes == 0x56499bb2e000 @  0x7f9225d89615 0x564999a0817c 0x564999ae847a 0x564999a0af9d 0x564999afcd4d 0x564999a7eec8 0x564999a79a2e 0x564999a0c88a 0x564999a7ed30 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999afdb76 0x564999a7ad95 0x564999a0cce9 0x564999a50579 0x564999a0b902 0x564999a7ec4d 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e 0x564999a0c88a 0x564999a7a8f6 0x564999a0c7aa 0x564999a7ab4f 0x564999a79a2e\n",
            "\u001b[K     |████████████████████████████████| 1821.5 MB 1.2 MB/s eta 0:00:01tcmalloc: large alloc 1821458432 bytes == 0x564a21490000 @  0x7f9225d881e7 0x564999a3e407 0x564999a0817c 0x564999ae847a 0x564999a0af9d 0x564999afcd4d 0x564999a7eec8 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a0c7aa 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e\n",
            "tcmalloc: large alloc 2276827136 bytes == 0x564a8dda4000 @  0x7f9225d89615 0x564999a0817c 0x564999ae847a 0x564999a0af9d 0x564999afcd4d 0x564999a7eec8 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7ab4f 0x564999a0c7aa 0x564999a7ab4f 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e 0x564999a0c88a 0x564999a7b719 0x564999a79a2e 0x564999a0cf21\n",
            "\u001b[K     |████████████████████████████████| 1821.5 MB 7.0 kB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.10.1+cu113) (4.1.1)\n",
            "Installing collected packages: torch\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.10.0+cu111\n",
            "    Uninstalling torch-1.10.0+cu111:\n",
            "      Successfully uninstalled torch-1.10.0+cu111\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchvision 0.11.1+cu111 requires torch==1.10.0, but you have torch 1.10.1+cu113 which is incompatible.\n",
            "torchtext 0.11.0 requires torch==1.10.0, but you have torch 1.10.1+cu113 which is incompatible.\n",
            "torchaudio 0.10.0+cu111 requires torch==1.10.0, but you have torch 1.10.1+cu113 which is incompatible.\u001b[0m\n",
            "Successfully installed torch-1.10.1+cu113\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "torch"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting bitsandbytes-cuda113\n",
            "  Downloading bitsandbytes_cuda113-0.26.0-py3-none-any.whl (4.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.3 MB 14.7 MB/s \n",
            "\u001b[?25hInstalling collected packages: bitsandbytes-cuda113\n",
            "Successfully installed bitsandbytes-cuda113-0.26.0\n",
            "tcmalloc: large alloc 1147494400 bytes == 0x55779b422000 @  0x7fd34eae1615 0x55776113b17c 0x55776121b47a 0x55776113df9d 0x55776122fd4d 0x5577611b1ec8 0x5577611aca2e 0x55776113f88a 0x5577611b1d30 0x5577611aca2e 0x55776113f88a 0x5577611ae719 0x557761230b76 0x5577611add95 0x557761230b76 0x5577611add95 0x557761230b76 0x5577611add95 0x55776113fce9 0x557761183579 0x55776113e902 0x5577611b1c4d 0x5577611aca2e 0x55776113f88a 0x5577611ae719 0x5577611aca2e 0x55776113f88a 0x5577611ad8f6 0x55776113f7aa 0x5577611adb4f 0x5577611aca2e\n"
          ]
        }
      ],
      "source": [
        "!pip install torchtext==0.11\n",
        "!pip install nlpaug==1.1.10\n",
        "!pip install nltk\n",
        "!pip install pytorch-lightning==1.6.1\n",
        "import pytorch_lightning as pl\n",
        "!pip install panda==1.1.5\n",
        "!pip install protobuf==3.19.4\n",
        "!pip install sklearn==\"0.22.2.post1\"\n",
        "!pip install wandb==0.12.14\n",
        "!pip install seaborn==0.11.2\n",
        "!pip install h5py==3.6.0\n",
        "!pip install rich==11.2.0\n",
        "!pip install transformers==4.17.0\n",
        "!pip install tokenizers==0.11.5\n",
        "!pip uninstall tensorflow -y\n",
        "from transformers import BertTokenizerFast as BertTokenizer\n",
        "!pip install torchmetrics==1.7.0\n",
        "!pip3 install torch==1.10.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html\n",
        "!pip install bitsandbytes-cuda113\n",
        "#!wget https://gist.githubusercontent.com/TimDettmers/1f5188c6ee6ed69d211b7fe4e381e713/raw/4d17c3d09ccdb57e9ab7eca0171f2ace6e4d2858/check_bnb_install.py && python check_bnb_install.py\n",
        "!pip install matplotlib>3.1\n",
        "!pip install horovod>=0.21.2,!=0.24.0  # no need to install with [pytorch] as pytorch is already installed\n",
        "!pip install torchtext>=0.9.*\n",
        "!pip install omegaconf>=2.0.5\n",
        "!pip install hydra-core>=1.0.5\n",
        "!pip install jsonargparse[signatures]>=4.6.0\n",
        "!pip install gcsfs>=2021.5.0\n",
        "!pip install rich>=10.2.2,!=10.15.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CW8PHrLbjAM2"
      },
      "source": [
        "# Get the python library version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TLdWnmBRVM_u",
        "outputId": "64bcddd4-89a1-4d21-86ff-deaaf3132f3a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import json\n",
        "import sklearn\n",
        "import math\n",
        "from torch.utils.data import Dataset\n",
        "from sklearn.model_selection import KFold\n",
        "import pytorch_lightning as pl\n",
        "from sklearn.model_selection import train_test_split\n",
        "from torch.utils.data import DataLoader, random_split\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import seaborn as sns\n",
        "import wandb\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "import h5py\n",
        "from typing import Optional\n",
        "import torchmetrics\n",
        "from torch.autograd import Variable\n",
        "from sklearn.decomposition import PCA\n",
        "from rich import print\n",
        "#from pytorch_lightning.callbacks import LearningRateLogger\n",
        "from pytorch_lightning.callbacks import LearningRateMonitor\n",
        "from pytorch_lightning.callbacks.early_stopping import EarlyStopping\n",
        "plt.show()\n",
        "from pytorch_lightning.loggers import TensorBoardLogger\n",
        "import gc\n",
        "gc.collect()\n",
        "from sklearn.model_selection import train_test_split\n",
        "torch.cuda.empty_cache()\n",
        "import numpy as onp\n",
        "import os.path as osp\n",
        "from abc import ABC, abstractmethod\n",
        "from copy import deepcopy\n",
        "from dataclasses import dataclass\n",
        "from typing import Any, Dict, List, Optional, Type\n",
        "from pytorch_lightning.loops.base import Loop\n",
        "from pytorch_lightning.loops.fit_loop import FitLoop\n",
        "from pytorch_lightning.trainer.states import TrainerFn\n",
        "from torch.utils.data import random_split\n",
        "from torch.utils.data.dataloader import DataLoader\n",
        "from torch.utils.data.dataset import Dataset, Subset\n",
        "from sklearn.model_selection import KFold\n",
        "import seaborn as sns\n",
        "from pylab import rcParams\n",
        "import wandb\n",
        "from transformers import BertModel, BertConfig\n",
        "import bitsandbytes as bnb\n",
        "from transformers import DistilBertModel, DistilBertConfig , DistilBertTokenizerFast\n",
        "import nlpaug.augmenter.word as naw\n",
        "import nltk\n",
        "import nlpaug.augmenter.char as nac\n",
        "import nlpaug.augmenter.word as naw\n",
        "import nlpaug.augmenter.sentence as nas\n",
        "import nlpaug.flow as nafc\n",
        "from nlpaug.util import Action\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')\n",
        "%matplotlib inline\n",
        "%config InlineBackend.figure_format='retina'\n",
        "sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
        "HAPPY_COLORS_PALETTE = [\"#01BEFE\", \"#FFDD00\", \"#FF7D00\", \"#FF006D\", \"#ADFF02\", \"#8F00FF\"]\n",
        "sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))\n",
        "rcParams['figure.figsize'] = 12, 8"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YC4QKxW7kQ62"
      },
      "source": [
        "# Constants"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MfYaeig2kTrZ"
      },
      "outputs": [],
      "source": [
        "LABEL_COLUMNS_NAME = \"labels_ids\"\n",
        "weight_and_bias_project = \"Github Multi-Label Classification\"\n",
        "BATCH_SIZE = 2<<4\n",
        "N_EPOCHS = 20\n",
        "MAX_TOKEN_LEN = 2<<8"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZV7hbCbBj6tK"
      },
      "source": [
        "# Library Version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Wy2spT2uLri",
        "outputId": "0fe30794-b2d2-4eb5-81b6-b88f2ab864d2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python 3.7.13\n"
          ]
        }
      ],
      "source": [
        "!python --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "ZJA73sSqj8Am",
        "outputId": "42f955f2-9232-47b8-96d8-f25277e876b4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "python3: can't open file 'collect_env_details.py': [Errno 2] No such file or directory\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Panda  \u001b[1;36m1.3\u001b[0m.\u001b[1;36m5\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Panda  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.3</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Numpy \u001b[1;36m1.21\u001b[0m.\u001b[1;36m6\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Numpy <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.21</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Weight and Bias Version \u001b[1;36m0.12\u001b[0m.\u001b[1;36m14\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Weight and Bias Version <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.12</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Sklearn \u001b[1;36m1.0\u001b[0m.\u001b[1;36m2\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Sklearn <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Seabron \u001b[1;36m0.11\u001b[0m.\u001b[1;36m2\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Seabron <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.11</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "torch \u001b[1;36m1.10\u001b[0m.\u001b[1;36m0\u001b[0m+cu111\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">torch <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.10</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>+cu111\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "h5py \u001b[1;36m3.6\u001b[0m.\u001b[1;36m0\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">h5py <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.6</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Torch Metric \u001b[1;36m0.8\u001b[0m.\u001b[1;36m0\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Torch Metric <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured (use `wandb login --relogin` to force relogin)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<wandb.apis.public.Api at 0x7fc69b50f750>"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "!python collect_env_details.py\n",
        "print( \"Panda \" , pd.__version__)\n",
        "print( \"Numpy\" , np.__version__)\n",
        "print(\"Weight and Bias Version\" , wandb.__version__)\n",
        "print(\"Sklearn\" , sklearn.__version__)\n",
        "print(\"Seabron\" , sns.__version__)\n",
        "print( \"torch\" , torch.__version__)\n",
        "print(\"h5py\" , h5py.__version__)\n",
        "print(\"Torch Metric\" , torchmetrics.__version__)\n",
        "wandb.finish()\n",
        "wandb.login(key=\"594c8e9a35d34a66e52ac0a49c6e08fdefda3053\")\n",
        "wandb.Api(timeout=19)\n",
        "## Weight and bias"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Sadfc_WkV3n"
      },
      "source": [
        "# "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PUlFXVAV1CA7"
      },
      "source": [
        "#  Reduce the Randomness of the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BCnphN-w1Hq-",
        "outputId": "d1a76ac2-d63c-42ba-f714-2f3a55bfa473"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Global seed set to 42\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "42"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "from pytorch_lightning import Trainer, seed_everything\n",
        "RANDOM_SEED = 42\n",
        "seed_everything(RANDOM_SEED, workers=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UsYiUi8t4XVG"
      },
      "source": [
        "# Get access of hte dataset "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BD5PvESdB6vV"
      },
      "source": [
        "## Setup Google Colab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wxTG6tEOB9-U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "00253daf-8b9a-440d-dcb4-a44f9a831dfc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive/\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/', force_remount=True)\n",
        "wandb.finish()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yZa8LVNiCejI"
      },
      "source": [
        "## Create the dataset pandas dataframes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "26EzPFRvC4Wl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "outputId": "f4d29e3a-63ac-4801-c95e-98d84628d603"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m<\u001b[0m\u001b[1;95mclass\u001b[0m\u001b[39m \u001b[0m\u001b[32m'NoneType'\u001b[0m\u001b[1m>\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">class</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008000; text-decoration-color: #008000\">'NoneType'</span><span style=\"font-weight: bold\">&gt;</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "The path file does not work data.h5\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">The path file does not work data.h5\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/generic.py:2718: PerformanceWarning: \n",
            "your performance may suffer as PyTables will pickle object types that it cannot\n",
            "map directly to c-types [inferred_type->mixed,key->block0_values] [items->Index(['body', 'labels_ids'], dtype='object')]\n",
            "\n",
            "  encoding=encoding,\n"
          ]
        }
      ],
      "source": [
        "def nlp_augmen():\n",
        "  ##dataset\n",
        "  ## loop through add augmenatition\n",
        "  ## linear combinatino one type of combinary each one\n",
        "  ## multiple combination. augmetnaino \n",
        "  ## will have tecxt functional tf-iff(text) = ouput\n",
        "  ## appen`\n",
        "  return 5\n",
        "id_name = {}\n",
        "def get_dataset():\n",
        "  file_path = \"/content/drive/MyDrive/nlp_dataset_json\"\n",
        "  ## determine the size of the dataset\n",
        "  files  = os.listdir(os.path.join(os.getcwd(),file_path))\n",
        "  dataset = None\n",
        "  #files=files[:100]\n",
        "  for x in files:\n",
        "    try:\n",
        "      f = open(os.path.join(os.path.join(os.getcwd(),file_path), x))\n",
        "      data = json.load(f)\n",
        "      labels = []\n",
        "      label_names = []\n",
        "      for x in  data[\"labels\"]:\n",
        "        labels.append(x[\"id\"])\n",
        "        id_name[x[\"id\"]] = x[\"name\"]\n",
        "      if dataset is None:\n",
        "        print(type(dataset))\n",
        "      if dataset is not  None:\n",
        "\n",
        "        dataset_object = [data['body'] , labels]\n",
        "        dataset_object = np.asarray(dataset_object , dtype=object).reshape(1,2)\n",
        "        dataset = pd.concat([pd.DataFrame(dataset_object , columns=[\"body\" , \"labels_ids\"] , dtype=object) , dataset] , ignore_index=True , axis = 0)\n",
        "      else:\n",
        "        dataset_object = [data['body'] , labels]\n",
        "        dataset_object = np.asarray(dataset_object , dtype=object).reshape(1,2)\n",
        "        dataset = pd.DataFrame(dataset_object , columns=[\"body\" , \"labels_ids\"] , dtype=object)\n",
        "      f.close()\n",
        "    except:\n",
        "      print(f\"The path file does not work {x}\")\n",
        "  ##convert the dataset to numpy because pandas give to much headache\n",
        "  dataset.to_hdf( os.path.join((os.path.join(os.getcwd(),file_path)) , \"data.h5\") , key=\"df\" , mode=\"w\" , complib='blosc:zlib' , complevel=9)\n",
        "  dataset = dataset.to_numpy()\n",
        "  return dataset\n",
        "file_path = \"/content/drive/MyDrive/nlp_dataset_json\"\n",
        "dataset = get_dataset()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(id_name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 672
        },
        "id": "OwQoiya67p7C",
        "outputId": "cd30c582-caed-4fc5-e8dc-67dc690ed0f3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m{\u001b[0m\n",
              "    \u001b[1;36m1297090688\u001b[0m: \u001b[32m'feature'\u001b[0m,\n",
              "    \u001b[1;36m1297090689\u001b[0m: \u001b[32m'help wanted'\u001b[0m,\n",
              "    \u001b[1;36m1297090686\u001b[0m: \u001b[32m'bug'\u001b[0m,\n",
              "    \u001b[1;36m1297090692\u001b[0m: \u001b[32m'question'\u001b[0m,\n",
              "    \u001b[1;36m1297090690\u001b[0m: \u001b[32m'good first issue'\u001b[0m,\n",
              "    \u001b[1;36m2253758944\u001b[0m: \u001b[32m'logger'\u001b[0m,\n",
              "    \u001b[1;36m1297090693\u001b[0m: \u001b[32m\"won't fix\"\u001b[0m,\n",
              "    \u001b[1;36m1475966033\u001b[0m: \u001b[32m'example'\u001b[0m,\n",
              "    \u001b[1;36m1840917107\u001b[0m: \u001b[32m'docs'\u001b[0m,\n",
              "    \u001b[1;36m1851720487\u001b[0m: \u001b[32m'priority: 0'\u001b[0m,\n",
              "    \u001b[1;36m2251044906\u001b[0m: \u001b[32m'strategy: dp'\u001b[0m,\n",
              "    \u001b[1;36m1893143017\u001b[0m: \u001b[32m\"let's do it!\"\u001b[0m,\n",
              "    \u001b[1;36m1862633788\u001b[0m: \u001b[32m'discussion'\u001b[0m,\n",
              "    \u001b[1;36m2014560835\u001b[0m: \u001b[32m'waiting on author'\u001b[0m,\n",
              "    \u001b[1;36m1297090687\u001b[0m: \u001b[32m'duplicate'\u001b[0m,\n",
              "    \u001b[1;36m2245077676\u001b[0m: \u001b[32m'design'\u001b[0m,\n",
              "    \u001b[1;36m1819743298\u001b[0m: \u001b[32m'ready'\u001b[0m,\n",
              "    \u001b[1;36m1851722664\u001b[0m: \u001b[32m'ci'\u001b[0m,\n",
              "    \u001b[1;36m1973825445\u001b[0m: \u001b[32m'accelerator: tpu'\u001b[0m,\n",
              "    \u001b[1;36m2122110652\u001b[0m: \u001b[32m'working as intended'\u001b[0m,\n",
              "    \u001b[1;36m2604053872\u001b[0m: \u001b[32m'enhancement'\u001b[0m,\n",
              "    \u001b[1;36m2604053875\u001b[0m: \u001b[32m'help wanted'\u001b[0m,\n",
              "    \u001b[1;36m2651599587\u001b[0m: \u001b[32m'has conflicts'\u001b[0m,\n",
              "    \u001b[1;36m2368743110\u001b[0m: \u001b[32m'3rd party'\u001b[0m,\n",
              "    \u001b[1;36m2477304581\u001b[0m: \u001b[32m'priority: 2'\u001b[0m,\n",
              "    \u001b[1;36m1934189816\u001b[0m: \u001b[32m'enhancement'\u001b[0m,\n",
              "    \u001b[1;36m1934189817\u001b[0m: \u001b[32m'good first issue'\u001b[0m,\n",
              "    \u001b[1;36m1934189818\u001b[0m: \u001b[32m'help wanted'\u001b[0m,\n",
              "    \u001b[1;36m1934189821\u001b[0m: \u001b[32m\"won't fix\"\u001b[0m,\n",
              "    \u001b[1;36m2615223038\u001b[0m: \u001b[32m'logger'\u001b[0m,\n",
              "    \u001b[1;36m2244367919\u001b[0m: \u001b[32m'data handling'\u001b[0m,\n",
              "    \u001b[1;36m2823174111\u001b[0m: \u001b[32m'New metric'\u001b[0m,\n",
              "    \u001b[1;36m3240345158\u001b[0m: \u001b[32m'wontfix'\u001b[0m,\n",
              "    \u001b[1;36m2324728524\u001b[0m: \u001b[32m'checkpointing'\u001b[0m,\n",
              "    \u001b[1;36m2240244232\u001b[0m: \u001b[32m'refactor'\u001b[0m,\n",
              "    \u001b[1;36m2237125337\u001b[0m: \u001b[32m'distributed'\u001b[0m,\n",
              "    \u001b[1;36m2604053874\u001b[0m: \u001b[32m'good first issue'\u001b[0m,\n",
              "    \u001b[1;36m2477301440\u001b[0m: \u001b[32m'priority: 1'\u001b[0m\n",
              "\u001b[1m}\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090688</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'feature'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090689</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'help wanted'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090686</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'bug'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090692</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'question'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090690</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'good first issue'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2253758944</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'logger'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090693</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"won't fix\"</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1475966033</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'example'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1840917107</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'docs'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1851720487</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'priority: 0'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2251044906</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'strategy: dp'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1893143017</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"let's do it!\"</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1862633788</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'discussion'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2014560835</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'waiting on author'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090687</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'duplicate'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2245077676</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'design'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1819743298</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'ready'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1851722664</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'ci'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1973825445</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'accelerator: tpu'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2122110652</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'working as intended'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053872</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'enhancement'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053875</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'help wanted'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2651599587</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'has conflicts'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2368743110</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'3rd party'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2477304581</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'priority: 2'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189816</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'enhancement'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189817</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'good first issue'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189818</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'help wanted'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189821</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"won't fix\"</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2615223038</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'logger'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2244367919</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'data handling'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2823174111</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'New metric'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3240345158</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'wontfix'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2324728524</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'checkpointing'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2240244232</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'refactor'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2237125337</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'distributed'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053874</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'good first issue'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2477301440</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'priority: 1'</span>\n",
              "<span style=\"font-weight: bold\">}</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "\n",
        "labels_file = open(\"labels.pkl\", \"wb\")\n",
        "pickle.dump(id_name, labels_file)\n",
        "labels_file.close()\n",
        "\n",
        "labels_file = open(\"labels.pkl\", \"rb\")\n",
        "output = pickle.load(labels_file)\n",
        "print(output)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 672
        },
        "id": "hnj2BnBr7yVV",
        "outputId": "699b6928-821f-4c8a-f2af-51edf64a2415"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m{\u001b[0m\n",
              "    \u001b[1;36m1297090688\u001b[0m: \u001b[32m'feature'\u001b[0m,\n",
              "    \u001b[1;36m1297090689\u001b[0m: \u001b[32m'help wanted'\u001b[0m,\n",
              "    \u001b[1;36m1297090686\u001b[0m: \u001b[32m'bug'\u001b[0m,\n",
              "    \u001b[1;36m1297090692\u001b[0m: \u001b[32m'question'\u001b[0m,\n",
              "    \u001b[1;36m1297090690\u001b[0m: \u001b[32m'good first issue'\u001b[0m,\n",
              "    \u001b[1;36m2253758944\u001b[0m: \u001b[32m'logger'\u001b[0m,\n",
              "    \u001b[1;36m1297090693\u001b[0m: \u001b[32m\"won't fix\"\u001b[0m,\n",
              "    \u001b[1;36m1475966033\u001b[0m: \u001b[32m'example'\u001b[0m,\n",
              "    \u001b[1;36m1840917107\u001b[0m: \u001b[32m'docs'\u001b[0m,\n",
              "    \u001b[1;36m1851720487\u001b[0m: \u001b[32m'priority: 0'\u001b[0m,\n",
              "    \u001b[1;36m2251044906\u001b[0m: \u001b[32m'strategy: dp'\u001b[0m,\n",
              "    \u001b[1;36m1893143017\u001b[0m: \u001b[32m\"let's do it!\"\u001b[0m,\n",
              "    \u001b[1;36m1862633788\u001b[0m: \u001b[32m'discussion'\u001b[0m,\n",
              "    \u001b[1;36m2014560835\u001b[0m: \u001b[32m'waiting on author'\u001b[0m,\n",
              "    \u001b[1;36m1297090687\u001b[0m: \u001b[32m'duplicate'\u001b[0m,\n",
              "    \u001b[1;36m2245077676\u001b[0m: \u001b[32m'design'\u001b[0m,\n",
              "    \u001b[1;36m1819743298\u001b[0m: \u001b[32m'ready'\u001b[0m,\n",
              "    \u001b[1;36m1851722664\u001b[0m: \u001b[32m'ci'\u001b[0m,\n",
              "    \u001b[1;36m1973825445\u001b[0m: \u001b[32m'accelerator: tpu'\u001b[0m,\n",
              "    \u001b[1;36m2122110652\u001b[0m: \u001b[32m'working as intended'\u001b[0m,\n",
              "    \u001b[1;36m2604053872\u001b[0m: \u001b[32m'enhancement'\u001b[0m,\n",
              "    \u001b[1;36m2604053875\u001b[0m: \u001b[32m'help wanted'\u001b[0m,\n",
              "    \u001b[1;36m2651599587\u001b[0m: \u001b[32m'has conflicts'\u001b[0m,\n",
              "    \u001b[1;36m2368743110\u001b[0m: \u001b[32m'3rd party'\u001b[0m,\n",
              "    \u001b[1;36m2477304581\u001b[0m: \u001b[32m'priority: 2'\u001b[0m,\n",
              "    \u001b[1;36m1934189816\u001b[0m: \u001b[32m'enhancement'\u001b[0m,\n",
              "    \u001b[1;36m1934189817\u001b[0m: \u001b[32m'good first issue'\u001b[0m,\n",
              "    \u001b[1;36m1934189818\u001b[0m: \u001b[32m'help wanted'\u001b[0m,\n",
              "    \u001b[1;36m1934189821\u001b[0m: \u001b[32m\"won't fix\"\u001b[0m,\n",
              "    \u001b[1;36m2615223038\u001b[0m: \u001b[32m'logger'\u001b[0m,\n",
              "    \u001b[1;36m2244367919\u001b[0m: \u001b[32m'data handling'\u001b[0m,\n",
              "    \u001b[1;36m2823174111\u001b[0m: \u001b[32m'New metric'\u001b[0m,\n",
              "    \u001b[1;36m3240345158\u001b[0m: \u001b[32m'wontfix'\u001b[0m,\n",
              "    \u001b[1;36m2324728524\u001b[0m: \u001b[32m'checkpointing'\u001b[0m,\n",
              "    \u001b[1;36m2240244232\u001b[0m: \u001b[32m'refactor'\u001b[0m,\n",
              "    \u001b[1;36m2237125337\u001b[0m: \u001b[32m'distributed'\u001b[0m,\n",
              "    \u001b[1;36m2604053874\u001b[0m: \u001b[32m'good first issue'\u001b[0m,\n",
              "    \u001b[1;36m2477301440\u001b[0m: \u001b[32m'priority: 1'\u001b[0m\n",
              "\u001b[1m}\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090688</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'feature'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090689</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'help wanted'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090686</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'bug'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090692</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'question'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090690</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'good first issue'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2253758944</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'logger'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090693</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"won't fix\"</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1475966033</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'example'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1840917107</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'docs'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1851720487</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'priority: 0'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2251044906</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'strategy: dp'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1893143017</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"let's do it!\"</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1862633788</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'discussion'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2014560835</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'waiting on author'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090687</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'duplicate'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2245077676</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'design'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1819743298</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'ready'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1851722664</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'ci'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1973825445</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'accelerator: tpu'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2122110652</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'working as intended'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053872</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'enhancement'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053875</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'help wanted'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2651599587</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'has conflicts'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2368743110</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'3rd party'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2477304581</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'priority: 2'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189816</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'enhancement'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189817</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'good first issue'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189818</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'help wanted'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189821</span>: <span style=\"color: #008000; text-decoration-color: #008000\">\"won't fix\"</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2615223038</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'logger'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2244367919</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'data handling'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2823174111</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'New metric'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3240345158</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'wontfix'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2324728524</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'checkpointing'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2240244232</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'refactor'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2237125337</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'distributed'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053874</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'good first issue'</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2477301440</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'priority: 1'</span>\n",
              "<span style=\"font-weight: bold\">}</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf = open(\"labels.json\", \"w\")\n",
        "json.dump(id_name,tf)\n",
        "tf.close()\n",
        "\n",
        "\n",
        "tf = open(\"myDictionary.json\", \"r\")\n",
        "new_dict = json.load(tf)\n",
        "print(new_dict)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "rDB2VGZKEQdF",
        "outputId": "d6ed19f9-7c63-42c1-ce6d-fac4f432b488"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-28-e2d1559adc4e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mtf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"myDictionary.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mnew_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'myDictionary.json'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9BNxXlFxbKe9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181
        },
        "outputId": "e55946c8-d94c-4127-d302-203878c18738"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m[\u001b[0m\u001b[32m'I am using Neptune.ML for logging, but I would like to keep the data for analysis \u001b[0m\n",
              "\u001b[32mlocally.\\r\\n\\r\\nTrying to write a simple logger that dumps info to the CSV \u001b[0m\n",
              "\u001b[32mfiles.\\r\\n\\r\\n`train.csv` like:\\r\\n\\r\\n```\\r\\nstep, loss1, loss2\\r\\n1, 0.5, 0.4\\r\\n2, 0.4, \u001b[0m\n",
              "\u001b[32m0.12\\r\\n....\\r\\n```\\r\\n\\r\\n\\r\\n`val.csv` should look like:\\r\\n\\r\\n```\\r\\nepoch, loss1, \u001b[0m\n",
              "\u001b[32mloss2\\r\\n1, 0.6, 0.23\\r\\n2, 0.13245, 0.2134\\r\\n....\\r\\n```\\r\\n\\r\\nI am looking at the docs at\u001b[0m\n",
              "\u001b[32mhttps://pytorch-lightning.readthedocs.io/en/latest/loggers.html\\r\\n\\r\\nand at the examples in\u001b[0m\n",
              "\u001b[32mhttps://github.com/PyTorchLightning/pytorch-lightning/tree/master/pytorch_lightning/loggers\\r\u001b[0m\n",
              "\u001b[32m\\n\\r\\nand cannot figure out how to do this.\\r\\n\\r\\nIt would be nice if you extend the \u001b[0m\n",
              "\u001b[32mdocumentation with a more detailed example.'\u001b[0m\n",
              " \u001b[1;35mlist\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m1840917107\u001b[0m, \u001b[1;36m2253758944\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'I am using Neptune.ML for logging, but I would like to keep the data for analysis </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">locally.\\r\\n\\r\\nTrying to write a simple logger that dumps info to the CSV </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">files.\\r\\n\\r\\n`train.csv` like:\\r\\n\\r\\n```\\r\\nstep, loss1, loss2\\r\\n1, 0.5, 0.4\\r\\n2, 0.4, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">0.12\\r\\n....\\r\\n```\\r\\n\\r\\n\\r\\n`val.csv` should look like:\\r\\n\\r\\n```\\r\\nepoch, loss1, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">loss2\\r\\n1, 0.6, 0.23\\r\\n2, 0.13245, 0.2134\\r\\n....\\r\\n```\\r\\n\\r\\nI am looking at the docs at</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">https://pytorch-lightning.readthedocs.io/en/latest/loggers.html\\r\\n\\r\\nand at the examples in</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">https://github.com/PyTorchLightning/pytorch-lightning/tree/master/pytorch_lightning/loggers\\r</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">\\n\\r\\nand cannot figure out how to do this.\\r\\n\\r\\nIt would be nice if you extend the </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">documentation with a more detailed example.'</span>\n",
              " <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">list</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1840917107</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2253758944</span><span style=\"font-weight: bold\">])]</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "print(dataset[0])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OBTqR_WOQifO"
      },
      "source": [
        "## Preprocess\n",
        "\n",
        "*   One Hot Encoding\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PUNyX9jfQn_x"
      },
      "outputs": [],
      "source": [
        "## find the number labels\n",
        "one_hot_elements = dict()\n",
        "for index  in range(dataset.shape[0]):\n",
        "  #print(row[\"labels_ids\"])\n",
        "  for label in dataset[index][1]:\n",
        "    if label in  one_hot_elements.keys():\n",
        "      continue\n",
        "    else:\n",
        "      one_hot_elements[label] = len(one_hot_elements.keys())\n",
        "def one_hot_encoder(labels):\n",
        "  one_hot_label = np.zeros( ( len(one_hot_elements.keys())))\n",
        "  for label in labels:\n",
        "     one_hot_label[one_hot_elements[label]] = 1\n",
        "  return one_hot_label"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# save the one hot encoding dictionary\n",
        "\n",
        "ohe_file = open(\"one_hot.pkl\", \"wb\")\n",
        "pickle.dump(one_hot_elements, ohe_file)\n",
        "ohe_file.close()\n",
        "\n",
        "ohe_file = open(\"one_hot.pkl\", \"rb\")\n",
        "output = pickle.load(ohe_file)\n",
        "print(output)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 672
        },
        "id": "m6pKVHlW8ewx",
        "outputId": "bf779381-66f6-496e-b19d-56bbfb7790c3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m{\u001b[0m\n",
              "    \u001b[1;36m1840917107\u001b[0m: \u001b[1;36m0\u001b[0m,\n",
              "    \u001b[1;36m2253758944\u001b[0m: \u001b[1;36m1\u001b[0m,\n",
              "    \u001b[1;36m1297090692\u001b[0m: \u001b[1;36m2\u001b[0m,\n",
              "    \u001b[1;36m1297090688\u001b[0m: \u001b[1;36m3\u001b[0m,\n",
              "    \u001b[1;36m1297090689\u001b[0m: \u001b[1;36m4\u001b[0m,\n",
              "    \u001b[1;36m1297090693\u001b[0m: \u001b[1;36m5\u001b[0m,\n",
              "    \u001b[1;36m1297090686\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
              "    \u001b[1;36m1819743298\u001b[0m: \u001b[1;36m7\u001b[0m,\n",
              "    \u001b[1;36m1851720487\u001b[0m: \u001b[1;36m8\u001b[0m,\n",
              "    \u001b[1;36m1851722664\u001b[0m: \u001b[1;36m9\u001b[0m,\n",
              "    \u001b[1;36m1862633788\u001b[0m: \u001b[1;36m10\u001b[0m,\n",
              "    \u001b[1;36m1973825445\u001b[0m: \u001b[1;36m11\u001b[0m,\n",
              "    \u001b[1;36m1297090687\u001b[0m: \u001b[1;36m12\u001b[0m,\n",
              "    \u001b[1;36m1893143017\u001b[0m: \u001b[1;36m13\u001b[0m,\n",
              "    \u001b[1;36m1297090690\u001b[0m: \u001b[1;36m14\u001b[0m,\n",
              "    \u001b[1;36m2014560835\u001b[0m: \u001b[1;36m15\u001b[0m,\n",
              "    \u001b[1;36m2324728524\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
              "    \u001b[1;36m1475966033\u001b[0m: \u001b[1;36m17\u001b[0m,\n",
              "    \u001b[1;36m2237125337\u001b[0m: \u001b[1;36m18\u001b[0m,\n",
              "    \u001b[1;36m2251044906\u001b[0m: \u001b[1;36m19\u001b[0m,\n",
              "    \u001b[1;36m2477301440\u001b[0m: \u001b[1;36m20\u001b[0m,\n",
              "    \u001b[1;36m2245077676\u001b[0m: \u001b[1;36m21\u001b[0m,\n",
              "    \u001b[1;36m2604053872\u001b[0m: \u001b[1;36m22\u001b[0m,\n",
              "    \u001b[1;36m2604053874\u001b[0m: \u001b[1;36m23\u001b[0m,\n",
              "    \u001b[1;36m2604053875\u001b[0m: \u001b[1;36m24\u001b[0m,\n",
              "    \u001b[1;36m2823174111\u001b[0m: \u001b[1;36m25\u001b[0m,\n",
              "    \u001b[1;36m2368743110\u001b[0m: \u001b[1;36m26\u001b[0m,\n",
              "    \u001b[1;36m2477304581\u001b[0m: \u001b[1;36m27\u001b[0m,\n",
              "    \u001b[1;36m2122110652\u001b[0m: \u001b[1;36m28\u001b[0m,\n",
              "    \u001b[1;36m2244367919\u001b[0m: \u001b[1;36m29\u001b[0m,\n",
              "    \u001b[1;36m2240244232\u001b[0m: \u001b[1;36m30\u001b[0m,\n",
              "    \u001b[1;36m3240345158\u001b[0m: \u001b[1;36m31\u001b[0m,\n",
              "    \u001b[1;36m1934189816\u001b[0m: \u001b[1;36m32\u001b[0m,\n",
              "    \u001b[1;36m1934189817\u001b[0m: \u001b[1;36m33\u001b[0m,\n",
              "    \u001b[1;36m1934189818\u001b[0m: \u001b[1;36m34\u001b[0m,\n",
              "    \u001b[1;36m1934189821\u001b[0m: \u001b[1;36m35\u001b[0m,\n",
              "    \u001b[1;36m2615223038\u001b[0m: \u001b[1;36m36\u001b[0m,\n",
              "    \u001b[1;36m2651599587\u001b[0m: \u001b[1;36m37\u001b[0m\n",
              "\u001b[1m}\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1840917107</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2253758944</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090692</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090688</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090689</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090693</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090686</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1819743298</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1851720487</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1851722664</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1862633788</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1973825445</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090687</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1893143017</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1297090690</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2014560835</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2324728524</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1475966033</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2237125337</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">18</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2251044906</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2477301440</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2245077676</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">21</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053872</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">22</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053874</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">23</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2604053875</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2823174111</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">25</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2368743110</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">26</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2477304581</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">27</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2122110652</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2244367919</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">29</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2240244232</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3240345158</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189816</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189817</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189818</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1934189821</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">35</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2615223038</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">36</span>,\n",
              "    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2651599587</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">37</span>\n",
              "<span style=\"font-weight: bold\">}</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g_XyDEBKhN-h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c7cc9c08-c6b5-40b1-b291-55efac88b740"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{}"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "one_hot_elements"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FHVgCZEvKUt6"
      },
      "source": [
        "## Pipeline Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UNFLFRAP5ar9"
      },
      "outputs": [],
      "source": [
        "from numpy.core.multiarray import datetime_as_string\n",
        "def pipeline_function(dataset):\n",
        "  indexes = []\n",
        "  number_of_error = 0\n",
        "  for x in range(dataset.shape[0]):\n",
        "    ## check\n",
        "    try:\n",
        "      if len(dataset[x][0])>10:\n",
        "        if len(dataset[x][1])>=1:\n",
        "          indexes.append(x)\n",
        "    except:\n",
        "      number_of_error  = number_of_error +1\n",
        "      print(f\"The index {x} does not work\")\n",
        "      if number_of_error ==10:\n",
        "        break\n",
        "  ## add lemmatization for the data\n",
        "  \n",
        "  return indexes\n",
        "\n",
        "indexes = pipeline_function(dataset)\n",
        "print(indexes)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1dd3V21Zs5b2"
      },
      "outputs": [],
      "source": [
        "dataset = dataset[indexes]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NywVx-K-LTTh"
      },
      "outputs": [],
      "source": [
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zl5bovRXXR6N"
      },
      "outputs": [],
      "source": [
        "print(dataset.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VWMKSh2KQW14"
      },
      "outputs": [],
      "source": [
        "print(indexes)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kxiQk4AX68r-"
      },
      "source": [
        "# Using the Tokenize "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fA-t7K4S8_Gg"
      },
      "outputs": [],
      "source": [
        "'''\n",
        "BERT_MODEL_NAME = 'bert-base-cased'\n",
        "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_NAME)\n",
        "'''\n",
        "BERT_MODEL_NAME = 'distilbert-base-uncased'\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained(\"distilbert-base-uncased\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WFpkEjA46_HW"
      },
      "outputs": [],
      "source": [
        "# testing the tokenizer\n",
        "sample_commet = dataset[0][0]\n",
        "encoding = tokenizer.encode_plus(\n",
        "  sample_commet,\n",
        "  add_special_tokens=True,\n",
        "  max_length=512,\n",
        "  return_token_type_ids=False,\n",
        "  padding=\"max_length\",\n",
        "  return_attention_mask=True,\n",
        "  return_tensors='pt',\n",
        ")\n",
        "print(encoding)\n",
        "print(tokenizer.convert_ids_to_tokens(encoding[\"input_ids\"].squeeze())[:20])\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iKsvmYPBbbM9"
      },
      "source": [
        "# Token Counts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7vtdgEz-8-aw"
      },
      "outputs": [],
      "source": [
        "token_counts = []\n",
        "for row_index in range(dataset.shape[0]):\n",
        "  token_count = len(tokenizer.encode(\n",
        "    dataset[row_index][0],\n",
        "    max_length=2048,\n",
        "    truncation=True\n",
        "  ))\n",
        "  token_counts.append(token_count)\n",
        "sns.histplot(token_counts)\n",
        "plt.xlim([0, 2048]);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YqbcHc6BsI1e"
      },
      "outputs": [],
      "source": [
        "data = np.asarray(token_counts)\n",
        "print(np.sum(data<MAX_TOKEN_LEN))\n",
        "print(f\"The precentage of the datset {np.sum(data<MAX_TOKEN_LEN) / len(token_counts)} \")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nQ82Alr19jV3"
      },
      "source": [
        "# How much words are there"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VDufbJdi9m_Q"
      },
      "outputs": [],
      "source": [
        "words_counts = []\n",
        "for row_index in range(dataset.shape[0]):\n",
        "  words_counts.append(len(dataset[row_index][0]))\n",
        "sns.histplot(words_counts)\n",
        "plt.xlim([0, 4096]);\n",
        "print(f\"The mean of the number of Words {np.mean(words_counts)}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D-DpNVdCQWEO"
      },
      "source": [
        "# Split The dataset to train and Test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FXAEj6YSQYWF"
      },
      "outputs": [],
      "source": [
        "train_ds , test_ds = train_test_split(dataset, test_size=0.33, random_state=42 , shuffle = True)\n",
        "train_ds , validation_ds = train_test_split(train_ds, test_size=0.33, random_state=42 , shuffle = True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i1DMGNvR6iYv"
      },
      "source": [
        "# NLP Augmentation Pipeline\n",
        "*   SynonmAug Agumentation\n",
        "*   Char Random Insertion \n",
        "*   Spelling Mistake\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5BDeL6GA_Ehl"
      },
      "outputs": [],
      "source": [
        "train_ds  = train_ds.tolist()\n",
        "length_of_the_orginal_dataset = len(train_ds) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JmQ9aDzm9a6z"
      },
      "source": [
        "## Synoynm Augmentation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e8t5GnnJ9amF"
      },
      "outputs": [],
      "source": [
        "aug = naw.SynonymAug(aug_src='wordnet' , \n",
        "aug_min = 10  ,  \n",
        "aug_max = 400 , aug_p = 0.3 , \n",
        "lang = \"eng\")\n",
        "number_of_augmentation = 5\n",
        "for idx in range(length_of_the_orginal_dataset):\n",
        "  augmented_texts = aug.augment(train_ds[idx][0],  n = number_of_augmentation )\n",
        "  assert isinstance(augmented_texts , list)\n",
        "  assert isinstance(augmented_texts , list)\n",
        "  for augment_idx in range(number_of_augmentation):\n",
        "    assert isinstance(augmented_texts[augment_idx] , str)\n",
        "    augmented_texts[augment_idx] = [augmented_texts[augment_idx] , train_ds[idx][1]]\n",
        "  train_ds += augmented_texts"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b6ao_wc1k-tS"
      },
      "source": [
        "## Char Random Insertion"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QN9G-S2tlsMQ"
      },
      "outputs": [],
      "source": [
        "\n",
        "aug = nac.RandomCharAug(action='insert', \n",
        "name='RandomChar_Aug', aug_char_min=1, aug_char_max=100, \n",
        "aug_char_p=0.3, aug_word_p=0.3, aug_word_min=1, aug_word_max=10, \n",
        "include_upper_case=True, include_lower_case=True, include_numeric=True, \n",
        "min_char=4, swap_mode='adjacent', spec_char='!@#$%^&*()_+', stopwords=None, \n",
        "tokenizer=None, reverse_tokenizer=None, verbose=0, stopwords_regex=None, candidiates=None)\n",
        "length_of_the_orginal_dataset = len(train_ds)\n",
        "for idx in range(length_of_the_orginal_dataset):\n",
        "  augmented_texts = aug.augment(train_ds[idx][0],  n = number_of_augmentation )\n",
        "  assert isinstance(augmented_texts , list)\n",
        "  assert isinstance(augmented_texts , list)\n",
        "  for augment_idx in range(number_of_augmentation):\n",
        "    assert isinstance(augmented_texts[augment_idx] , str)\n",
        "    augmented_texts[augment_idx] = [augmented_texts[augment_idx] , train_ds[idx][1]]\n",
        "  train_ds += augmented_texts\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0UKsD4Ou6myt"
      },
      "outputs": [],
      "source": [
        "## get the shape of train Dataset\n",
        "## convert the dataset into  a list of tupple\n",
        "## loop through all list by adding in augmentation \n",
        "'''\n",
        "aug = naw.SpellingAug(dict_path='./spelling_en.txt', aug_min = 1 , aug_max = 15 , aug_p = 0.3)\n",
        "augmented_length = len(train_ds)\n",
        "for idx in range(augmented_length):\n",
        "  augmented_texts = aug.augment(train_ds[idx][0],  n = 2)\n",
        "  assert isinstance(augmented_texts , list)\n",
        "  assert isinstance(augmented_texts , list)\n",
        "  for augment_idx in range(3):\n",
        "    assert isinstance(augmented_texts[augment_idx] , str)\n",
        "    augmented_texts[augment_idx] = [augmented_texts[augment_idx] , train_ds[idx][1]]\n",
        "  train_ds += augmented_texts\n",
        "train_ds = np.asarray(train_ds)\n",
        "print(type(type(train_ds)))\n",
        "print(f\"The Shape of Train Dataset {train_ds.shape}\")\n",
        "print(train_ds[:2])\n",
        "'''\n",
        "train_ds = np.asarray(train_ds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v8oSprOrV9ww"
      },
      "outputs": [],
      "source": [
        "print(type(train_ds))\n",
        "print(type(train_ds[0]))\n",
        "print(train_ds[0])\n",
        "print(type(train_ds[986]))\n",
        "print(train_ds.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FMI7xlxAd7dQ"
      },
      "source": [
        "# Pytorch Dataset for the Open Source"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uVMqeCs2d_Ob"
      },
      "outputs": [],
      "source": [
        "class OPIsssueDataset(Dataset):\n",
        "  def __init__(\n",
        "    self,\n",
        "    data: pd.DataFrame,\n",
        "    tokenizer: BertTokenizer,\n",
        "    max_token_len: int = 128\n",
        "  ):\n",
        "    self.tokenizer = tokenizer\n",
        "    self.data = data\n",
        "    self.max_token_len = max_token_len\n",
        "  def __len__(self):\n",
        "      return self.data.shape[0]\n",
        "  def __getitem__(self, index):\n",
        "    #print(\"Start fetch\")\n",
        "    comment_text = self.data[index][0]\n",
        "    labels = self.data[index][1]\n",
        "\n",
        "    encoding = self.tokenizer.encode_plus(\n",
        "      comment_text,\n",
        "      add_special_tokens=True,\n",
        "      max_length=self.max_token_len,\n",
        "      return_token_type_ids=False,\n",
        "      padding=\"max_length\",\n",
        "      truncation=True,\n",
        "      return_attention_mask=True,\n",
        "      return_tensors='pt',\n",
        "    )\n",
        "    assert isinstance(self.data , np.ndarray)\n",
        "    return dict(\n",
        "      comment_text=comment_text,\n",
        "      input_ids=encoding[\"input_ids\"].flatten(),\n",
        "      attention_mask=encoding[\"attention_mask\"].flatten(),\n",
        "      labels=torch.FloatTensor(one_hot_encoder(labels))\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9OMuUMZUlt1c"
      },
      "source": [
        "# Lightning Data Module"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HBFmkpFemHd8"
      },
      "outputs": [],
      "source": [
        "class OPDataModule(pl.LightningDataModule):\n",
        "  def __init__(self , train_df ,  validation_df , test_df ,tokenizer , batch_size = 2<<5 , max_token_len=2<<8):\n",
        "    super().__init__()\n",
        "    self.batch_size = batch_size\n",
        "    self.train_df = train_df\n",
        "    self.test_ds = test_df\n",
        "    self.validation_ds = validation_df\n",
        "    self.tokenizer = tokenizer\n",
        "    self.max_token_len = max_token_len\n",
        "  def setup(self, stage=None):\n",
        "    self.train_dataset = OPIsssueDataset(\n",
        "      self.train_df,\n",
        "      self.tokenizer,\n",
        "      self.max_token_len\n",
        "    )\n",
        "    self.validation_dataset = OPIsssueDataset(\n",
        "        self.validation_ds,\n",
        "        self.tokenizer,\n",
        "        self.max_token_len\n",
        "    )\n",
        "    self.test_dataset = OPIsssueDataset(\n",
        "      self.test_ds,\n",
        "      self.tokenizer,\n",
        "      self.max_token_len\n",
        "    )\n",
        "    #assert isinstance()\n",
        "  def train_dataloader(self):\n",
        "    return DataLoader(\n",
        "      self.train_dataset,\n",
        "      batch_size=self.batch_size,\n",
        "      shuffle=True,\n",
        "      num_workers=0\n",
        "    )\n",
        "  def val_dataloader(self):\n",
        "    return DataLoader(\n",
        "      self.validation_dataset,\n",
        "      batch_size=self.batch_size,\n",
        "      num_workers=0\n",
        "    )\n",
        "  def test_dataloader(self):\n",
        "    return DataLoader(\n",
        "      self.test_dataset,\n",
        "      batch_size=self.batch_size,\n",
        "      num_workers=0\n",
        "    )\n",
        "data_module = OPDataModule(\n",
        "    train_df = train_ds,\n",
        "    validation_df = validation_ds,\n",
        "    test_df = test_ds,\n",
        "    tokenizer = tokenizer,\n",
        "    batch_size = BATCH_SIZE,\n",
        "    max_token_len = MAX_TOKEN_LEN\n",
        ")\n",
        "data_module.setup()\n",
        "print(data_module.train_dataset)\n",
        "print(data_module.train_dataset.data)\n",
        "print(type(data_module.train_dataset.data))\n",
        "print(type(data_module.train_dataset.data)==np.ndarray)\n",
        "print(isinstance(data_module.train_dataset.data , np.ndarray))\n",
        "print(type(data_module.train_dataset.data[0]))\n",
        "print(data_module.train_dataset.data[0][1])\n",
        "print(type(data_module.train_dataset.data[0][1]))\n",
        "'''\n",
        "for idx ,  data in enumerate(data_module.train_dataset):\n",
        "  print(data)\n",
        "  print(type(data))\n",
        "'''"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oXF4tGkX-jaZ"
      },
      "outputs": [],
      "source": [
        "data_module.train_dataset[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fmIZ7v2CvjM2"
      },
      "source": [
        "# Pytorch Lightnig  Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TmejZSUl2HvZ"
      },
      "outputs": [],
      "source": [
        "import transformers\n",
        "from torchmetrics import Accuracy , Recall , Precision , F1Score\n",
        "class OPClassifier(pl.LightningModule):\n",
        "  def __init__(self, n_classes: int, n_training_steps=None, n_warmup_steps=None, learning_rate = 2e-5):\n",
        "    super().__init__()\n",
        "    self.bert = DistilBertModel.from_pretrained(BERT_MODEL_NAME, return_dict=True)\n",
        "    self.classifier = nn.Linear(self.bert.config.hidden_size, n_classes)\n",
        "    self.n_training_steps = n_training_steps\n",
        "    self.n_warmup_steps = n_warmup_steps\n",
        "    self.learning_rate = learning_rate\n",
        "    ##https://discuss.pytorch.org/t/bceloss-are-unsafe-to-autocast/110407\n",
        "    self.criterion = nn.BCEWithLogitsLoss()\n",
        "    ## multi-label accuracy classficition\n",
        "    self.train_acc = Accuracy(subset_accuracy = True , average = \"micro\" , threshold = 0.5)\n",
        "    self.test_acc = Accuracy(subset_accuracy = True , average = \"micro\" , threshold = 0.5)\n",
        "    self.validation_acc = Accuracy(subset_accuracy = True , average = \"micro\" , threshold = 0.5)\n",
        "    ## Recall\n",
        "    ### Micro\n",
        "    self.train_micro_recall = Recall(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.validation_micro_recall = Recall(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.test_micro_recall = Recall(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    ### Weighted\n",
        "    self.train_weighted_recall = Recall(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.validation_weighted_recall = Recall(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.test_weighted_recall = Recall(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    ## Precision\n",
        "    ### Micro\n",
        "    self.train_micro_precision = Precision(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.validation_micro_precision = Precision(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.test_micro_precision = Precision(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    ### Weighted\n",
        "    self.train_weighted_precision  = Precision(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.validation_weighted_precision  = Precision(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.test_weighted_precision  = Precision(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    ## f1Score\n",
        "    ### Micro\n",
        "    self.train_micro_f1score = F1Score(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.validation_micro_f1score = F1Score(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.test_micro_f1score = F1Score(average = \"micro\" , num_classes = len(one_hot_elements.keys()))\n",
        "    ### Weighted\n",
        "    self.train_weighted_f1score  = F1Score(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.validation_weighted_f1score  = F1Score(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "    self.test_weighted_f1score  = F1Score(average = \"weighted\" , num_classes = len(one_hot_elements.keys()))\n",
        "  def forward(self, input_ids, attention_mask, labels=None):\n",
        "    output = self.bert(input_ids = input_ids, attention_mask=attention_mask , output_hidden_states = True)\n",
        "    output = self.classifier(output.last_hidden_state[:, 0])\n",
        "    ##https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html#torch.nn.BCEWithLogitsLoss\n",
        "    #output = torch.sigmoid(output)\n",
        "    loss = 0\n",
        "    if labels is not None:\n",
        "        loss = self.criterion(output, labels)\n",
        "    return loss, output\n",
        "  def training_step(self, batch, batch_idx):\n",
        "    input_ids = batch[\"input_ids\"]\n",
        "    attention_mask = batch[\"attention_mask\"]\n",
        "    labels = batch[\"labels\"]\n",
        "    loss, outputs = self(input_ids, attention_mask, labels)\n",
        "    labels = labels.detach().cpu().type(torch.IntTensor)\n",
        "    outputs = outputs.detach().cpu()\n",
        "    self.train_acc(outputs , labels)\n",
        "    self.train_micro_recall(outputs , labels)\n",
        "    #self.train_weighted_recall(outputs , labels)\n",
        "    self.train_micro_precision(outputs , labels)\n",
        "    #self.train_weighted_precision(outputs , labels)\n",
        "    self.log(\"train/micro_precision\" , self.train_micro_precision , on_step = True , prog_bar = True)\n",
        "    #self.log(\"train/weighted_precision\" , self.train_weighted_precision , on_step = True , prog_bar = True)\n",
        "    self.log(\"train/micro_recall\" , self.train_micro_recall , on_step = True , prog_bar = True)\n",
        "    #self.log(\"train/weighted_recall\" , self.train_weighted_recall , on_step = True , prog_bar = True)\n",
        "    self.log(\"train/subset_accuracy_micro_average\"  , self.train_acc , on_step = True , prog_bar = True )\n",
        "    self.log(\"train/loss\", loss, prog_bar=True)\n",
        "    return {\"loss\": loss}\n",
        "  def validation_step(self, batch, batch_idx):\n",
        "    input_ids = batch[\"input_ids\"]\n",
        "    attention_mask = batch[\"attention_mask\"]\n",
        "    labels = batch[\"labels\"]\n",
        "    loss, outputs = self(input_ids, attention_mask, labels)\n",
        "    labels = labels.detach().cpu().type(torch.IntTensor)\n",
        "    outputs = outputs.detach().cpu()\n",
        "    self.validation_acc(outputs , labels)\n",
        "    self.validation_micro_recall(outputs , labels)\n",
        "    #self.validation_weighted_recall(outputs , labels)\n",
        "    self.validation_micro_precision(outputs , labels)\n",
        "    #self.validation_weighted_precision(outputs , labels)\n",
        "    self.log(\"validation/micro_precision\" , self.validation_micro_precision , on_step = True , prog_bar = True , on_epoch  = True)\n",
        "    #self.log(\"validation/weighted_precision\" , self.validation_weighted_precision , on_step = True , prog_bar = True , on_epoch = True)\n",
        "    self.log(\"validation/micro_recall\" , self.validation_micro_recall , on_step = True , prog_bar = True , on_epoch = True)\n",
        "    #self.log(\"validation/weighted_recall\" , self.validation_weighted_recall , on_step = True , prog_bar = True , on_epoch = True)\n",
        "    self.log(\"validation/subset_accuracy_micro_average\"  , self.validation_acc , on_step = True , prog_bar = True  , on_epoch = True)\n",
        "    self.log(\"validation/loss\", loss, prog_bar=True, logger=True)\n",
        "    return loss\n",
        "  def test_step(self, batch, batch_idx):\n",
        "    input_ids = batch[\"input_ids\"]\n",
        "    attention_mask = batch[\"attention_mask\"]\n",
        "    labels = batch[\"labels\"]\n",
        "    loss, outputs = self(input_ids, attention_mask, labels)\n",
        "    labels = labels.detach().cpu().type(torch.IntTensor)\n",
        "    outputs = outputs.detach().cpu()\n",
        "    self.test_acc(outputs , labels)\n",
        "    self.test_micro_recall(outputs , labels)\n",
        "    #self.test_weighted_recall(outputs , labels)\n",
        "    self.test_micro_precision(outputs , labels)\n",
        "    #self.test_weighted_precision(outputs , labels)\n",
        "    self.log(\"test/micro_precision\" , self.test_micro_precision , on_step = True , prog_bar = True , on_epoch  = True)\n",
        "    #self.log(\"test/weighted_precision\" , self.test_weighted_precision , on_step = True , prog_bar = True , on_epoch = True)\n",
        "    self.log(\"test/micro_recall\" , self.test_micro_recall , on_step = True , prog_bar = True , on_epoch = True)\n",
        "    #self.log(\"test/weighted_recall\" , self.test_weighted_recall , on_step = True , prog_bar = True , on_epoch = True)\n",
        "    self.log(\"test/subset_accuracy_micro_average\"  , self.test_acc , on_step = True , prog_bar = True  , on_epoch = True)\n",
        "    self.log(\"test/loss\", loss, prog_bar=True, logger=True)\n",
        "    return loss\n",
        "  '''\n",
        "  def training_epoch_end(self, outputs):\n",
        "    labels = []\n",
        "    predictions = []\n",
        "    for output in outputs:\n",
        "      for out_labels in output[\"labels\"].detach().cpu():\n",
        "        labels.append(out_labels)\n",
        "      for out_predictions in output[\"predictions\"].detach().cpu():\n",
        "        predictions.append(out_predictions)\n",
        "    labels = torch.stack(labels).int()\n",
        "    predictions = torch.stack(predictions)\n",
        "    #for i, name in enumerate(LABEL_COLUMNS):\n",
        "      #class_roc_auc = auroc(predictions[:, i], labels[:, i])\n",
        "      #self.logger.experiment.add_scalar(f\"{name}_roc_auc/Train\", class_roc_auc, self.current_epoch)\n",
        "  '''\n",
        "  def configure_optimizers(self):\n",
        "    optimizer = bnb.optim.AdamW8bit(self.bert.parameters(), lr=self.learning_rate, betas=(0.9, 0.995)) # add bnb optimizer\n",
        "    scheduler = transformers.get_linear_schedule_with_warmup(\n",
        "      optimizer,\n",
        "      num_warmup_steps=self.n_warmup_steps,\n",
        "      num_training_steps=self.n_training_steps\n",
        "    )\n",
        "    #return optimizer\n",
        "    return dict(\n",
        "      optimizer=optimizer,\n",
        "      lr_scheduler=dict(\n",
        "        scheduler=scheduler,\n",
        "        interval='step'\n",
        "      )\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hwwOJHOWqJEr"
      },
      "source": [
        "# Trainer Loop"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7dU80I-0qK6B"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VvHkvQmUA-Ri"
      },
      "outputs": [],
      "source": [
        "steps_per_epoch=len(train_ds) // BATCH_SIZE\n",
        "total_training_steps = steps_per_epoch * N_EPOCHS\n",
        "warmup_steps = total_training_steps // 5\n",
        "print(f\"The step for for an epoch {steps_per_epoch}\")\n",
        "print(f\"Total Training Steps {total_training_steps}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qy7Lz0lbCMkQ"
      },
      "source": [
        "# Initialize the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cDui5dtGCPdL"
      },
      "outputs": [],
      "source": [
        "model = OPClassifier(\n",
        "  n_classes=len(one_hot_elements.keys()),\n",
        "  n_warmup_steps=warmup_steps,\n",
        "  n_training_steps=total_training_steps\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UvF-4qi68OwQ"
      },
      "source": [
        "# Check Point"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kCNK36nt8Qqv"
      },
      "outputs": [],
      "source": [
        "from pytorch_lightning.callbacks import ModelCheckpoint\n",
        "checkpoint_callback = ModelCheckpoint(\n",
        "  dirpath=\"checkpoints\",\n",
        "  filename=\"best-checkpoint\",\n",
        "  save_top_k=2,\n",
        "  verbose=True,\n",
        "  monitor='validation/subset_accuracy_micro_average',\n",
        "  mode=\"max\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eqHw63kY8YXE"
      },
      "source": [
        "# Logger "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b6pFT_mKGaGe"
      },
      "outputs": [],
      "source": [
        "import imp\n",
        "try:\n",
        "    imp.find_module('wandb')\n",
        "    found = True\n",
        "except ImportError:\n",
        "    found = False\n",
        "print(found)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tFEtJa1n8Z2u"
      },
      "outputs": [],
      "source": [
        "## Weight and bias =\n",
        "from pytorch_lightning.loggers import WandbLogger\n",
        "logger    = WandbLogger(\n",
        "      project=weight_and_bias_project,\n",
        "      log_model = \"all\",\n",
        "      save_code = True\n",
        "  )\n",
        "#logger = TensorBoardLogger(\"lightning_logs\", name=\"toxic-comments\")\n",
        "print(logger)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M95P6FGe-fdX"
      },
      "source": [
        "# Early Stopping"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aYuz4pIv-hFg"
      },
      "outputs": [],
      "source": [
        "early_stopping_callback = EarlyStopping(monitor='validation/subset_accuracy_micro_average', patience=0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XecTvbBGYkbk"
      },
      "source": [
        "# Training Step"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7OLN-pn3YmEx"
      },
      "outputs": [],
      "source": [
        "trainer = pl.Trainer(\n",
        "  logger=logger,\n",
        "  checkpoint_callback=True,\n",
        "  callbacks=[early_stopping_callback , checkpoint_callback],\n",
        "  max_epochs=N_EPOCHS,\n",
        "  gpus=1,\n",
        "  fast_dev_run = False,\n",
        "  progress_bar_refresh_rate=30,\n",
        "  precision = \"bf16\" if torch.cuda.is_bf16_supported() else 16\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_2DqdyGefwh5"
      },
      "source": [
        "# Trainer Fit"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UON7Om9Qd1c6"
      },
      "outputs": [],
      "source": [
        "!nvidia-smi -L"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qxmbI2pZCKxQ"
      },
      "outputs": [],
      "source": [
        "torch.cuda.is_bf16_supported()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OT-eB10Pfxqg"
      },
      "outputs": [],
      "source": [
        "print(type(data_module))\n",
        "trainer.fit(model , data_module )\n",
        "trainer.test(model , data_module , ckpt_path=\"best\")\n",
        "wandb.finish()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fF5lSQ-ZlADf"
      },
      "outputs": [],
      "source": [
        "checkpoint_callback.best_model_path"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nFxaJ3Xj7704"
      },
      "source": [
        "# Sweep"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k3zULH0e7855"
      },
      "outputs": [],
      "source": [
        "sweep_config = {\n",
        "  \"name\" : \"my-sweep\",\n",
        "  \"method\" : \"bayes\",\n",
        "  \"parameters\" : {\n",
        "    \"learning_rate\" :{\n",
        "      \"min\": 0.0001,\n",
        "      \"max\": 0.1\n",
        "    }\n",
        "  }\n",
        "}\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hy48lMGV87FA"
      },
      "outputs": [],
      "source": [
        "def sweep_function():\n",
        "  with wandb.init() as runn:\n",
        "    config = wandb.config\n",
        "    model = OPClassifier(\n",
        "    n_classes=len(one_hot_elements.keys()),\n",
        "    n_warmup_steps=warmup_steps,\n",
        "    n_training_steps=total_training_steps,\n",
        "    learning_rate = config[\"learning_rate\"]\n",
        "    )\n",
        "    model = OPClassifier(\n",
        "      n_classes=len(one_hot_elements.keys()),\n",
        "      n_warmup_steps=warmup_steps,\n",
        "      n_training_steps=total_training_steps\n",
        "    )\n",
        "    checkpoint_callbacks = ModelCheckpoint(\n",
        "      dirpath=\"checkpoints\",\n",
        "      filename=\"best-checkpoint\",\n",
        "      save_top_k=1,\n",
        "      verbose=True,\n",
        "      monitor='validation/subset_accuracy_micro_average',\n",
        "      mode=\"min\"\n",
        "    )\n",
        "    early_stopping_callback = EarlyStopping(monitor='validation/subset_accuracy_micro_average', patience=0)\n",
        "    trainer = pl.Trainer(\n",
        "      logger=logger,\n",
        "      checkpoint_callback=checkpoint_callbacks,\n",
        "      callbacks=[early_stopping_callback],\n",
        "      max_epochs=N_EPOCHS,\n",
        "      gpus=1,\n",
        "      fast_dev_run = False,\n",
        "      progress_bar_refresh_rate=30,\n",
        "      precision = \"bf16\" if torch.cuda.is_bf16_supported() else 16\n",
        "    )\n",
        "    print(type(data_module))\n",
        "    trainer.fit(model , data_module )\n",
        "    trainer.test(model , data_module , ckpt_path=\"best\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model.state_dict(), \"classifier.pth\")"
      ],
      "metadata": {
        "id": "kCr4kC6IJDEw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wLp8quwXje5T"
      },
      "source": [
        "# Predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "61RevIv6jgYF"
      },
      "outputs": [],
      "source": [
        "trained_model = OPClassifier.load_from_checkpoint(\n",
        "  trainer.checkpoint_callback.best_model_path,\n",
        "  n_classes=len(LABEL_COLUMNS)\n",
        ")\n",
        "trained_model.eval()\n",
        "trained_model.freeze()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2rrcmJ9XhjDO"
      },
      "outputs": [],
      "source": [
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xKCRNqFuHs71"
      },
      "source": [
        "# Citation\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NhNhuAwKHyXH"
      },
      "outputs": [],
      "source": [
        "'''\n",
        "@misc{dettmers2021optim8bit,\n",
        "      title={8-bit Optimizers via Block-wise Quantization},\n",
        "      author={Tim Dettmers and Mike Lewis and Sam Shleifer and Luke Zettlemoyer},\n",
        "      year={2021},\n",
        "      eprint={2110.02861},\n",
        "      archivePrefix={arXiv},\n",
        "      primaryClass={cs.LG}\n",
        "}\n",
        "https://curiousily.com/posts/multi-label-text-classification-with-bert-and-pytorch-lightning/\n",
        "https://devblog.pytorchlightning.ai/introducing-faster-training-with-lightning-and-brain-float16-861505eabbfc\n",
        "https://devblog.pytorchlightning.ai/introducing-faster-training-with-lightning-and-brain-float16-861505eabbfc\n",
        "''''"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "background_execution": "on",
      "collapsed_sections": [
        "YC4QKxW7kQ62",
        "PUlFXVAV1CA7",
        "FHVgCZEvKUt6",
        "kxiQk4AX68r-",
        "iKsvmYPBbbM9",
        "nQ82Alr19jV3",
        "D-DpNVdCQWEO",
        "FMI7xlxAd7dQ",
        "9OMuUMZUlt1c",
        "hwwOJHOWqJEr"
      ],
      "name": "Mentor Copy AIM Project.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}