Dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Dataset.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "SWWkSNpBt1V4",
        "_2Ko0tH81Nu3",
        "7CnGlisHxtCm",
        "rfuebZ0_wT7m"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jincy-p-janardhanan/SPP-Pneumonia-Net/blob/ml/Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q1EW_qQws1kY"
      },
      "source": [
        "# Preliminaries\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXRX5Ggxs5ro"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RKP_KKmhN2UN"
      },
      "source": [
        "import os\n",
        "import shutil"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_6L3uTpq56jl"
      },
      "source": [
        "normal = 0\n",
        "bacterial = 0\n",
        "viral = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QsNLI7PLc8gy"
      },
      "source": [
        "# Create and organize dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SWWkSNpBt1V4"
      },
      "source": [
        "## Dataset: [COVID-19 Radiography Database](https://www.kaggle.com/tawsifurrahman/covid19-radiography-database)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_2Ko0tH81Nu3"
      },
      "source": [
        "### Download dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1g8QGTy_tGMe"
      },
      "source": [
        "os.environ['KAGGLE_CONFIG_DIR'] = \"/content/drive/MyDrive/radiography/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q88UvNa1OKyE"
      },
      "source": [
        "% cd /content/drive/MyDrive/radiography/\n",
        "% pwd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5cB2F12HP1-4"
      },
      "source": [
        "! kaggle datasets download -d tawsifurrahman/covid19-radiography-database"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8p-p1e1YQWCo"
      },
      "source": [
        "!unzip \\*.zip && rm *.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7CnGlisHxtCm"
      },
      "source": [
        "### Copy Images"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cjdg-1Qp2mEq"
      },
      "source": [
        "Copy normal images to main_project/normal"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ukRsfBYd2lGl"
      },
      "source": [
        "normal_folder = '/content/drive/MyDrive/radiography/COVID-19_Radiography_Dataset/Normal'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2edOUWJJ3O-Q"
      },
      "source": [
        "# change working directory before copying files\n",
        "% cd '/content/drive/MyDrive/radiography/COVID-19_Radiography_Dataset/Normal'\n",
        "! pwd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GJ6xvUiv3fuQ"
      },
      "source": [
        "files = os.listdir(normal_folder)\n",
        "for f in files:\n",
        "  if 'Normal' in f and normal < 2780:\n",
        "    normal += 1\n",
        "    shutil.copy(normal_folder+f,'/content/drive/MyDrive/main_project/Normal')\n",
        "print(\"normal: \", normal, \"\\t bacterial: \", bacterial, \"\\t viral: \", viral)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3P8hPKCKvZCJ"
      },
      "source": [
        "Copy viral pneumonia images to main_project/Viral"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8RpP1RS7ujrF"
      },
      "source": [
        "viral_folder = '/content/drive/MyDrive/radiography/COVID-19_Radiography_Dataset/Viral Pneumonia'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ajot-ENU4rlZ"
      },
      "source": [
        "% cd '/content/drive/MyDrive/radiography/COVID-19_Radiography_Dataset/Viral Pneumonia'\n",
        "! pwd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QPKqQNF8Sm0Q"
      },
      "source": [
        "files = os.listdir(viral_folder)\n",
        "for f in files:\n",
        "  if 'Viral' in f and viral < 2780:\n",
        "    viral += 1\n",
        "    shutil.copy(pneumonia_folder+f,'/content/drive/MyDrive/main_project/Viral')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rfuebZ0_wT7m"
      },
      "source": [
        "## Dataset: [Chest X-ray Images Pneumonia](https://www.kaggle.com/paultimothymooney/chest-xray-pneumonia)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rJTsVDbT0-Mk"
      },
      "source": [
        "### Download dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kkQQGQjMwTYZ"
      },
      "source": [
        "os.environ['KAGGLE_CONFIG_DIR'] = \"/content/drive/MyDrive/paultimothymooney/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "63vu3LXHxLzw"
      },
      "source": [
        "% cd /content/drive/MyDrive/paultimothymooney/\n",
        "% pwd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a1I-bpMoxGGr"
      },
      "source": [
        "! kaggle datasets download -d paultimothymooney/chest-xray-pneumonia"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gB0AXEeyyE51"
      },
      "source": [
        "!unzip \\*.zip && rm *.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g7owGZPCzHcp"
      },
      "source": [
        "### Copy Images"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "edjmbTWbzDV_"
      },
      "source": [
        "Copy bacterial and viral pneumonia images from train folder to main_project/Bacterial and main_project/Viral"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6IPc8SLzO_P"
      },
      "source": [
        "pneumonia_folder = '/content/drive/MyDrive/paultimothymooney/chest_xray/train/PNEUMONIA/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p6Qr_f961iSH"
      },
      "source": [
        "% cd /content/drive/MyDrive/paultimothymooney/chest_xray/train/PNEUMONIA/\n",
        "! pwd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-AFYWeZnzP9v"
      },
      "source": [
        "for f in files:\n",
        "  if 'bacteria' in f and bacterial < 2780:\n",
        "    bacterial += 1\n",
        "    shutil.copy(pneumonia_folder+f,'/content/drive/MyDrive/main_project/Bacterial')\n",
        "  elif 'virus' in f and viral < 2780:\n",
        "    viral += 1\n",
        "    shutil.copy(pneumonia_folder+f,'/content/drive/MyDrive/main_project/Viral')\n",
        "print(\"normal: \", normal, \"\\t bacterial: \", bacterial, \"\\t viral: \", viral)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JaAj25qwz4yI"
      },
      "source": [
        "Copy the 8 bacterial pneumonia images in val folder to main_project/Bacterial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z0xqk8tSzPlT"
      },
      "source": [
        "pneumonia_folder = '/content/drive/MyDrive/paultimothymooney/chest_xray/val/PNEUMONIA/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vASx0jy11Bk"
      },
      "source": [
        "% cd /content/drive/MyDrive/paultimothymooney/chest_xray/val/PNEUMONIA/\n",
        "! pwd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54vYJ8Cj0Nzx"
      },
      "source": [
        "for f in files:\n",
        "  if 'bacteria' in f and bacterial < 2780:\n",
        "    bacterial += 1\n",
        "    shutil.copy(pneumonia_folder+f,'/content/drive/MyDrive/main_project/Bacterial')\n",
        "print(\"normal: \", normal, \"\\t bacterial: \", bacterial, \"\\t viral: \", viral)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMMc3ieVcrca"
      },
      "source": [
        "# Convert dataset to h5 file"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V_I250HRd1Ge"
      },
      "source": [
        "## Utility functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZXYIyuY2dfs9"
      },
      "source": [
        "- Convert images to numpy array and save in h5 file\n",
        "- For faster training <br>\n",
        "[Github Reference](https://github.com/selvam85/Cat-Dog-Classifier/blob/master/DNN_using_plain_TF_Cat_vs_Dog_classifier_Kaggle_dataset/Convert%20Images%20to%20Numpy%20array%20and%20save%20in%20h5%20fomat%20v2.1.ipynb) <br>\n",
        "\n",
        "Other references: \n",
        "[1](https://medium.datadriveninvestor.com/speed-up-your-image-training-on-google-colab-dc95ea1491cf), \n",
        "[2](https://medium.com/@selvam85/how-to-work-with-large-training-dataset-in-google-colab-platform-c3499fc10c24)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m9PED4sqdk7B"
      },
      "source": [
        "### Normalize and write data to h5 file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a_LB7CRYcqeW"
      },
      "source": [
        "def normalize_and_write_data_into_h5_file(dest_filepath, filepaths_list, n_px, n_channels = 3):\n",
        "    \n",
        "    '''\n",
        "        This function converts images to numpy arrays and writes the array data into a h5 file.\n",
        "        \n",
        "        dest_filepath - the name of the file with full path that is being created\n",
        "        filepaths_list - source image file paths which is being converted to numpy arrays\n",
        "        n_px - number of pixels - will be used as image's height and width\n",
        "        n_channels - 3 for rgb\n",
        "    '''\n",
        "    \n",
        "    data_shape = (len(filepaths_list), n_px * n_px * n_channels)\n",
        "    dataset_name = \"input_data\"\n",
        "\n",
        "    with h5py.File(dest_filepath, 'a') as f:\n",
        "        \n",
        "        f.create_dataset(dataset_name, data_shape, np.float32)\n",
        "        \n",
        "        for i in range(len(filepaths_list)):\n",
        "            #if (i+1) % 512 == 0:\n",
        "            #    print('{}/{} files converted'.format((i+1), len(filepaths_list)))\n",
        "\n",
        "            filepath = filepaths_list[i]\n",
        "            img = cv2.imread(filepath)\n",
        "            img = cv2.resize(img, (n_px, n_px), interpolation=cv2.INTER_CUBIC)\n",
        "            img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # To convert colormap from BGR to GRAY\n",
        "            \n",
        "            #Normalize the image - convert the each pixel value between 0 and 1\n",
        "            img = img / 255\n",
        "            #Reshape the image - roll it up into a column vector\n",
        "            img = img.ravel()\n",
        "            \n",
        "            #img[None] makes it a proper array instead of rank 1 array\n",
        "            f[dataset_name][i, ...] = img[None]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BZI7uI7CeWTf"
      },
      "source": [
        "### Write labels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8tAPi8gydyNk"
      },
      "source": [
        "Write corresponding labels for each image into the h5 file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "03OWNNMMduIG"
      },
      "source": [
        "def write_labels_into_h5_file(dest_filepath, labels):\n",
        "    dataset_name = \"input_labels\"\n",
        "    with h5py.File(dest_filepath, 'a') as f:\n",
        "        f.create_dataset(dataset_name, (len(labels),), np.int8)\n",
        "        f[dataset_name][...] = labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZPtIOKCmedPG"
      },
      "source": [
        "### Set labels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B0RLXvn4eAub"
      },
      "source": [
        "Numbers for labelling\n",
        "- 0: Normal\n",
        "- 1: Bacterial\n",
        "- 2: Viral"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gM9aQH3ldt7c"
      },
      "source": [
        "def set_label(filepath):\n",
        "  if 'Bacterial' in filepath:\n",
        "    return 1\n",
        "  elif 'Viral' in filepath:\n",
        "    return 2\n",
        "  else:\n",
        "    return 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bmMVs0TneGdz"
      },
      "source": [
        "### Combined function for converting images and writing labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XmLOed2bdtv0"
      },
      "source": [
        "def convert_images_to_data_in_h5_file(src_img_filepath, dest_h5_file_path, n_px, n_channels = 3, batch_size = 1024):\n",
        "    \n",
        "    # Returns a list of filepaths ending with .jpeg or .png extension in the source directory and its sub-directories\n",
        "    src_filepaths = [\n",
        "                     os.path.join(dp, f) \n",
        "                     for dp, dn, filenames in os.walk(src_img_filepath) \n",
        "                     for f in filenames \n",
        "                     if os.path.splitext(f)[1] in ['.jpeg', '.png']\n",
        "                     ]\n",
        "    print('total no. of images = ', len(src_filepaths))\n",
        "    # Create Labels based upon the substring contained in the filename\n",
        "    labels = [set_label(filepath) for filepath in src_filepaths]\n",
        "\n",
        "    count_normal, count_bacterial, count_viral = 0, 0, 0\n",
        "    for l in labels:\n",
        "      if l == 0:\n",
        "        count_normal+=1\n",
        "      elif l==1:\n",
        "        count_bacterial+=1\n",
        "      else:\n",
        "        count_viral+=1\n",
        "    print('Normal:', count_normal, '\\t Bacterial:', count_bacterial, '\\t Viral:', count_viral)\n",
        "    \n",
        "    #The zip(source_filepaths, labels) combines each element of source_filepaths list \n",
        "    #with each element of labels list forming a pair (tuple). t is the list which contains these tuples\n",
        "    t = list(zip(src_filepaths, labels))\n",
        "\n",
        "    #Shuffle the list\n",
        "    shuffle(t)\n",
        "    \n",
        "    #Get the shuffled filepaths & labels\n",
        "    src_filepaths, labels = zip(*t)\n",
        "    \n",
        "    #Number of images\n",
        "    m = len(src_filepaths)\n",
        "    n_complete_batches = math.ceil(m / batch_size)\n",
        "\n",
        "    print('No. of complete batches = ', n_complete_batches)\n",
        "    \n",
        "    for i in range(n_complete_batches):\n",
        "        print('Creating file', (i+1))\n",
        "        \n",
        "        dest_file_path = dest_h5_file_path + str(i + 1) + \".h5\"   \n",
        "        \n",
        "        start_pos = i * batch_size\n",
        "        end_pos = min(start_pos + batch_size, m)\n",
        "        src_filepaths_batch = src_filepaths[start_pos: end_pos]\n",
        "        labels_batch = labels[start_pos: end_pos]\n",
        "        \n",
        "        normalize_and_write_data_into_h5_file(dest_file_path, src_filepaths_batch, n_px, n_channels)\n",
        "        write_labels_into_h5_file(dest_file_path, labels_batch)\n",
        "    return n_complete_batches"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YXer_6SjexHG"
      },
      "source": [
        "## Create h5 files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rHfX4UNifLHb"
      },
      "source": [
        "# Create dataseth5 folder if not already existing\n",
        "% mkdir -p /content/drive/MyDrive/main_project/dataseth5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yz3SubRHdtiJ"
      },
      "source": [
        "# root directory for image files\n",
        "root_dir = '/content/drive/MyDrive/main_project/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nwQiasNWh-ys"
      },
      "source": [
        "# image shape parameters\n",
        "n_px = 128\n",
        "n_channels = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vbJcPEsrguKl"
      },
      "source": [
        "Divide dataset to 10 batches and convert to h5 files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQ2LueCnhBIc"
      },
      "source": [
        "# destination file path including filename (starting), for each batch\n",
        "dest_filepath = '/content/drive/MyDrive/main_project/dataseth5/dataseth5_'\n",
        "\n",
        "# divides dataset to 10\n",
        "batch_size = int(2780 * 3 / 10)\n",
        "\n",
        "# create h5 files\n",
        "tic = time.process_time()\n",
        "n_complete_batches = convert_images_to_data_in_h5_file(root_dir, dest_filepath, n_px, n_channels, batch_size)\n",
        "toc = time.process_time()\n",
        "print('No. of complete batches = ', n_complete_batches, '\\n Time taken for creating the h5 files is', (toc-tic)*1000, 'ms')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BNRNc8DvgnhZ"
      },
      "source": [
        "Convert complete dataset to h5 file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xLJ8rdxff1LO"
      },
      "source": [
        "# destination file path including filename (starting), for complete dataset\n",
        "dest_filepath = '/content/drive/MyDrive/main_project/dataseth5/complete_dataseth5_'\n",
        "\n",
        "batch_size = 2780 * 3\n",
        "\n",
        "# create h5 file\n",
        "tic = time.process_time()\n",
        "convert_images_to_data_in_h5_file(root_dir, dest_filepath, n_px, n_channels, batch_size)\n",
        "toc = time.process_time()\n",
        "print('Time taken for creating the h5 file is', (toc-tic)*1000, 'ms')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d-JNzAzmjKYv"
      },
      "source": [
        "# Loading data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tPiXQUWmkJJW"
      },
      "source": [
        "Load a single h5 file to arrays x and y"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pkuw7vrgkIMS"
      },
      "source": [
        "batch_no = 1\n",
        "with h5py.File(dest_filepath+ str(batch_no) + '.h5', \"r\") as f:\n",
        "  x = f[\"input_data\"][:]\n",
        "  y = f[\"input_labels\"][:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ImQO9oQLjm1P"
      },
      "source": [
        "Combine data from all batches and load to arrays x and y"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xERR6ks0jO89"
      },
      "source": [
        "for j in range(n_complete_batches):\n",
        "  with h5py.File(dest_filepath+ str(j+1) + '.h5', \"r\") as f:\n",
        "    if j == 0:\n",
        "      x = f[\"input_data\"][:]\n",
        "      y = f[\"input_labels\"][:]\n",
        "    else:\n",
        "      x = np.concatenate((x, f[\"input_data\"][:]), axis=0)\n",
        "      y = np.concatenate((y, f[\"input_labels\"][:]), axis=0)\n",
        "  f.close()\n",
        "x = np.reshape(x, (x.shape[0], 128, 128, 1))\n",
        "print('x shape =', x.shape, '| y shape =', y.shape)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}