transfer_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Copy of Assignment-5 transfer_learning.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ApoorvaKhanna/TensorFlow_Intro/blob/master/Copy_of_Assignment_5_transfer_learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "hRTa3Ee15WsJ"
      },
      "source": [
        "# Transfer Learning Using Pretrained ConvNets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "2X4KyhORdSeO"
      },
      "source": [
        "You will follow the general machine learning workflow.\n",
        "\n",
        "1. Examine and understand the data\n",
        "2. Build an input pipeline\n",
        "3. Compose our model\n",
        "  * Part-1: Load in our pretrained base model (and pretrained weights)\n",
        "  * Part-2: Stack our classification layers on top\n",
        "4. Train our model\n",
        "5. Evaluate model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iBMcobPHdD8O",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "import os\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "TqOt6Sv7AsMi",
        "colab": {}
      },
      "source": [
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "import tensorflow as tf\n",
        "\n",
        "keras = tf.keras"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "v77rlkCKW0IJ"
      },
      "source": [
        "## Data preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "0GoKGm1duzgk"
      },
      "source": [
        "### Data download"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "KVh7rDVAuW8Y",
        "colab": {}
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "tfds.disable_progress_bar()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ro4oYaEmxe4r",
        "colab": {}
      },
      "source": [
        "SPLIT_WEIGHTS = (8, 1, 1)\n",
        "splits = tfds.Split.TRAIN.subsplit(weighted=SPLIT_WEIGHTS)\n",
        "(raw_train, raw_validation, raw_test), metadata = tfds.load(name=\"tf_flowers\", \n",
        "                                                            with_info=True,\n",
        "                                                            split=list(splits),                                         \n",
        "                                                            as_supervised=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "yO1Q2JaW5sIy"
      },
      "source": [
        "Show the first two images and labels from the training set:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "K5BeQyKThC_Y",
        "colab": {}
      },
      "source": [
        "get_label_name = metadata.features['label'].int2str\n",
        "\n",
        "for image, label in raw_train.take(2):\n",
        "  plt.figure()\n",
        "  plt.imshow(image)\n",
        "  plt.title(get_label_name(label))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wvidPx6jeFzf"
      },
      "source": [
        "### Format the Data\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "y3PM6GVHcC31",
        "colab": {}
      },
      "source": [
        "IMG_SIZE = 160\n",
        "\n",
        "def format_example(image, label):\n",
        "  image = tf.cast(image, tf.float32)\n",
        "  image = (image/255)\n",
        "  image = tf.image.resize(image, (IMG_SIZE, IMG_SIZE))\n",
        "z  return image, label"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "i2MRh_AeBtOM"
      },
      "source": [
        "Apply this function to each item in the dataset using the map method:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SFZ6ZW7KSXP9",
        "colab": {}
      },
      "source": [
        "train = raw_train.map(format_example)\n",
        "validation = raw_validation.map(format_example)\n",
        "test = raw_test.map(format_example)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "E5ifgXDuBfOC"
      },
      "source": [
        "Now shuffle and batch the data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Yic-I66m6Isv",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 32\n",
        "SHUFFLE_BUFFER_SIZE = 1000"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "p3UUPdm86LNC",
        "colab": {}
      },
      "source": [
        "train_batches = train.shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE)\n",
        "validation_batches = validation.batch(BATCH_SIZE)\n",
        "test_batches = test.batch(BATCH_SIZE)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "02rJpcFtChP0"
      },
      "source": [
        "\n",
        "### Q5 What is the shape of one batch of data?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "OkH-kazQecHB"
      },
      "source": [
        "# 1. Create the base model from the pre-trained convnets\n",
        "You will create the base model from the **Inception V3** model developed at Google. This is pre-trained on the ImageNet dataset, a large dataset of 1.4M images and 1000 classes of web images. This model has already learned the basic features that are common in 1000 objects that we see daily. Hence, it has a strong feature extraction capability. We download a network that doesn’t include the classification layers at the top, by specifying include_top=False argument, because we only want to use the feature extraction portion of these pre-trained convnets (convolutional base)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "19IQ2gqneqmS",
        "colab": {}
      },
      "source": [
        "IMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)\n",
        "\n",
        "# Create the base model from the pre-trained model Inception V3\n",
        "base_model = tf.keras.applications.InceptionV3(input_shape=IMG_SHAPE,\n",
        "                                               include_top=False,\n",
        "                                               weights='imagenet')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PgAzSNQJkBdk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for image_batch, label_batch in train_batches.take(1):\n",
        "   pass\n",
        "\n",
        "image_batch.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "AqcsxoJIEVXZ"
      },
      "source": [
        "### Q6 What is the shape of a new block of features converted by the feature extractor?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Y-2LJL0EEUcx",
        "colab": {}
      },
      "source": [
        "feature_batch = base_model(image_batch)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "aNPvfQjakROJ",
        "colab": {}
      },
      "source": [
        "print(feature_batch.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "rlx56nQtfe8Y"
      },
      "source": [
        "## Feature extraction\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "CnMLieHBCwil"
      },
      "source": [
        "### Freeze the convolutional base"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OTCJH4bphOeo",
        "colab": {}
      },
      "source": [
        "# Write the code here to freeze convolutional base\n",
        "base_model.trainable = False\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wdMRM8YModbk"
      },
      "source": [
        "### Add a classification head"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "dLnpMF5KOALm",
        "colab": {}
      },
      "source": [
        "global_average_layer = tf.keras.layers.GlobalAveragePooling2D()\n",
        "feature_batch_average = global_average_layer(feature_batch)\n",
        "print(feature_batch_average.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Wv4afXKj6cVa",
        "colab": {}
      },
      "source": [
        "prediction_layer = keras.layers.Dense(metadata.features['label'].num_classes, activation='softmax')\n",
        "prediction_batch = prediction_layer(feature_batch_average)\n",
        "print(prediction_batch.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "0iqnBeZrfoIc"
      },
      "source": [
        "Now stack the feature extractor, and these two layers using a `tf.keras.Sequential` model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eApvroIyn1K0",
        "colab": {}
      },
      "source": [
        "# Write the code here to stack the all these layers\n",
        "\n",
        "# model = ...\n",
        "model = tf.keras.Sequential([\n",
        "  base_model,\n",
        "  global_average_layer,\n",
        "  prediction_layer\n",
        "])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kFt6FCVixm4t",
        "colab_type": "text"
      },
      "source": [
        "### Q7 What is the total number trainable TF variables in the model?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hMwx_o1N9Bue",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nsbBvbY5lRlV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(model.trainable_variables)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "g0ylJXE_kRLi"
      },
      "source": [
        "### Compile the model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RpR8HdyMhukJ",
        "colab": {}
      },
      "source": [
        "model.compile(optimizer='adam',\n",
        "              loss='sparse_categorical_crossentropy',\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5b4Ez654luLe",
        "colab_type": "text"
      },
      "source": [
        "### Q8 What is the total numebr of Trainable parameters in this model?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbiI1N-glVdy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_model.summary()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Ma2tVH7m-Yj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RxvgOYTDSWTx"
      },
      "source": [
        "### Train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JsaRFlZ9B6WK",
        "colab": {}
      },
      "source": [
        "history = model.fit(train_batches,\n",
        "                    epochs=10,\n",
        "                    validation_data=validation_batches)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Hd94CKImf8vi"
      },
      "source": [
        "### Learning curves\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "53OTCh3jnbwV",
        "colab": {}
      },
      "source": [
        "acc = history.history['accuracy']\n",
        "val_acc = history.history['val_accuracy']\n",
        "\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "\n",
        "plt.figure(figsize=(8, 8))\n",
        "plt.subplot(2, 1, 1)\n",
        "plt.plot(acc, label='Training Accuracy')\n",
        "plt.plot(val_acc, label='Validation Accuracy')\n",
        "plt.legend(loc='lower right')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.ylim([min(plt.ylim()),1])\n",
        "plt.title('Training and Validation Accuracy')\n",
        "\n",
        "plt.subplot(2, 1, 2)\n",
        "plt.plot(loss, label='Training Loss')\n",
        "plt.plot(val_loss, label='Validation Loss')\n",
        "plt.legend(loc='upper right')\n",
        "plt.ylabel('Cross Entropy')\n",
        "plt.title('Training and Validation Loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "CqwV-CRdS6Nv"
      },
      "source": [
        "# 2. Fine tuning\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "CPXnzUK0QonF"
      },
      "source": [
        "### Un-freeze the top layers of the model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4nzcagVitLQm",
        "colab": {}
      },
      "source": [
        "# Write code here for un-freezing the top layers\n",
        "\n",
        "base_model.trainable = True\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "stzQkv2J8E4C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_model.summary()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-4HgVAacRs5v",
        "colab": {}
      },
      "source": [
        "# Write the code for freezing layers before layer 250 in the model\n",
        "for i,layer in enumerate(base_model.layers):\n",
        "    print(i,layer) \n",
        "    if i<250:\n",
        "      print(\"Freezing\")\n",
        "      layer.trainable = False\n",
        "    else:\n",
        "      print(\"Done\")\n",
        "      break\n",
        "  #model.compile(...)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EaKL0-C74jnZ",
        "colab_type": "text"
      },
      "source": [
        "### Q9 What is the total number trainable TF variables in the model?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "znabhHFN9GmN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(base_model.trainable_variables)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W-fs3BrT9n61",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# model = tf.keras.Sequential([\n",
        "#   base_model,\n",
        "#   global_average_layer,\n",
        "#   prediction_layer\n",
        "# ])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4Uk1dgsxT0IS"
      },
      "source": [
        "### Compile the model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "NtUnaz0WUDva",
        "colab": {}
      },
      "source": [
        "\n",
        "model.compile(optimizer = 'adam',\n",
        "              loss='sparse_categorical_crossentropy',\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VQHq0gmlo3_l",
        "colab_type": "text"
      },
      "source": [
        "### Q10 What is the total numebr of Trainable parameters in this model now?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "861rtny69uS7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EDBOMhnM_Bj0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(model.trainable_variables)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4G5O4jd6TuAG"
      },
      "source": [
        "### Continue Train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ECQLkAsFTlun",
        "colab": {}
      },
      "source": [
        "initial_epochs=10\n",
        "total_epochs = initial_epochs + 10\n",
        "\n",
        "history_fine = model.fit(train_batches,\n",
        "                         epochs=total_epochs,\n",
        "                         initial_epoch = initial_epochs,\n",
        "                         validation_data=validation_batches)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kQA8MmbN4szm",
        "colab_type": "text"
      },
      "source": [
        "### Q11 Let the training accuracy in part-1 be x and training accuracy in part-2 be y (both after training is done). The value of | y-x | lies between?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "PpA8PlpQKygw",
        "colab": {}
      },
      "source": [
        "#print(history['accuracy']-history_fine['accuracy'])\n",
        "\n",
        "\n",
        "acc += history_fine.history['accuracy']\n",
        "val_acc += history_fine.history['val_accuracy']\n",
        "loss += history_fine.history['loss']\n",
        "val_loss += history_fine.history['val_loss']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "chW103JUItdk",
        "colab": {}
      },
      "source": [
        "plt.figure(figsize=(8, 8))\n",
        "plt.subplot(2, 1, 1)\n",
        "plt.plot(acc, label='Training Accuracy')\n",
        "plt.plot(val_acc, label='Validation Accuracy')\n",
        "plt.plot([initial_epochs-1,initial_epochs-1],\n",
        "          plt.ylim(), label='Start Fine Tuning')\n",
        "plt.legend(loc='lower right')\n",
        "plt.title('Training and Validation Accuracy')\n",
        "\n",
        "plt.subplot(2, 1, 2)\n",
        "plt.plot(loss, label='Training Loss')\n",
        "plt.plot(val_loss, label='Validation Loss')\n",
        "plt.plot([initial_epochs-1,initial_epochs-1],\n",
        "         plt.ylim(), label='Start Fine Tuning')\n",
        "plt.legend(loc='upper right')\n",
        "plt.title('Training and Validation Loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_E6hyup_Q9h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}