notebooks/VisualizingFilters.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "enhanced-bicycle",
   "metadata": {},
   "source": [
    "# Plotting the filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "impaired-tension",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys; sys.path.insert(0, \"..\")\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "absent-wilderness",
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_devices = tf.config.experimental.list_physical_devices(\"GPU\")\n",
    "for device in gpu_devices:\n",
    "    tf.config.experimental.set_memory_growth(device, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "welcome-closure",
   "metadata": {},
   "outputs": [],
   "source": [
    "# modelPath = \".model_checkpoint/validationset/bps-augmentednet-rmsproplr01-210526T143649/94-4.336-0.7938.hdf5\" # best accuracy\n",
    "# modelPath = \".model_checkpoint/prototyping/dropout-210528T215602/82-4.383-0.7917.hdf5\" # with dropout\n",
    "# modelPath = \".model_checkpoint/validationset/rmsprop-lr0001-200epochs-210529T090623/171-4.237-0.7809.hdf5\" # low learning rate, 200 epochs\n",
    "# modelPath = \".model_checkpoint/testset/haydnsun-augmentednet-alldata-210513T195252/23-3.821-0.8027.hdf5\" # all data, evaluated on haydn\n",
    "# modelPath = \".model_checkpoint/validationset/rmsprop-learningschedule-piecewise-210530T101943/55-4.175-0.7928.hdf5\" # piecewise learning schedule\n",
    "# modelPath = \".model_checkpoint/validationset/rmsprop-learningschedule-piecewise-stair-210530T170008/44-4.311-0.7949.hdf5\" # piecewise stair\n",
    "# modelPath = \".model_checkpoint/validationset/wirwtc-rotatinglrschedule-210530T215055/55-5.002-0.7453.hdf5\" # wirwtc rotating lr\n",
    "# modelPath = \".model_checkpoint/validationset/bps-lr01-0001-210530T233203/55-4.109-0.7956.hdf5\" # bps, 01 to 0001\n",
    "modelPath = \".model_checkpoint/validationset/bps-lr01-0001-210530T233203/46-4.088-0.7950.hdf5\" # same as before, but lower loss instead of highest accuracy"
   ]
  },
  {
   "cell_type": "raw",
   "id": "planned-service",
   "metadata": {},
   "source": [
    "model = keras.models.load_model(modelPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "native-texas",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-6eced0fce677>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfilters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbiases\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "filters, biases = model.layers[2].get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "three-thought",
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = (filters - filters.min()) / (filters.max() - filters.min())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "conditional-batch",
   "metadata": {},
   "source": [
    "First filter of first convolutional layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "russian-brooklyn",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.heatmap(filters[0], cmap=\"inferno\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "coastal-output",
   "metadata": {},
   "source": [
    "# Exploring outputs of model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "honey-point",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baking-price",
   "metadata": {},
   "outputs": [],
   "source": [
    "[l.name for l in model.layers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amateur-covering",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adjusted-associate",
   "metadata": {},
   "outputs": [],
   "source": [
    "newModel = keras.Model(inputs=model.inputs[0], outputs=model.layers[6].output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "distinct-operations",
   "metadata": {},
   "outputs": [],
   "source": [
    "newModel.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "authentic-plastic",
   "metadata": {},
   "outputs": [],
   "source": [
    "exampleInput = np.zeros((640, 19))\n",
    "spellings = {\n",
    "    (0, 10): \"C--\", (0, 11): \"C-\", (0, 0): \"C\", (0, 1): \"C#\", (0, 2): \"C##\",\n",
    "    (1, 0): \"D--\", (1, 1): \"D-\", (1, 2): \"D\", (1, 3): \"D#\", (1, 4): \"D##\",\n",
    "    (2, 2): \"E--\", (2, 3): \"E-\", (2, 4): \"E\", (2, 5): \"E#\", (2, 6): \"E##\",\n",
    "    (3, 3): \"F--\", (3, 4): \"F-\", (3, 5): \"F\", (3, 6): \"F#\", (3, 7): \"F##\",\n",
    "    (4, 5): \"G--\", (4, 6): \"G-\", (4, 7): \"G\", (4, 8): \"G#\", (4, 9): \"G##\",\n",
    "    (5, 7): \"A--\", (5, 8): \"A-\", (5, 9): \"A\", (5, 10): \"A#\", (5, 11): \"A##\",\n",
    "    (6, 9): \"B--\", (6, 10): \"B-\", (6, 11): \"B\", (6, 0): \"B#\", (6, 1): \"B##\",\n",
    "}\n",
    "numberOfSpellings = len(spellings)\n",
    "\n",
    "for idx, (letter, pc) in enumerate(spellings.keys()):\n",
    "    exampleInput[idx][letter] = 1\n",
    "    exampleInput[idx][7 + pc] = 1\n",
    "\n",
    "exampleInput = exampleInput.reshape((1, 640, 19))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "guided-locking",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "plt.pcolormesh(exampleInput[0][:numberOfSpellings+5].T, edgecolors=\"k\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alternate-contest",
   "metadata": {},
   "outputs": [],
   "source": [
    "activations = newModel.predict(exampleInput)[0][:numberOfSpellings+5].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "awful-queens",
   "metadata": {},
   "outputs": [],
   "source": [
    "activations[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "suffering-harbor",
   "metadata": {},
   "outputs": [],
   "source": [
    "activations.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "blind-nebraska",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "plt.pcolormesh(activations, edgecolors=\"k\", cmap=\"inferno\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "duplicate-institution",
   "metadata": {},
   "outputs": [],
   "source": [
    "for fidx, f in enumerate(activations):\n",
    "    threshold = 0.5\n",
    "    idxAct = [(idx, act) for idx, act in enumerate(f[:numberOfSpellings]) if act > threshold]\n",
    "    idxAct.sort(key=lambda x: x[1], reverse=True)\n",
    "    highActivations = [x[0] for x in idxAct]\n",
    "#     print(highActivations)\n",
    "    noteSpellings = list(spellings.values())\n",
    "    notesActivated = [noteSpellings[idx] for idx in highActivations]\n",
    "    print(f\"Filter{fidx}: {notesActivated}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "convinced-antique",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}