pw04/CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HkLUJVa1bfNW"
   },
   "source": [
    "# Convolutional Neural Networks\n",
    "This notebook will guide you through the use of the `keras` package to train convolutional neural networks for handwritten digits classification. You are going to use the `mnist` dataset from LeCun et al. 1998."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SjQUQygFbfNb"
   },
   "source": [
    "## Loading the packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "h7O8MewabfNj",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from keras.datasets import mnist\n",
    "from keras.layers import Input, Dropout\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D\n",
    "from keras.layers.core import Dense, Flatten\n",
    "from keras.models import Sequential\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.utils import np_utils\n",
    "from matplotlib import pyplot as pl\n",
    "from sklearn import metrics as me\n",
    "\n",
    "from utils import plot_confusion_matrix\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vbttXt8SbfOH"
   },
   "source": [
    "First, create some useful functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jAR9jPgybfOO",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "def build_grid_of_images(array):\n",
    "    assert len(array.shape) == 3\n",
    "    dim_0 = np.sqrt(array.shape[0])\n",
    "    assert dim_0.is_integer()\n",
    "\n",
    "    temp_out = np.reshape(array, (dim_0, dim_0, array.shape[1], array.shape[2]))\n",
    "    temp_out = np.rollaxis(temp_out, 1, 3)\n",
    "    return np.reshape(temp_out, (dim_0 * array.shape[1], dim_0 * array.shape[2]))\n",
    "\n",
    "\n",
    "def plot_conv_layer_output(temp_out, title):\n",
    "    temp_to_plot = build_grid_of_images(temp_out)\n",
    "    pl.imshow(temp_to_plot, interpolation='nearest', cmap=pl.get_cmap('Greys'))\n",
    "    ax = pl.gca()\n",
    "    ax.set_xticks(np.arange(-0.5, temp_to_plot.shape[0] + 0.5, temp_out.shape[1]))\n",
    "    ax.set_yticks(np.arange(-0.5, temp_to_plot.shape[0] + 0.5, temp_out.shape[2]))\n",
    "    pl.grid()\n",
    "    pl.tick_params(axis='both', which='both', bottom='off', top='off', left='off', right='off', labelbottom='off',\n",
    "                   labelleft='off')\n",
    "    pl.title(title)\n",
    "\n",
    "\n",
    "def plot_dense_layer_output(temp_out, title):\n",
    "    pl.bar(np.arange(temp_out.shape[1]) - 0.4, temp_out[0, :])\n",
    "    pl.xlim(-0.5, temp_out.shape[1])\n",
    "    pl.grid()\n",
    "    pl.title(title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3fVOW4jGbfOb"
   },
   "source": [
    "Load the `mnist` dataset and normalize in the range [0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "id": "2H2K3S4MbfOf",
    "outputId": "87306b90-6068-49ae-d1c2-5d5ea69927e2",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "n_train, height, width = X_train.shape\n",
    "n_test, _, _ = X_test.shape\n",
    "\n",
    "X_train = X_train.reshape(n_train, height, width, 1).astype('float32')\n",
    "X_test = X_test.reshape(n_test, height, width, 1).astype('float32')\n",
    "\n",
    "X_train /= 255.0\n",
    "X_test /= 255.0\n",
    "\n",
    "n_classes = 10\n",
    "\n",
    "print(n_train, 'train samples')\n",
    "print(n_test, 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "Y_train = np_utils.to_categorical(y_train, n_classes)\n",
    "Y_test = np_utils.to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "print('X_train.shape = ', X_train.shape)\n",
    "print('Y_train.shape = ', Y_train.shape)\n",
    "print('X_test.shape = ', X_test.shape)\n",
    "print('Y_test.shape = ', Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2B-WNiQUbfOq"
   },
   "source": [
    "Create the CNN and show its architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 493
    },
    "id": "yhOscSNPbfOt",
    "outputId": "6e1a0da0-e48f-4fd1-eb5c-dadacdad548a",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "model = Sequential([\n",
    "    Input(shape=(height, width, 1), name='l0'),\n",
    "    Conv2D(32, kernel_size=(3, 3), activation='relu', name='l1'),\n",
    "    MaxPooling2D(pool_size=(2, 2), name='l1_mp'),\n",
    "    Conv2D(64, kernel_size=(3, 3), activation='relu', name='l2'),\n",
    "    MaxPooling2D(pool_size=(2, 2), name='l2_mp'),\n",
    "    Flatten(name='flat'),\n",
    "    Dropout(0.5, name='drop'),\n",
    "    Dense(n_classes, activation='softmax', name='l3'),\n",
    "])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sP_cugPCbfO6"
   },
   "source": [
    "Define some constants and train de CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 374
    },
    "id": "l8eKqD80bfPB",
    "outputId": "bb20620f-70c6-4330-8c5f-d67cb5176464",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "422/422 [==============================] - 4s 9ms/step - loss: 0.0807 - accuracy: 0.9751 - val_loss: 0.0552 - val_accuracy: 0.9853\n",
      "Epoch 4/15\n",
      "422/422 [==============================] - 4s 9ms/step - loss: 0.0688 - accuracy: 0.9784 - val_loss: 0.0468 - val_accuracy: 0.9863\n",
      "Epoch 5/15\n",
      "422/422 [==============================] - 4s 8ms/step - loss: 0.0622 - accuracy: 0.9801 - val_loss: 0.0452 - val_accuracy: 0.9878\n",
      "Epoch 6/15\n",
      "422/422 [==============================] - 4s 9ms/step - loss: 0.0572 - accuracy: 0.9826 - val_loss: 0.0379 - val_accuracy: 0.9902\n",
      "Epoch 7/15\n",
      "422/422 [==============================] - 4s 9ms/step - loss: 0.0517 - accuracy: 0.9849 - val_loss: 0.0388 - val_accuracy: 0.9893\n",
      "Epoch 8/15\n",
      "422/422 [==============================] - 4s 9ms/step - loss: 0.0476 - accuracy: 0.9851 - val_loss: 0.0358 - val_accuracy: 0.9905\n",
      "Epoch 9/15\n",
      "422/422 [==============================] - 4s 9ms/step - loss: 0.0447 - accuracy: 0.9863 - val_loss: 0.0334 - val_accuracy: 0.9908\n",
      "Epoch 10/15\n",
      "422/422 [==============================] - 4s 8ms/step - loss: 0.0421 - accuracy: 0.9865 - val_loss: 0.0329 - val_accuracy: 0.9910\n",
      "Epoch 11/15\n",
      "422/422 [==============================] - 4s 9ms/step - loss: 0.0402 - accuracy: 0.9879 - val_loss: 0.0316 - val_accuracy: 0.9912\n",
      "Epoch 12/15\n",
      "275/422 [==================>...........] - ETA: 1s - loss: 0.0376 - accuracy: 0.9885"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "n_epoch = 15\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(X_train, Y_train, batch_size=batch_size, epochs=n_epoch, verbose=1, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IBX2Tc_0bfPK"
   },
   "source": [
    "Show the performance of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 299
    },
    "id": "WeaMjl-EbfPS",
    "outputId": "e219b778-8eb2-4fd9-a2f5-baaa3d936a3f",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "pl.plot(history.history['loss'], label='Training')\n",
    "pl.plot(history.history['val_loss'], label='Testing')\n",
    "pl.legend()\n",
    "pl.grid()\n",
    "\n",
    "score = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test score:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rXNZ2ZHSbfPm"
   },
   "source": [
    "Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 187
    },
    "id": "XfgRipFebfPm",
    "outputId": "b435b340-a409-423f-bd5a-d989f6f76a8a",
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "pred = model.predict(X_test)\n",
    "print('pred.shape = ', pred.shape)\n",
    "pred = np.argmax(pred, axis=-1)\n",
    "cm = me.confusion_matrix(y_test, pred)\n",
    "\n",
    "plot_confusion_matrix(cm, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "CNN.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}