notMNIST_SGD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (20000, 28, 28) (20000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "#Reloading the data we generated in notMNIST.ipynb for simple gradient descent\n",
    "\n",
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save #hint to help gc free up memory\n",
    "    print(\"Training set\", train_dataset.shape, train_labels.shape)\n",
    "    print(\"Validation set\", valid_dataset.shape, valid_labels.shape)\n",
    "    print(\"Test set\", test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (20000, 784) (20000, 10)\n",
      "Validation set (10000, 784) (10000, 10)\n",
      "Test set (10000, 784) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "#Reformatting into a shape that's more adapted to the models we are going to train,i.e.,\n",
    "#1. Data as flat matrix, and\n",
    "#2. Labels as float 1-hot encodings.\n",
    "\n",
    "image_size = 28\n",
    "num_labels = 10\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape((-1, image_size*image_size)).astype(np.float32)\n",
    "    labels = (np.arange(num_labels) == labels[:, None]).astype(np.float32)\n",
    "    return dataset, labels\n",
    "\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "\n",
    "print(\"Training set\", train_dataset.shape, train_labels.shape)\n",
    "print(\"Validation set\", valid_dataset.shape, valid_labels.shape)\n",
    "print(\"Test set\", test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-0340dd231168>:36: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#First training multinomial logistic regression using Gradient Descent.\n",
    "#TensorFlow works like this...\n",
    "\n",
    "#Firstly, we need to describe the computation we want to see performed: what the inputs, the variables and \n",
    "#the operations looks like. These get created as nodes over a computation graph,i.e., with graph.as_default():\n",
    "\n",
    "#Secondly, after creating nodes on the graph, we can run the operations on the graph as many times as we want by\n",
    "#calling session.run() , providing it's outputs to fetch from the graph that get returned. The run time operation is:\n",
    "#    with tf.Session(graph=graph) as session:..\n",
    "\n",
    "#Now loading all the data into the tensorflow and building the computation graph corresponding to our training.\n",
    "\n",
    "#with gradient descent training this much data is prohibitive, hence, we are taking the subset of data for faster turnaround.\n",
    "\n",
    "train_subset = 10000\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    \n",
    "    #loading the traing, validation and test data into constants that are attached to the graph.\n",
    "    tf_train_dataset = tf.constant(train_dataset[:train_subset, :])\n",
    "    tf_train_labels = tf.constant(train_labels[:train_subset])\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "    \n",
    "    #Variables\n",
    "    #These are the parameters that are going to be trained.The weight matrix will be initialized using random values\n",
    "    #following a (truncated) normal distribution. The biases get initialized to zero.\n",
    "    weights = tf.Variable(tf.truncated_normal([image_size * image_size, num_labels]))\n",
    "    biases = tf.Variable(tf.zeros([num_labels]))\n",
    "    \n",
    "    #Training computation.\n",
    "    #We multiply the inputs with the weight matrix,and add biases. WEe compute the softmax and cross-entropy(it's one \n",
    "    #operation in tensorflow). We take the average of this cross-entropy across all training examples(that'sour loss).\n",
    "    logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "    \n",
    "    #finding the minimum of this loss using gradient descent with learning rate as low as ~0.5.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "    \n",
    "    #predictions for training, validation and test data. These are not part of training nut merely here to report\n",
    "    #accuracy figure as we train.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Loss at step 0: 13.042050\n",
      "Training accuracy: 15.2%\n",
      "Validation accuracy: 22.0%\n",
      "Loss at step 100: 2.334176\n",
      "Training accuracy: 72.1%\n",
      "Validation accuracy: 71.0%\n",
      "Loss at step 200: 1.876901\n",
      "Training accuracy: 74.8%\n",
      "Validation accuracy: 73.3%\n",
      "Loss at step 300: 1.636075\n",
      "Training accuracy: 76.3%\n",
      "Validation accuracy: 74.3%\n",
      "Loss at step 400: 1.473008\n",
      "Training accuracy: 77.1%\n",
      "Validation accuracy: 74.7%\n",
      "Loss at step 500: 1.350556\n",
      "Training accuracy: 77.6%\n",
      "Validation accuracy: 74.7%\n",
      "Loss at step 600: 1.253723\n",
      "Training accuracy: 78.2%\n",
      "Validation accuracy: 75.0%\n",
      "Loss at step 700: 1.174543\n",
      "Training accuracy: 78.7%\n",
      "Validation accuracy: 75.2%\n",
      "Loss at step 800: 1.108290\n",
      "Training accuracy: 78.9%\n",
      "Validation accuracy: 75.3%\n",
      "Test accuracy: 82.8%\n"
     ]
    }
   ],
   "source": [
    "#running the above computation with iterating 801 time.\n",
    "\n",
    "num_steps = 801\n",
    "\n",
    "def accuracy(predictions, labels):\n",
    "    return (100.0*np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    #this is a one time operation which ensures the parameters get initialized as we describe in the graph:\n",
    "    #random weights for the matrix and zeros for the biases.\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        #Running the computations. We tell .run() that we want to run the optimizer, get the loss value and the training\n",
    "        #predictions returned as numpy array.\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction])\n",
    "        if(step % 100 == 0):\n",
    "            print(\"Loss at step %d: %f\" % (step, l))\n",
    "            print('Training accuracy: %.1f%%' % accuracy(predictions, train_labels[:train_subset, :]))\n",
    "            #calling .eval() on vallid_prediction is basically like callin run(), but just to get \n",
    "            #that one numpy array. It recomputes all it's graph dependencies.\n",
    "            print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n",
    "            \n",
    "    print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#switching to Stochastic Gradient Descent (SGD) training, which is much faster. \n",
    "#The graph will be similar, except that instead of holding all the training data in the constant node,\n",
    "#we create a placeholder nodewhich will be fed actual data at every call of session.run()\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data. For the training data, we use a placeholder that will be fed\n",
    "  # at run time with a training minibatch.\n",
    "  tf_train_dataset = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, image_size * image_size))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  weights = tf.Variable(\n",
    "    tf.truncated_normal([image_size * image_size, num_labels]))\n",
    "  biases = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "  \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_prediction = tf.nn.softmax(\n",
    "    tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "  test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 16.489494\n",
      "Minibatch accuracy: 4.7%\n",
      "Validation accuracy: 11.8%\n",
      "Minibatch loss at step 100: 2.819848\n",
      "Minibatch accuracy: 66.4%\n",
      "Validation accuracy: 70.7%\n",
      "Minibatch loss at step 200: 1.133360\n",
      "Minibatch accuracy: 79.7%\n",
      "Validation accuracy: 72.8%\n",
      "Minibatch loss at step 300: 2.144049\n",
      "Minibatch accuracy: 71.1%\n",
      "Validation accuracy: 73.3%\n",
      "Minibatch loss at step 400: 1.580330\n",
      "Minibatch accuracy: 77.3%\n",
      "Validation accuracy: 74.4%\n",
      "Minibatch loss at step 500: 0.934724\n",
      "Minibatch accuracy: 80.5%\n",
      "Validation accuracy: 73.6%\n",
      "Minibatch loss at step 600: 1.603681\n",
      "Minibatch accuracy: 77.3%\n",
      "Validation accuracy: 74.5%\n",
      "Minibatch loss at step 700: 1.337184\n",
      "Minibatch accuracy: 75.8%\n",
      "Validation accuracy: 74.2%\n",
      "Minibatch loss at step 800: 1.727438\n",
      "Minibatch accuracy: 74.2%\n",
      "Validation accuracy: 75.2%\n",
      "Minibatch loss at step 900: 1.515379\n",
      "Minibatch accuracy: 71.9%\n",
      "Validation accuracy: 74.9%\n",
      "Minibatch loss at step 1000: 1.150156\n",
      "Minibatch accuracy: 75.8%\n",
      "Validation accuracy: 75.2%\n",
      "Minibatch loss at step 1100: 1.017728\n",
      "Minibatch accuracy: 76.6%\n",
      "Validation accuracy: 75.8%\n",
      "Minibatch loss at step 1200: 1.081184\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 75.5%\n",
      "Minibatch loss at step 1300: 1.027996\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 75.5%\n",
      "Minibatch loss at step 1400: 1.368840\n",
      "Minibatch accuracy: 71.1%\n",
      "Validation accuracy: 75.0%\n",
      "Minibatch loss at step 1500: 1.039569\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 75.6%\n",
      "Minibatch loss at step 1600: 0.777838\n",
      "Minibatch accuracy: 85.2%\n",
      "Validation accuracy: 76.0%\n",
      "Minibatch loss at step 1700: 1.005737\n",
      "Minibatch accuracy: 75.8%\n",
      "Validation accuracy: 75.8%\n",
      "Minibatch loss at step 1800: 0.925603\n",
      "Minibatch accuracy: 75.8%\n",
      "Validation accuracy: 75.3%\n",
      "Minibatch loss at step 1900: 0.664854\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 76.1%\n",
      "Minibatch loss at step 2000: 0.963673\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 76.1%\n",
      "Minibatch loss at step 2100: 0.706349\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 76.5%\n",
      "Minibatch loss at step 2200: 0.975133\n",
      "Minibatch accuracy: 84.4%\n",
      "Validation accuracy: 76.4%\n",
      "Minibatch loss at step 2300: 0.848303\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 76.2%\n",
      "Minibatch loss at step 2400: 1.034439\n",
      "Minibatch accuracy: 74.2%\n",
      "Validation accuracy: 76.6%\n",
      "Minibatch loss at step 2500: 0.671277\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 75.9%\n",
      "Minibatch loss at step 2600: 0.731275\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 76.2%\n",
      "Minibatch loss at step 2700: 0.763310\n",
      "Minibatch accuracy: 79.7%\n",
      "Validation accuracy: 75.8%\n",
      "Minibatch loss at step 2800: 0.540561\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 76.5%\n",
      "Minibatch loss at step 2900: 0.745586\n",
      "Minibatch accuracy: 83.6%\n",
      "Validation accuracy: 75.8%\n",
      "Minibatch loss at step 3000: 0.591962\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 77.0%\n",
      "Test accuracy: 85.0%\n"
     ]
    }
   ],
   "source": [
    "#running the above SGD computation and iterating it over 3000 times.\n",
    "\n",
    "num_steps = 3001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        #picking an offset with training data which has been randomized (we could use better randomization across epoch).\n",
    "        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        #generating a minbatch\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "        batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "        #preparing a dictionary, telling the session where to fit the minibatch.\n",
    "        #The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        #and the value is the numpy array to feed to it.\n",
    "        feed_dict = {tf_train_dataset: batch_data, tf_train_labels: batch_labels}\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict = feed_dict)\n",
    "        if(step%100 == 0):\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.1f%%\" %accuracy(predictions, batch_labels))\n",
    "            print(\"Validation accuracy: %.1f%%\" %accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print(\"Test accuracy: %.1f%%\" %accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#turning the logistic regression with SGD into a 1-hidden layer neural network with rectified linear units nn.relu() and 1024 hidden nodes. This model should improve our validation/test accuracy.\n",
    "\n",
    "num_nodes = 1024\n",
    "batch_size = 128\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    #input data\n",
    "    #For the training data, we use a placeholder that will be fed at runtime with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "    \n",
    "    #variables\n",
    "    weights1 = tf.Variable(tf.truncated_normal([image_size * image_size, num_nodes]))\n",
    "    biases1 = tf.Variable(tf.zeros([num_nodes]))\n",
    "    weights2 = tf.Variable(tf.truncated_normal([num_nodes, num_labels]))\n",
    "    biases2 = tf.Variable(tf.zeros([num_labels]))\n",
    "    \n",
    "    #training computation\n",
    "    logits1 = tf.matmul(tf_train_dataset, weights1) + biases1\n",
    "    relu_layer = tf.nn.relu(logits1)\n",
    "    logits2 = tf.matmul(relu_layer, weights2) + biases2\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = tf_train_labels, logits = logits2))\n",
    "    \n",
    "    #optimizer\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "    \n",
    "    #predictions for training\n",
    "    train_prediction = tf.nn.softmax(logits2)\n",
    "    \n",
    "    #predictions for validation\n",
    "    logits1 = tf.matmul(tf_valid_dataset, weights1) + biases1\n",
    "    relu_layer = tf.nn.relu(logits1)\n",
    "    logits2 = tf.matmul(relu_layer, weights2) + biases2\n",
    "    valid_prediction = tf.nn.softmax(logits2)\n",
    "    \n",
    "    #predictions for test\n",
    "    logits1 = tf.matmul(tf_test_dataset, weights1) + biases1\n",
    "    relu_layer = tf.nn.relu(logits1)\n",
    "    logits2 = tf.matmul(relu_layer, weights2) + biases2\n",
    "    test_prediction = tf.nn.softmax(logits2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\users\\navneet\\desktop\\udacity\\ml\\deeplearning\\myvenv\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:118: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "Initialized\n",
      "Minibatch loss at step 0: 395.735962\n",
      "Minibatch accuracy: 8.6%\n",
      "Validation accuracy: 40.5%\n",
      "Minibatch loss at step 100: 35.624657\n",
      "Minibatch accuracy: 78.1%\n",
      "Validation accuracy: 68.6%\n",
      "Minibatch loss at step 200: 23.435997\n",
      "Minibatch accuracy: 82.0%\n",
      "Validation accuracy: 78.9%\n",
      "Minibatch loss at step 300: 21.818375\n",
      "Minibatch accuracy: 78.9%\n",
      "Validation accuracy: 80.1%\n",
      "Minibatch loss at step 400: 6.638374\n",
      "Minibatch accuracy: 85.2%\n",
      "Validation accuracy: 79.6%\n",
      "Minibatch loss at step 500: 2.314669\n",
      "Minibatch accuracy: 93.0%\n",
      "Validation accuracy: 80.1%\n",
      "Minibatch loss at step 600: 7.265114\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.4%\n",
      "Minibatch loss at step 700: 6.322563\n",
      "Minibatch accuracy: 89.1%\n",
      "Validation accuracy: 79.8%\n",
      "Minibatch loss at step 800: 2.709992\n",
      "Minibatch accuracy: 87.5%\n",
      "Validation accuracy: 80.6%\n",
      "Minibatch loss at step 900: 2.385769\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 80.9%\n",
      "Minibatch loss at step 1000: 1.758142\n",
      "Minibatch accuracy: 93.0%\n",
      "Validation accuracy: 81.3%\n",
      "Minibatch loss at step 1100: 0.521940\n",
      "Minibatch accuracy: 96.1%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 1200: 0.311139\n",
      "Minibatch accuracy: 97.7%\n",
      "Validation accuracy: 82.0%\n",
      "Minibatch loss at step 1300: 0.494494\n",
      "Minibatch accuracy: 96.1%\n",
      "Validation accuracy: 81.6%\n",
      "Minibatch loss at step 1400: 4.770925\n",
      "Minibatch accuracy: 88.3%\n",
      "Validation accuracy: 80.6%\n",
      "Minibatch loss at step 1500: 1.362307\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 81.5%\n",
      "Minibatch loss at step 1600: 0.997683\n",
      "Minibatch accuracy: 93.0%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 1700: 0.722042\n",
      "Minibatch accuracy: 96.1%\n",
      "Validation accuracy: 81.6%\n",
      "Minibatch loss at step 1800: 0.458693\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 81.2%\n",
      "Minibatch loss at step 1900: 0.564925\n",
      "Minibatch accuracy: 97.7%\n",
      "Validation accuracy: 81.7%\n",
      "Minibatch loss at step 2000: 0.061205\n",
      "Minibatch accuracy: 99.2%\n",
      "Validation accuracy: 82.1%\n",
      "Minibatch loss at step 2100: 0.192065\n",
      "Minibatch accuracy: 97.7%\n",
      "Validation accuracy: 81.7%\n",
      "Minibatch loss at step 2200: 0.049258\n",
      "Minibatch accuracy: 99.2%\n",
      "Validation accuracy: 81.5%\n",
      "Minibatch loss at step 2300: 0.000001\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 2400: 3.904585\n",
      "Minibatch accuracy: 97.7%\n",
      "Validation accuracy: 81.8%\n",
      "Minibatch loss at step 2500: 1.265383\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 80.8%\n",
      "Minibatch loss at step 2600: 0.032160\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 82.4%\n",
      "Minibatch loss at step 2700: 0.549247\n",
      "Minibatch accuracy: 96.9%\n",
      "Validation accuracy: 82.4%\n",
      "Minibatch loss at step 2800: 0.848779\n",
      "Minibatch accuracy: 97.7%\n",
      "Validation accuracy: 82.2%\n",
      "Minibatch loss at step 2900: 0.166693\n",
      "Minibatch accuracy: 96.1%\n",
      "Validation accuracy: 82.1%\n",
      "Minibatch loss at step 3000: 0.022060\n",
      "Minibatch accuracy: 99.2%\n",
      "Validation accuracy: 81.9%\n",
      "Test accuracy: 89.5%\n"
     ]
    }
   ],
   "source": [
    "#running and iterating the relu computation\n",
    "\n",
    "num_steps = 3001\n",
    "\n",
    "with tf.Session(graph = graph) as session:\n",
    "    tf.initialize_all_variables().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        #picking an offset with training data that has been normalized(we could use better randomization across epochs)\n",
    "        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        #generating minibatch\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "        batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "        #preparing a dixtionary. Telling a session where to feed the minibatch. The key of the dictionary is the placeholder node of the graph to be fed, and the value is th numpy array to fed it\n",
    "        feed_dict = {tf_train_dataset: batch_data, tf_train_labels: batch_labels}\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict = feed_dict)\n",
    "        if(step%100 ==0):\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.1f%%\" %accuracy(predictions, batch_labels))\n",
    "            print(\"Validation accuracy: %.1f%%\" %accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print(\"Test accuracy: %.1f%%\" %accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}