Fine-tuning models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-tuning pre-trained models (torchvision)\n",
    "ref:https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pytorch Version:  0.4.1.post2\n",
      "Torchvision Version:  0.2.1\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "\n",
    "print(\"Pytorch Version: \", torch.__version__)\n",
    "print(\"Torchvision Version: \", torchvision.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We will use the hymenoptera_data dataset which can be downloaded here. This dataset contains two classes, bees and ants."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/home/zxw/datasets/hymenoptera_data/\"\n",
    "model_name = \"squeezenet\"\n",
    "num_classes = 2\n",
    "batch_size = 8\n",
    "num_epochs = 15\n",
    "\n",
    "# Flag for feature extracting. When False, we finetune the whole model,\n",
    "#   when True we only update the reshaped layer params\n",
    "feature_extract = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, dataloaders, criterion, optimizer, num_epochs = 25, is_inception = False):\n",
    "    since = time.time()\n",
    "    val_acc_history = []\n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    \n",
    "    best_acc = 0.0\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "        \n",
    "        for phase in ['train', 'val']:\n",
    "            \n",
    "            if phase == 'train':\n",
    "                model.train()\n",
    "            else:\n",
    "                model.eval()\n",
    "            \n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            \n",
    "            \n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                \n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # Get model outputs and calculate loss\n",
    "                    # Special case for inception because in training it has an auxiliary output. In train\n",
    "                    #   mode we calculate the loss by summing the final output and the auxiliary output\n",
    "                    #   but in testing we only consider the final output.\n",
    "                    if is_inception and phase == 'train':\n",
    "                        # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958\n",
    "                        outputs, aux_outputs = model(inputs)\n",
    "                        loss1 = criterion(outputs, labels)\n",
    "                        loss2 = criterion(aux_outputs, labels)\n",
    "                        loss = loss1 + 0.4*loss2\n",
    "                    else:\n",
    "                        outputs = model(inputs)\n",
    "                        loss = criterion(outputs, labels)\n",
    "\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                # statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "\n",
    "            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n",
    "            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "\n",
    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "\n",
    "            \n",
    "            # deep copy the model\n",
    "            if phase == 'val' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "            if phase == 'val':\n",
    "                val_acc_history.append(epoch_acc)\n",
    "\n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "\n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, val_acc_history\n",
    "\n",
    "\n",
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "            \n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SqueezeNet(\n",
      "  (features): Sequential(\n",
      "    (0): Conv2d(3, 96, kernel_size=(7, 7), stride=(2, 2))\n",
      "    (1): ReLU(inplace)\n",
      "    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)\n",
      "    (3): Fire(\n",
      "      (squeeze): Conv2d(96, 16, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "    (4): Fire(\n",
      "      (squeeze): Conv2d(128, 16, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(16, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "    (5): Fire(\n",
      "      (squeeze): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "    (6): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)\n",
      "    (7): Fire(\n",
      "      (squeeze): Conv2d(256, 32, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(32, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "    (8): Fire(\n",
      "      (squeeze): Conv2d(256, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(48, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "    (9): Fire(\n",
      "      (squeeze): Conv2d(384, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(48, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "    (10): Fire(\n",
      "      (squeeze): Conv2d(384, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "    (11): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)\n",
      "    (12): Fire(\n",
      "      (squeeze): Conv2d(512, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (squeeze_activation): ReLU(inplace)\n",
      "      (expand1x1): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (expand1x1_activation): ReLU(inplace)\n",
      "      (expand3x3): Conv2d(64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (expand3x3_activation): ReLU(inplace)\n",
      "    )\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.5)\n",
      "    (1): Conv2d(512, 2, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (2): ReLU(inplace)\n",
      "    (3): AvgPool2d(kernel_size=13, stride=1, padding=0)\n",
      "  )\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zxw/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "/home/zxw/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n"
     ]
    }
   ],
   "source": [
    "def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):  # feture_extract = True\n",
    "    # Initialize these variables which will be set in this if statement. Each of these\n",
    "    #   variables is model specific.\n",
    "    model_ft = None\n",
    "    input_size = 0\n",
    "\n",
    "    if model_name == \"resnet\":\n",
    "        \"\"\" Resnet18\n",
    "        \"\"\"\n",
    "        model_ft = models.resnet18(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.fc.in_features\n",
    "        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n",
    "        input_size = 224\n",
    "\n",
    "    elif model_name == \"alexnet\":\n",
    "        \"\"\" Alexnet\n",
    "        \"\"\"\n",
    "        model_ft = models.alexnet(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.classifier[6].in_features\n",
    "        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n",
    "        input_size = 224\n",
    "\n",
    "    elif model_name == \"vgg\":\n",
    "        \"\"\" VGG11_bn\n",
    "        \"\"\"\n",
    "        model_ft = models.vgg11_bn(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.classifier[6].in_features\n",
    "        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n",
    "        input_size = 224\n",
    "\n",
    "    elif model_name == \"squeezenet\":\n",
    "        \"\"\" Squeezenet\n",
    "        \"\"\"\n",
    "        model_ft = models.squeezenet1_0(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))\n",
    "        model_ft.num_classes = num_classes\n",
    "        input_size = 224\n",
    "\n",
    "    elif model_name == \"densenet\":\n",
    "        \"\"\" Densenet\n",
    "        \"\"\"\n",
    "        model_ft = models.densenet121(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.classifier.in_features\n",
    "        model_ft.classifier = nn.Linear(num_ftrs, num_classes)\n",
    "        input_size = 224\n",
    "\n",
    "    elif model_name == \"inception\":\n",
    "        \"\"\" Inception v3\n",
    "        Be careful, expects (299,299) sized images and has auxiliary output\n",
    "        \"\"\"\n",
    "        model_ft = models.inception_v3(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        # Handle the auxilary net\n",
    "        num_ftrs = model_ft.AuxLogits.fc.in_features\n",
    "        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)\n",
    "        # Handle the primary net\n",
    "        num_ftrs = model_ft.fc.in_features\n",
    "        model_ft.fc = nn.Linear(num_ftrs,num_classes)\n",
    "        input_size = 299\n",
    "\n",
    "    else:\n",
    "        print(\"Invalid model name, exiting...\")\n",
    "        exit()\n",
    "\n",
    "    return model_ft, input_size\n",
    "\n",
    "# Initialize the model for this run\n",
    "model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)\n",
    "\n",
    "# Print the model we just instantiated\n",
    "print(model_ft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing Datasets and Dataloaders...\n"
     ]
    }
   ],
   "source": [
    "# Data augmentation and normalization for training\n",
    "# Just normalization for validation\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop(input_size),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.Resize(input_size),\n",
    "        transforms.CenterCrop(input_size),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "\n",
    "print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "# Create training and validation datasets\n",
    "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val']}\n",
    "# Create training and validation dataloaders\n",
    "dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'val']}\n",
    "\n",
    "# Detect if we have a GPU available\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Params to learn:\n",
      "\t classifier.1.weight\n",
      "\t classifier.1.bias\n"
     ]
    }
   ],
   "source": [
    "# send the model to GPU\n",
    "model_ft = model_ft.to(device)\n",
    "\n",
    "# Gather the parameters to be optimized/updated in this run. If we are\n",
    "#  finetuning we will be updating all parameters. However, if we are\n",
    "#  doing feature extract method, we will only update the parameters\n",
    "#  that we have just initialized, i.e. the parameters with requires_grad\n",
    "#  is True.\n",
    "params_to_update = model_ft.parameters()\n",
    "\n",
    "print(\"Params to learn:\")\n",
    "if feature_extract:\n",
    "    params_to_update = []\n",
    "    for name,param in model_ft.named_parameters():\n",
    "        if param.requires_grad == True:\n",
    "            params_to_update.append(param)\n",
    "            print(\"\\t\",name)\n",
    "else:\n",
    "    for name,param in model_ft.named_parameters():\n",
    "        if param.requires_grad == True:\n",
    "            print(\"\\t\",name)\n",
    "\n",
    "# Observe that all parameters are being optimized\n",
    "optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/14\n",
      "----------\n",
      "train Loss: 0.6088 Acc: 0.6762\n",
      "val Loss: 0.3699 Acc: 0.8431\n",
      "\n",
      "Epoch 1/14\n",
      "----------\n",
      "train Loss: 0.3394 Acc: 0.8893\n",
      "val Loss: 0.4454 Acc: 0.7974\n",
      "\n",
      "Epoch 2/14\n",
      "----------\n",
      "train Loss: 0.3229 Acc: 0.8607\n",
      "val Loss: 0.2836 Acc: 0.9216\n",
      "\n",
      "Epoch 3/14\n",
      "----------\n",
      "train Loss: 0.2216 Acc: 0.9057\n",
      "val Loss: 0.2790 Acc: 0.9216\n",
      "\n",
      "Epoch 4/14\n",
      "----------\n",
      "train Loss: 0.2087 Acc: 0.9180\n",
      "val Loss: 0.2840 Acc: 0.9150\n",
      "\n",
      "Epoch 5/14\n",
      "----------\n",
      "train Loss: 0.2111 Acc: 0.9139\n",
      "val Loss: 0.3400 Acc: 0.8824\n",
      "\n",
      "Epoch 6/14\n",
      "----------\n",
      "train Loss: 0.1982 Acc: 0.9016\n",
      "val Loss: 0.2774 Acc: 0.9281\n",
      "\n",
      "Epoch 7/14\n",
      "----------\n",
      "train Loss: 0.1734 Acc: 0.9344\n",
      "val Loss: 0.3251 Acc: 0.9085\n",
      "\n",
      "Epoch 8/14\n",
      "----------\n",
      "train Loss: 0.1445 Acc: 0.9385\n",
      "val Loss: 0.3286 Acc: 0.9281\n",
      "\n",
      "Epoch 9/14\n",
      "----------\n",
      "train Loss: 0.1858 Acc: 0.9262\n",
      "val Loss: 0.3084 Acc: 0.9281\n",
      "\n",
      "Epoch 10/14\n",
      "----------\n",
      "train Loss: 0.1717 Acc: 0.9098\n",
      "val Loss: 0.3523 Acc: 0.9085\n",
      "\n",
      "Epoch 11/14\n",
      "----------\n",
      "train Loss: 0.1250 Acc: 0.9426\n",
      "val Loss: 0.3603 Acc: 0.9085\n",
      "\n",
      "Epoch 12/14\n",
      "----------\n",
      "train Loss: 0.1831 Acc: 0.9262\n",
      "val Loss: 0.3646 Acc: 0.9085\n",
      "\n",
      "Epoch 13/14\n",
      "----------\n",
      "train Loss: 0.1282 Acc: 0.9508\n",
      "val Loss: 0.3069 Acc: 0.9216\n",
      "\n",
      "Epoch 14/14\n",
      "----------\n",
      "train Loss: 0.1469 Acc: 0.9426\n",
      "val Loss: 0.3155 Acc: 0.9346\n",
      "\n",
      "Training complete in 0m 25s\n",
      "Best val Acc: 0.934641\n"
     ]
    }
   ],
   "source": [
    "# setup the loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# train and evaluate\n",
    "model_ft, hist = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft, num_epochs = num_epochs,\n",
    "                             is_inception = (model_name == 'inception'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/14\n",
      "----------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zxw/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "/home/zxw/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train Loss: 0.8218 Acc: 0.5410\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 1/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 2/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 3/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 4/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 5/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 6/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 7/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 8/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 9/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 10/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 11/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 12/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 13/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Epoch 14/14\n",
      "----------\n",
      "train Loss: 0.6931 Acc: 0.5041\n",
      "val Loss: 0.6931 Acc: 0.4575\n",
      "\n",
      "Training complete in 0m 41s\n",
      "Best val Acc: 0.457516\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8FeXZ//HPl7CEsIeIrAooliUgakTcKCpY0Qq2aq3VutRKq3XpYvvY+tNaq31camsftSJ1qQtad4vWrSiKuywiBBAERAn7GggESOD6/TF3DoeQ5YSck0PC9X698sqs91xnzpy5Zu6ZuUdmhnPOOQfQKN0BOOec23t4UnDOORfjScE551yMJwXnnHMxnhScc87FeFJwzjkX40khAZK6SzJJjUP/q5IuTGTaPVjW7yQ9UJt4Xf1X2+0oCcs/VtIXkooknZHiZWWE5RyQzGnrA0mPS7ox3XHE2yeSgqTXJd1UwfBRkpbX9IdnZiPM7JEkxDVUUkG5sv9kZj+ubdnVLNMk/SZVy2iIJF0U1tuvyw0vkDQ0TWGl0k3APWbW0sxejB8RdsplfzskFcf1n1fTBZnZ9rCcr5M5bU1JullSSbnPtzrZy9nb7RNJAfgn8ENJKjf8h8A4Myut+5DS5kJgbfhfp9J11JtEa4H/kdQ63YHUxB6u9wOBWRWNCDvllmbWEvgaOD1u2LgkLT9dxsV/PjPLSXdAdW1fSQovAtnA8WUDJLUDvg08GvpPk/SppA2SFld1SifpbUk/Dt0Zkv4sabWkhcBp5aa9WNIcSRslLZT0kzC8BfAq0DnuqKSzpBslPR43/0hJsyStD8vtEzdukaRrJM2QVCjpKUmZVcSdBZwF/AzoJSmv3PjjJH0QlrVY0kVheHNJd0r6KiznvTBstzOdENOw0H2jpGfDKfIG4CJJgyR9GJaxTNI9kprGzd9P0n8lrZW0IlSndZS0WVL7uOmOkLRKUpNyy+8cjlyz44YdFr6fJpIOlvRO+ByrJT1V2fqqwBzgQ+AXlazff0q6Oa5/l/UT1s2vw/e1SdKDkvZXVB25UdKEsF3G+5GkpWFd/SqurEaSrpW0QNIaSU+XfWbtrHq6RNLXwFuVxHuppPlhXY+X1DkMXwD0BF4K22WzGqyjsiPupyQ9KWkjcL6koyV9FPe9/1/ZdyepcYi3e+h/PIwvWy8fSupR02nD+BGS5oXv+25J75dt1zX8TGXLvVLSl2HbuVVSozC+kaQbwm9kZdgWWsfNPyR8/kJFv60fxhWfXclnbRQ+28ow3wxJfWsae42Z2T7xB/wDeCCu/yfA9Lj+oUB/okQ5AFgBnBHGdQcMaBz63wZ+HLp/CnwOdCNKPBPLTXsacBAg4JvAZuDwuGUWlIvzRuDx0H0IsAkYDjQBfgPMB5qG8YuAT4DOYdlzgJ9WsQ5+CCwDMoCXgP+LG3cAsBE4NyyrPTAwjLs3fOYuYd5jgGaVxL8IGBb3WUqAM8J6bQ4cAQwGGof1Ogf4eZi+VYjvV0Bm6D8qjHsFuCxuOX8F7q7kc74FXBrXfwcwJnQ/CVwX4skEjktw+7kIeA8YCKwHssPwAmBo6P4ncHO5baqg3Lr5CNg/rMuVwDTgsLA+3wJ+X26bexJoQbRtropbtz8PZXUN894PPFlu3kfDvM0r+DwnAquBw8P8dwOTKvoeq1kvu00H3AxsA06P+96PBI4K33tPYB5wRZi+cYi3e+h/PMSWR7QtPsXO30RNpu1AtE2PCuN+SbQ9XlTJZ7kZ+Gcl48qWOwFoF9bx/LKygNHhM/Ug2m7/DTwcxvUIcXwvlJPDzt9WVfGfRvT7bhPWY1+gY8r3lalewN7yBxwHFJb9QID3gV9UMf1dwF/L/cgqSgpvEbcjBk6On7aCcl8Erg7dQ6k6KVwPPB03rhGwhJ07oUXA+XHjbyfs/CpZ9gTgrtB9LtFOpkno/y3wQgXzNAKKgUMrGFdR/IvYNSlMqiyeMM3Py5YbYvq0kunOAd4P3RnAcmBQJdP+GHgrdAtYDAwJ/Y8CY4GuNdx+LgLeC91PA7eF7pomhfPi+p8D7ovrvxJ4sdw217vc9/tg6J4DnBQ3rhPRDq9x3Lw9q/g8DwK3x/W3DPN3L/89VrNedpuOaOf6VjXzXQM8E7or2tGPiZt2JJC/B9P+CHg3bpyIDjouqiSmsmS2Pu7vv+WWOyxu+quA10P3O8DouHH9gK1Ev5/ryz5rBcusKv6TiQ44jwIa1WR7rc3fvlJ9hJm9R7QTHCWpJ9GRyxNl4yUdJWliqJIoJDoDSKQ+sTPRTqfMV/Ejw+nrR+EUfT1waoLllpUdK8/MdoRldYmbZnlc92aiH/duJHUDTgDK6nz/TXSkXFbd1Q1YUMGsOWG6isYlIn7dIOkQSS8rusC/AfgTO9dHZTGUxds3fHfDgUIz+6SSaZ8Fjg7VIUOIfszvhnG/Ido5fKKoWu5He/CZbgAuk9RxD+ZdEdddXEF/+e+v/LbVOXQfCLwQqmPWEyWJ7URnIRXNW175basIWMOu21ZtlP/ee0v6T9z3fhNV/w4S2q6rmXaX36ZFe9pdqjsr8ISZtY37G15ufGXfxy7rM3Q3Bfaj6u260vjN7A1gDHAfsELSGEmtqom/1vaZpBA8ClxAVI3yhpnF/yCfAMYD3cysDdGXUf7CdEWWEX3pZWK3yoW62OeAPwP7m1lbomqQsnKtmrKXEv34y8pTWNaSBOIq74dE3/dLkpYDC4l29heE8YuJqrnKWw1sqWTcJiArLr4Moh9BvPKf8T6io59eZtYa+B0710dlMWBmW4iO0M8Ln+WxiqYL064H3iA6Xf8BUbWKhXHLzexSM+tMVIX4d0kHV1ZWJeV/DjwfYo+3y/oA9iRplFd+21oauhcDI8rtwDLNLH7bqGr7Kr9ttSCqMtyTbasi5Zd9P5APHBy+9xtI7PdVG8uIqteA2O+ntkmvsu9jl/UZxm0jOhCtdLuujpndZWaHA7lE1Ue/3JNyamJfTArDgEuB8reUtgLWmtkWSYOIdiaJeBq4SlLXcJHw2rhxTYnqa1cBpZJGEJ0SllkBtJfUpoqyT5N0Urgo9yuiU9IPEowt3gXAH4jqxMv+zgzltyc6gxgm6Xvholp7SQPD2clDwF8UXcTNCBcNmxHVoWYqukjfBPh/4fNWpRWwASiS1Bu4LG7cy0BHST+X1ExSK0lHxY1/lKgaZyTRaXdVngif+Ux2PSM8W1LZjmId0c5rezVlVeQPwMVA27hh04FTJWWHs4if70G55V0vKUtSv7C8sgvjY4BbJB0IIGk/SaNqUO4TwMWSBobv8k/Ax2a2KAkxV6QVUfXtJkU3S/wkRcuJ9zJwuKTTFd0BdTW7H7TU1G8ktVX0nMRV7Pw+ngR+qegifyvgFqKDkR1E2+opks4Mv60cSYdWtyBFN2UMCrFvIkoye7Kt1sg+lRTCBv8B0cW38eVGXw7cpOhuiRuIdsiJ+AfwOvAZ0UXD5+OWt5Fow3maaAf0g/jlhiPOJ4GFoRqgc1y5mNlc4Hyii4CriS7cnW5m2xKMDQBJg4nqme8NR8plf+OJLpada9F936cSJZ61RDu4sg33GmAmMDmMu42ojrOQaL09QHSEuYnqT8+vCethI9G6i939E9bX8PA5lwNfEFV5lY1/H9gBTEtg5zUe6AWsMLPP4oYfCXwsqShMc7WZfRnW0ywleJ99mOcxom2pzGNE28EiojOVmtzZVJl3iL6jN4E/hyoFgL+F+N8I2+xHRHXPCTGzN4nqup8jOqI+CPh+EuKtzK+IboPeSHTWkIx1U6VQE3AO8BeiqrGDgE+JDqwqc552fU6hSHF3vRHdoDE9lPMC0XUk2Lktv0t0Fr6RKAmVbSunA/9D9PuZRnTjQHXaEl37WU+0TS0jusEipRTOqp2rFyS9RVTv6099uxoJ1ZtLgbPM7N3qpi83b2OiC/E9Ung2tVfYp84UXP0m6UiiWyhTfpTpGgZJp0hqE6rIrgdKiW7zdJVIWVKQ9FB46CK/kvEKD2bMDw9lHJ6qWFz9J+kRoltqfx6qmZxLxHFE1TmrgVOInj2qqvpon5ey6iNJQ4Ai4FEzy61g/KlE92WfSlQX+jczS7hO1DnnXPKl7EzBzCYRXVSpzCiihGFm9hHQVlKnVMXjnHOueulsqKoLuz4IUhCGLSs/oaTRRI+R06JFiyN69+5dJwE651xDMXXq1NVmVu0tuelMChU9uFJhXZaZjSVqmoC8vDybMmVKKuNyzrkGR9JX1U+V3ruPCtj16cCu7Hw60DnnXBqkMymMBy4IdyENJmrLZreqI+ecc3UnZdVHkp4kaiUyR1Gb8r8nahoWMxtD1AbQqURPa24meoTfOedcGqUsKZjZudWMN6KXvTjnnNtL+BPNzjnnYjwpOOeci/Gk4JxzLsaTgnPOuRhPCs4552I8KTjnnIvxpOCccy7Gk4Jzzu3lirdtZ96KjawpSv2rINLZIJ5zzjlg+w5jWWExi9cWs3jtZhav28zXazezeO1mvl5bzOqQDG75Ti7nHXVgSmPxpOD2GWbGe/NX8+QnX5PVtDG5nVuT26UNfTq1pkWzveenULS1lFlLCslfuoFZSwopWFeckuXs16oZ/bq0JrdzG3K7tCG7RdOULMdF2976zSXRjj62w9+ZAJasK6Z0x85GojMaiU5tMjkgO4uTenegW3ZzumVnccSB7VIe697zS3C7yF9SyKI1m5JerhAtMxvTpnkT2jRvQuvMxrRu3oQmGQ23JrFk+w7+M2MZYyctZPayDeS0bAqIZ6cWACBBz5wW9O8S7Rz7dW5Dvy6taZ3ZJOWxFW4uIX9pIflxSWDh6p3f+/6tm3Fg+xZkqKKW5vecYcxcUsh/Zu5sg7Jzm0xywzrIDcmiQ+vMpC63IdtSsp2CddHOfudR/mYWr4t2/kVbS3eZvn2LpnTNzqJ/lzac1r8T3bKzOCA7i27tsujUNjNtv0lPCnuhSfNWcdHDn7AjNW9KrVCLphlRkgh/UcJoEksebZo3jg2PJZTwP7NJRt0FWgNFW0v51ydf8/D7i1iyvpheHVpy+1kDGDWwM80aZ7Biw5ZoZ7xkA/lLC/n4y7W8OH1n6+0Hts+KdpCdd+4k29XiaHp10VbylxQya+mGkAQKWbx251lAl7bNye3Smu8c1iVKTl1a06FVanfKhZtLmLWskFlLNjAzxPTfOSsoe0vvfq2aRcmyc2v6hYTRuU0mSnKSqg+27zBWbNiy286+rH/lxl3r+zObNKJbu2hHf1SPbLplZ9GtXXMOaJ9F13ZZtNyLzk7jpewdzanS0F+y89WaTYy85306tcnkru8PTPoR4nYziraUUlhcwoYtJRRuLqGwOK6/OPrbEP4Ki0vYtG17lWU2bdyIzm0yGdZnf0b078Rh3drSqFH6dhorN2zhnx8s4vGPvmLDllIG9cjmp9/sydBDOlQb1+qirbGd9qylhcxcUvGOu6zKpaIdt5mxcuNWZhYUhrOADcxaWsiywi2xabq3z4p2sklKOMlUtLWUOcuidTBzSZQwvli5MXaQkt2iKf1C1VtZ/AdkZzWIRFG4WxXPzp3/knXFbNu+IzZtI0GnNs3plt08doRftsPvlt2c/Vo226vWiaSpZpZX7XSeFPYem7aW8t2/f8CKjVsY/7PjOKB9VrpDAqLql40hkcQnjfjueSs28t781ZRsNzq2zuSU3I6MyO1IXvdsMuooQcxfWcQ/Ji3khU+XULpjB6fkdmT0kIMY2K1trcot3FzCrKXRDn7mkt2reDq0akZulzZ0b9+ChauLyF+yIXZhUIKD9msZu37Rr3Mb+nZuTZvmqa+aSqbibdv5fPkG8pduID8ku3krNlKyPdp/tMpsTG74bHVR7ZYsRVtLorr9kAQ2btm1iqddVpPoCL9sp5+dFUsCndo0p2nj+lPt6kmhnjEzLh83jddnLefRHx3Fcb1y0h1SjRUWl/DW5yt4deZy3p63im2lO8hp2Yxv9dufU/t34qge2TROcj2pmTF50TrGTlrAhDkrada4EWfndeXHx/Wke06LpC4r3sYtJcxZtjFUP0U7yUVrNtMzp0U4gt47L2In09bS7Xyxoih2RpG/dANzlm1gW+mO6mfeSzRr3Iiu7cKRfqjT7xq3829VjxJcdTwp1DP3TpzPHa/P5f+d1ocfH98z3eHUWtHWUiZ+vpLX8pfz1ucrKS7ZTrusJpzctyMj+nfkmINyanWUtX2H8d/ZyxnzzkKmL15Pu6wmXHB0dy44+kDat2yWxE/iasLMqE+7FIm9qoonlTwp1CNvfb6CSx6ZwqhDO/PXcwY2uI20eNt23pm3klfzl/PmnJUUbS2ldWZjhvXdn1NzO3Fcr5yEL1ZvKdnOs1MLeODdhSxas5kDsrO49PgenHVEN5o33TsveDu3N/CkUE8sWFXEGfe8z4E5WTz702P22jt5kmVLyXben7+aV2Yu57+zl7NhSyktmzXmpD4dGJHbkW8e0qHCnfu6Tdt49MOvePTDRazZtI1Du7Zh9JCDOCW3Y51ds3CuPks0KTTMys56YsOWEi59dApNGzfi/h/mNfiEAJDZJIOT+uzPSX32Z1tpfz5YsJrX8pfz+qzl/Hv6Upo3yeDE3h04JbcjJ/buwJqibTzw3kKenrKYLSU7OLF3B0YP6clRPbIb3BmVc3sDP1NIkx07jNGPTeHtuasY9+OjOKpn+3SHlFal23fwyZdreSV/Ga/lr2B10VaaNW5EyfYdZDQSZwzswqVDenLI/q3SHapz9ZKfKezl7nrzCybMWclNo/rt8wkBoHFGI445OIdjDs7hDyNzmfrVOl7LX05W0wzOH3wgHdv4k7XO1QVPCmnwWv5y/u/NL/heXld+ODi1jVvVRxmNxKAe2QzqkZ3uUJzb59SfJy8aiHkrNvKrp6czsFtbbhqV6/Xizrm9iieFOlS4uYTRj04hq1ljxpx/xD5xYdk5V794Uqgj23cYV/3rU5asL2bM+Yd7Hblzbq/k1xTqyB2vz+Wdeav43+/254gDva7cObd38jOFOvDSZ0sZ884CzjvqAM4ddEC6w3HOuUrtM0lheeEW3vtiNXX9XMbspRv49bOfcWT3dvz+9H51umznnKupfab66PGPvuKeifPp26k1P/lmT07r3ynpLXaWt3bTNkY/NoW2zZvy9/OOqFfN7Drn9k37zF7qihMP5tbv9mdL6Xau/td0vnnH2zz03pdsKveKvGQp3b6DK56YxsqNW7n/h0ewXytvudM5t/fb55q52LHDePPzlYydtIDJi9bRpnkTzh98ABce0z2prz7848uzefC9L7nz7EM584iuSSvXOef2hDdzUYlGjcTwvvszvO/+TPt6HWPfWcjf317APyZ9yXcPj9rXOWi/lrVaxvPTCnjwvS+5+NjunhCcc/XKPnemUJEvV2/igXcX8szUAraV7mBYn/356Td7kte95reOzihYz1ljPuSIA9rx6CWDaJLi6xbOOZcIf5/CHlhdtJVHP1jEox99xfrNJRx+QFtGDzmI4X33T6jN/lUbtzLynvdoJDH+imP9DWDOub2GJ4Va2LytlGemFPDAewtZvLaYHjkt+PHxPTjz8K6VNk2xrXQH5z3wETOXFPLsT48ht0ublMbonHM1kWhSSGndhqRTJM2VNF/StRWMP0DSREmfSpoh6dRUxpOorKaNufCY7kz81VDu+cFhtMpszHUv5HPcbW9x95tfsG7Ttt3m+ePLs5m8aB23nTnAE4Jzrt5K2ZmCpAxgHjAcKAAmA+ea2ey4acYCn5rZfZL6Aq+YWfeqyk3HS3bMjI8WrmXspAVMnLuK5k0yOOfIblxyXA+6ZWfxr0++5trnZ/KTIT357al96jQ255xLxN5w99EgYL6ZLQwB/QsYBcyOm8aA1qG7DbA0hfHsMUkcfVB7jj6oPXOXb2TspIWM+zh6X/CwPvvz9txVHN8rh9+c0jvdoTrnXK2ksvqoC7A4rr8gDIt3I3C+pALgFeDKigqSNFrSFElTVq1alYpYE/aNjq2483uH8u5vTuTS43vy4YI1dG6byd3nHuYvkHfO1XupPFOoaA9Zvq7qXOCfZnanpKOBxyTlmtmOXWYyGwuMhaj6KCXR1lDHNpn89tQ+XD2sF2bQotk+98iHc64BSuWerADoFtffld2rhy4BTgEwsw8lZQI5wMoUxpVUWU09GTjnGo5UVh9NBnpJ6iGpKfB9YHy5ab4GTgKQ1AfIBNJbP+Scc/uwlCUFMysFrgBeB+YAT5vZLEk3SRoZJvsVcKmkz4AngYusvj044ZxzDUhK6z7M7BWiC8jxw26I654NHJvKGJxzziXOG+ZxzjkX40nBOedcjCcF55xzMZ4UnHPOxXhScM45F+NJwTnnXIwnBeecczGeFJxzzsV4UnDOORfjScE551yMJwXnnHMx1SaF8FpN55xz+4BEzhTmS7ojvEPZOedcA5ZIUhgAzAMekPRReDVm6+pmcs45V/9UmxTMbKOZ/cPMjgF+A/weWCbpEUkHpzxC55xzdSahawqSRkp6AfgbcCfQE3iJcu9KcM45V78l8pKdL4CJwB1m9kHc8GclDUlNWM4559IhkaQwwMyKKhphZlclOR7nnHNplMiF5nsltS3rkdRO0kMpjMk551yaJHT3kZmtL+sxs3XAYakLyTnnXLokkhQaSWpX1iMpm8SqnZxzztUziezc7wQ+kPRs6D8buCV1ITnnnEuXapOCmT0qaSpwAiDgu2Y2O+WROeecq3MJVQOZ2SxJq4BMAEkHmNnXKY3MOedcnUvk4bWRkr4AvgTeARYBr6Y4Luecc2mQyIXmPwKDgXlm1gM4CXg/pVE555xLi0SSQomZrSG6C6mRmU0EBqY4Luecc2mQyDWF9ZJaApOAcZJWAqWpDcs551w6JHKmMArYDPwCeA1YAJyeyqCcc86lR5VnCuGta/82s2HADuCROonKOedcWlR5pmBm24HNktrUUTzOOefSKJFrCluAmZL+C2wqG+gtpDrnXMOTSFL4T/hzzjnXwCXSzIVfR3DOuX1EIk80fylpYfm/RAqXdIqkuZLmS7q2kmm+J2m2pFmSnqjpB3DOOZc8iVQf5cV1ZxK1kppd3UzhzqV7geFAATBZ0vj4xvQk9QJ+CxxrZuskdahJ8M4555Kr2jMFM1sT97fEzO4CTkyg7EHAfDNbaGbbgH8RPfMQ71Lg3vDiHsxsZQ3jd845l0TVnilIOjyutxHRmUOrBMruAiyO6y8Ajio3zSFhGe8DGcCNZvZaBTGMBkYDHHDAAQks2jnn3J5I9CU7ZUqJWkv9XgLzqYJhVsHyewFDga7Au5Jy41//CWBmY4GxAHl5eeXLcM45lySJ3H10wh6WXQB0i+vvCiytYJqPzKwE+FLSXKIkMXkPl+mcc64WErn76E+S2sb1t5N0cwJlTwZ6SeohqSnwfWB8uWleJHqjG5JyiKqTErqzyTnnXPIl0iDeiPjqnHBR+NTqZjKzUuAK4HVgDvB0eIPbTZJGhsleB9ZImg1MBH4dmul2zjmXBolcU8iQ1MzMtgJIag40S6RwM3sFeKXcsBviug34ZfhzzjmXZokkhceBNyU9THSh+Ed4a6nOOdcgJXKh+XZJM4BhRHcU/dHMXk95ZM455+pcIs8p9ADeLnt+QFJzSd3NbFGqg3POOVe3ErnQ/AzRC3bKbA/DnHPONTCJJIXGoZkKAEJ309SF5JxzLl0SSQqr4m4hRdIoYHXqQnLOOZcuidx99FNgnKR7iC40LwYuSGlUzjnn0iKRu48WAIMltQRkZhsl7Z/60JxzztW1RKqPymQAZ0uaAExLUTzOOefSqMozhfD08kjgB8DhRE1mnwFMSn1ozjnn6lqlZwqSxgHzgJOBe4DuwDoze9vMdlQ2n3POufqrquqjXGAdUWN2n5vZdnZ/H4JzzrkGpNKkYGaHEr1MpzUwQdK7QCtJHesqOOecc3WrygvNZva5md1gZt8AfgE8Cnwi6YM6ic4551ydSuQ5BQDMbAowRdI1wJDUheSccy5dEk4KZcI7EN5JQSzOOefSrCbPKTjnnGvgPCk455yLSeR9Cs2AM4meU4hNb2Y3pS4s55xz6ZDINYV/A4XAVGBrasNxzjmXTokkha5mdkrKI3HOOZd2iVxT+EBS/5RH4pxzLu0SOVM4DrhI0pdE1UciujN1QEojc845V+cSSQojUh6Fc865vUIiL9n5StKhwPFh0Ltm9llqw0qBV6+F5TPTHYVzzu25jv1hxK0pXUS11xQkXQ2MAzqEv8clXZnSqJxzzqVFItVHlwBHmdkmAEm3AR8Cd6cysKRLcXZ1zrmGIJG7jwRsj+vfHoY555xrYBI5U3gY+FjSC6H/DODB1IXknHMuXRK50PwXSW8T3Zoq4GIz+zTVgTnnnKt7lSYFSa3NbIOkbGBR+Csbl21ma1MfnnPOubpU1ZnCE8C3ido8in83s0J/zxTG5ZxzLg0qTQpm9u3wv0fdheOccy6dEnlO4c1EhjnnnKv/qrqmkAlkATmS2rHzNtTWQOc6iM0551wdq+pM4SdE1xN6h/9lf/8G7k2kcEmnSJorab6ka6uY7ixJJikv8dCdc84lW1XXFP4G/E3SlWZW46eXJWUQJY/hQAEwWdJ4M5tdbrpWwFXAxzVdhnPOueRK5DmFuyXlAn2BzLjhj1Yz6yBgvpktBJD0L2AUMLvcdH8EbgeuqUHczjnnUiCRC82/J2rn6G7gBKId+MgEyu4CLI7rLwjD4ss+DOhmZi9XE8NoSVMkTVm1alUCi3bOObcnEmn76CzgJGC5mV0MHAo0S2C+itpHij3vIKkR8FfgV9UVZGZjzSzPzPL222+/BBbtnHNuTySSFIrNbAdQKqk1sJLEHlwrALrF9XcFlsb1twJygbclLQIGA+P9YrNzzqVPIg3iTZHUFvgH0d1HRcAnCcw3GeglqQewBPg+8IOykWZWCOSU9Yf2la4xsykJR++ccy6pErnQfHnoHCPpNaC1mc1IYL5SSVcArwMZwENmNkvSTcAUMxtfm8Cdc84lX1UPrx1e1Tgzm1Zd4Wb2CvBKuWE3VDLt0OrKc845l1pVnSncGf5nAnnAZ0QXjwcQPVNwXGpDc845V9ddMPYOAAAUa0lEQVQqvdBsZieY2QnAV8Dh4e6fI4DDgPl1FaBzzrm6k8jdR73NbGZZj5nlAwNTF5Jzzrl0SeTuozmSHgAeJ3rO4HxgTkqjcs45lxaJJIWLgcuAq0P/JOC+lEXknHMubRK5JXUL0ZPHf019OM4559KpqltSnzaz70maya6v4wTAzAakNDLnnHN1rqozhbLqom/XRSDOOefSr6r3KSwL/7+qu3Ccc86lU1XVRxupoNqI6AE2M7PWKYvKOedcWlR1ptCqLgNxzjmXfonckgqApA7s+ua1r1MSkXPOubRJ5M1rIyV9AXwJvAMsAl5NcVzOOefSIJFmLv5I9AKceWbWg+gtbO+nNCrnnHNpkUhSKDGzNUAjSY3MbCLe9pFzzjVIiVxTWC+pJVHzFuMkrQRKUxuWc865dEjkTGEUUAz8AngNWACcnsqgnHPOpUdVzyncAzxhZh/EDX4k9SE555xLl6rOFL4A7pS0SNJtkvw6gnPONXBVvXntb2Z2NPBNYC3wsKQ5km6QdEidReicc67OVHtNwcy+MrPbzOww4AfAd/CX7DjnXIOUyMNrTSSdLmkc0UNr84AzUx6Zc865OlfVhebhwLnAacAnwL+A0Wa2qY5ic845V8eqek7hd8ATwDVmtraO4nHOOZdGVbWSekJdBuKccy79Enl4zTnn3D7Ck4JzzrkYTwrOOediPCk455yL8aTgnHMuxpOCc865GE8KzjnnYjwpOOeci/Gk4JxzLialSUHSKZLmSpov6doKxv9S0mxJMyS9KenAVMbjnHOuailLCpIygHuBEUBf4FxJfctN9imQZ2YDgGeB21MVj3POueql8kxhEDDfzBaa2TaiVlZHxU9gZhPNbHPo/QjomsJ4nHPOVSOVSaELsDiuvyAMq8wlRO9r2I2k0ZKmSJqyatWqJIbonHMuXiqTgioYZhVOKJ0P5AF3VDTezMaaWZ6Z5e23335JDNE551y8qt6nUFsFQLe4/q7A0vITSRoGXAd808y2pjAe55xz1UjlmcJkoJekHpKaAt8HxsdPIOkw4H5gpJmtTGEszjnnEpCypGBmpcAVwOvAHOBpM5sl6SZJI8NkdwAtgWckTZc0vpLinHPO1YFUVh9hZq8Ar5QbdkNc97BULt8551zNpDQp1JWSkhIKCgrYsmVLukNpcDIzM+natStNmjRJdyjOuTrQIJJCQUEBrVq1onv37kgV3fTk9oSZsWbNGgoKCujRo0e6w3HO1YEG0fbRli1baN++vSeEJJNE+/bt/QzMuX1Ig0gKgCeEFPH16ty+pcEkBeecc7XnSSFJMjIyGDhwILm5uZx99tls3ry5+pni3HXXXTWeB+CGG25gwoQJNZ6vIkOHDmXKlClJKcs5Vz95UkiS5s2bM336dPLz82natCljxozZZbyZsWPHjkrnryopbN++vdL5brrpJoYN8zt7nXPJ0SDuPor3h5dmMXvphqSW2bdza35/er+Epz/++OOZMWMGixYtYsSIEZxwwgl8+OGHvPjii8ydO5ff//73bN26lYMOOoiHH36Yhx56iKVLl3LCCSeQk5PDxIkTadmyJb/85S95/fXXufPOO3nrrbd46aWXKC4u5phjjuH+++9HEhdddBHf/va3Oeuss+jevTsXXnghL730EiUlJTzzzDP07t2bTZs2ceWVVzJz5kxKS0u58cYbGTVqFMXFxVx88cXMnj2bPn36UFxcnNT15pyrf/xMIclKS0t59dVX6d+/PwBz587lggsu4NNPP6VFixbcfPPNTJgwgWnTppGXl8df/vIXrrrqKjp37szEiROZOHEiAJs2bSI3N5ePP/6Y4447jiuuuILJkyeTn59PcXExL7/8coXLz8nJYdq0aVx22WX8+c9/BuCWW27hxBNPZPLkyUycOJFf//rXbNq0ifvuu4+srCxmzJjBddddx9SpU+tmJTnn9loN7kyhJkf0yVRcXMzAgQOB6EzhkksuYenSpRx44IEMHjwYgI8++ojZs2dz7LHHArBt2zaOPvroCsvLyMjgzDPPjPVPnDiR22+/nc2bN7N27Vr69evH6aefvtt83/3udwE44ogjeP755wF44403GD9+fCxJbNmyha+//ppJkyZx1VVXATBgwAAGDBiQjFXhnKvHGlxSSJeyawrltWjRItZtZgwfPpwnn3yy2vIyMzPJyMgAop345ZdfzpQpU+jWrRs33nhjpc8ONGvWDIiSSmlpaWy5zz33HN/4xjd2m95vOXXOxfPqozo0ePBg3n//febPnw/A5s2bmTdvHgCtWrVi48aNFc5XlgBycnIoKiri2WefrdFyv/Wtb3H33XdjFr3O4tNPPwVgyJAhjBs3DoD8/HxmzJhR8w/lnGtQPCnUof32249//vOfnHvuuQwYMIDBgwfz+eefAzB69OjYReny2rZty6WXXkr//v0544wzOPLII2u03Ouvv56SkhIGDBhAbm4u119/PQCXXXYZRUVFDBgwgNtvv51BgwbV/kM65+o1lR091hd5eXlW/l76OXPm0KdPnzRF1PD5+nWu/pM01czyqpvOzxScc87FeFJwzjkX40nBOedcjCcF55xzMZ4UnHPOxXhScM45F+NJIYluueUW+vXrx4ABAxg4cCAff/xxrcpbv349f//736udzpu8ds4liyeFJPnwww95+eWXmTZtGjNmzGDChAl069at2vnKmqKoSKJJwTnnkqXhtX306rWwfGZyy+zYH0bcWuUky5YtIycnJ9b2UE5ODgCTJ0/m6quvZtOmTTRr1ow333yT5557jv/85z9s2bKFTZs2MX78eEaNGsW6desoKSnh5ptvZtSoUVx77bUsWLCAgQMHMnz4cO644w5uv/12HnvsMRo1asSIESO49dYormeeeYbLL7+c9evX8+CDD3L88ccndx045/YJDS8ppMnJJ5/MTTfdxCGHHMKwYcM455xzOProoznnnHN46qmnOPLII9mwYQPNmzcHojOLGTNmkJ2dTWlpKS+88AKtW7dm9erVDB48mJEjR3LrrbeSn58fa2jv1Vdf5cUXX+Tjjz8mKyuLtWvXxpZfWlrKJ598wiuvvMIf/vCHpL2NzTm3b2l4SaGaI/pUadmyJVOnTuXdd99l4sSJnHPOOVx33XV06tQp1lZR69atY9MPHz6c7OxsIGrF9He/+x2TJk2iUaNGLFmyhBUrVuy2jAkTJnDxxReTlZUFEJsfdm0ye9GiRan6mM65Bq7hJYU0ysjIYOjQoQwdOpT+/ftz7733Vto0dXyT2uPGjWPVqlVMnTqVJk2a0L179wqbxjazSsurqMls55yrKb/QnCRz587liy++iPVPnz6dPn36sHTpUiZPngzAxo0bK9xhFxYW0qFDB5o0acLEiRP56quvgN2b0z755JN56KGHYu9yjq8+cs65ZPAzhSQpKiriyiuvZP369TRu3JiDDz6YsWPHcvHFF3PllVdSXFxM8+bNK6zrP++88zj99NPJy8tj4MCB9O7dG4D27dtz7LHHkpuby4gRI7jjjjuYPn06eXl5NG3alFNPPZU//elPdf1RnXMNmDed7arl69e5+s+bznbOOVdjnhScc87FNJikUN+qweoLX6/O7VsaRFLIzMxkzZo1vgNLMjNjzZo1ZGZmpjsU51wdaRB3H3Xt2pWCggJWrVqV7lAanMzMTLp27ZruMJxzdaRBJIUmTZrQo0ePdIfhnHP1XkqrjySdImmupPmSrq1gfDNJT4XxH0vqnsp4nHPOVS1lSUFSBnAvMALoC5wrqW+5yS4B1pnZwcBfgdtSFY9zzrnqpfJMYRAw38wWmtk24F/AqHLTjAIeCd3PAiepssZ9nHPOpVwqryl0ARbH9RcAR1U2jZmVSioE2gOr4yeSNBoYHXqLJM3dw5hyypedJF5u/Yo1VeXWp1jrW7n1Kda9tdwDE5kolUmhoiP+8veMJjINZjYWGFvrgKQpiTzm7eXuHWXWt3LrU6z1rdz6FGt9LDdeKquPCoD491F2BZZWNo2kxkAbwJv+dM65NEllUpgM9JLUQ1JT4PvA+HLTjAcuDN1nAW+ZP4HmnHNpk7Lqo3CN4ArgdSADeMjMZkm6CZhiZuOBB4HHJM0nOkP4fqriCWpdBeXl1mmZ9a3c+hRrfSu3PsVaH8uNqXdNZzvnnEudBtH2kXPOueTwpOCccy5mn0gKkh6StFJSfpLL7SZpoqQ5kmZJujoJZWZK+kTSZ6HMPyQj1rjyMyR9KunlJJa5SNJMSdMlTal+joTLbSvpWUmfh3V8dC3L+0aIsexvg6SfJynWX4TvK1/Sk5KS0rSspKtDmbNqE2tFvwFJ2ZL+K+mL8L9dEso8O8S6Q9Ie3TpZSbl3hO1ghqQXJLVNUrl/DGVOl/SGpM7JKDdu3DWSTFJOEmK9UdKSuO331JrGmhAza/B/wBDgcCA/yeV2Ag4P3a2AeUDfWpYpoGXobgJ8DAxOYsy/BJ4AXk5imYuAnBR8b48APw7dTYG2SSw7A1gOHJiEsroAXwLNQ//TwEVJKDcXyAeyiG4KmQD02sOydvsNALcD14bua4HbklBmH+AbwNtAXhJjPRloHLpvq2msVZTbOq77KmBMMsoNw7sR3WjzVU1/H5XEeiNwTW23q+r+9okzBTObRAqefzCzZWY2LXRvBOYQ7SBqU6aZWVHobRL+knI3gKSuwGnAA8koL5UktSb6YTwIYGbbzGx9EhdxErDAzL5KUnmNgebheZssdn8mZ0/0AT4ys81mVgq8A3xnTwqq5DcQ38zMI8AZtS3TzOaY2Z62OFBVuW+EdQDwEdFzT8kod0Ncbwv24LdWxf7lr8Bvklxmyu0TSaEuhBZeDyM6sq9tWRmSpgMrgf+aWa3LDO4i2kh3JKm8Mga8IWlqaJIkGXoCq4CHQ3XXA5JaJKlsiG5/fjIZBZnZEuDPwNfAMqDQzN5IQtH5wBBJ7SVlAaey6wOhtbW/mS2D6AAH6JDEslPpR8CrySpM0i2SFgPnATckqcyRwBIz+ywZ5cW5IlR3PVTT6r5EeVJIAkktgeeAn5c78tgjZrbdzAYSHQ0NkpSbhBi/Daw0s6m1LasCx5rZ4UQt4v5M0pAklNmY6PT5PjM7DNhEVMVRa+FhypHAM0kqrx3RUXcPoDPQQtL5tS3XzOYQVZX8F3gN+AworXKmBk7SdUTrYFyyyjSz68ysWyjzitqWFxL4dSQpwcS5DzgIGEh08HFnkssHPCnUmqQmRAlhnJk9n8yyQ3XJ28ApSSjuWGCkpEVELdaeKOnxJJSLmS0N/1cCLxC1kFtbBUBB3FnSs0RJIhlGANPMbEWSyhsGfGlmq8ysBHgeOCYZBZvZg2Z2uJkNIapO+CIZ5QYrJHUCCP9XJrHspJN0IfBt4DwLlexJ9gRwZhLKOYjoAOGz8HvrCkyT1LE2hZrZinDAuAP4B8n5ne3Gk0ItSBJRnfccM/tLksrcr+zOCknNiXY4n9e2XDP7rZl1NbPuRFUnb5lZrY9mJbWQ1Kqsm+iCYK3v8jKz5cBiSd8Ig04CZte23OBcklR1FHwNDJaUFbaJk4iuL9WapA7h/wHAd0lu3PHNzFwI/DuJZSeVpFOA/wFGmtnmJJbbK653JMn5rc00sw5m1j383gqIbkhZXptyyxJ48B2S8DurUKqvZO8Nf0Q/pGVACdEXdEmSyj2OqD59BjA9/J1ayzIHAJ+GMvOBG1KwPoaSpLuPiOr+Pwt/s4DrkhjnQGBKWBcvAu2SUGYWsAZok+R1+geiHUo+8BjQLEnlvkuUDD8DTqpFObv9BoiaqX+T6OzjTSA7CWV+J3RvBVYArycp1vlEzeyX/c725C6hisp9LnxnM4CXgC7JKLfc+EXU/O6jimJ9DJgZYh0PdErmNlz2581cOOeci/HqI+ecczGeFJxzzsV4UnDOORfjScE551yMJwXnnHMxnhRcvRGaeyhrIXJ5uRYjmyZYxsNxzz5UNs3PJJ2XpJjfkzQ3Ls6nklFuXPkFe9JiqHOV8VtSXb0k6UagyMz+XG64iLbrZLfvtEckvQdcYWbTU1R+AZBryW0s0O3D/EzB1XuSDg7vHBgDTAM6SRoraUpo2/+GuGnfkzRQUmNJ6yXdqujdFR/GPT18s8K7C8L0typ6x8VcSceE4S0kPRfmfTIsa2ANYn5c0n2S3pU0T9KIMLy5pEcUvZ9iWlk7UiHev4bPOUPS5XHF/Tw0GjhD0iFh+hNDbNNDOclsTNA1YJ4UXEPRF3jQzA6zqNXSa80sDzgUGC6pbwXztAHeMbNDgQ+JWt+siMxsEPBrdjZydiWwPMx7K1ELuZV5Kq766Na44d2AbwKnA2MlNSNq03+bmfUHfgg8FqrGLiNqbO9QMxtA1H5VmRUWNRr4ANH7MgixjraoYcUhwJYq4nMuxpOCaygWmNnkuP5zJU0jOnPoQ5Q0yis2s7ImmKcC3Ssp+/kKpjmOsGO2qHnkWVXEdo6ZDQx/8S29Pm1mOyx6/8BioFco97FQ7iyi9zIcTNQG1hgz2x7Gxbe1X1F87wN3SbqS6EUy26uIz7kYTwquodhU1hEaObsaODEcVb8GVPR6zG1x3duJmuuuyNYKplGtoo2Uv6BnVZSrCqYvs1t8ZnYz8BOgJTC5XMNvzlXKk4JriFoDG4ENoWXJb6VgGe8B3wOQ1J+Kz0Sqc7YihxBVJX0BTCJ62QuS+hC98nU+8AZwmaSMMC67qoIlHWRmM8zsf4kaWKzyjivnylR2ZORcfTaNqGXRfGAhUVVKst0NPCppRlhePlBYybRPSSoO3SvMrCxJzSdKAh2I6v+3SbobuF/STKIWMi8Iw+8nql6aIamU6IUrY6qI7xpJxxO9ZW8GUVJxrlp+S6pze0DRu5gbm9mWUDXzBtDLdr5HuLr5HweeNbMXUxmnczXlZwrO7ZmWwJshOQj4SaIJwbm9mZ8pOOeci/ELzc4552I8KTjnnIvxpOCccy7Gk4JzzrkYTwrOOedi/j/TDglw4ZG4/gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize the non-pretrained version of the model used for this run\n",
    "scratch_model,_ = initialize_model(model_name, num_classes, feature_extract=False, use_pretrained=False)\n",
    "scratch_model = scratch_model.to(device)\n",
    "scratch_optimizer = optim.SGD(scratch_model.parameters(), lr=0.001, momentum=0.9)\n",
    "scratch_criterion = nn.CrossEntropyLoss()\n",
    "_,scratch_hist = train_model(scratch_model, dataloaders_dict, scratch_criterion, scratch_optimizer, num_epochs=num_epochs, is_inception=(model_name==\"inception\"))\n",
    "\n",
    "# Plot the training curves of validation accuracy vs. number\n",
    "#  of training epochs for the transfer learning method and\n",
    "#  the model trained from scratch\n",
    "ohist = []\n",
    "shist = []\n",
    "\n",
    "ohist = [h.cpu().numpy() for h in hist]\n",
    "shist = [h.cpu().numpy() for h in scratch_hist]\n",
    "\n",
    "plt.title(\"Validation Accuracy vs. Number of Training Epochs\")\n",
    "plt.xlabel(\"Training Epochs\")\n",
    "plt.ylabel(\"Validation Accuracy\")\n",
    "plt.plot(range(1,num_epochs+1),ohist,label=\"Pretrained\")\n",
    "plt.plot(range(1,num_epochs+1),shist,label=\"Scratch\")\n",
    "plt.ylim((0,1.))\n",
    "plt.xticks(np.arange(1, num_epochs+1, 1.0))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}