train.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IlyXbx03KuyH",
        "outputId": "2050f51d-e353-479e-9f82-e8f4b792336e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount = True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append('/content/drive/MyDrive')"
      ],
      "metadata": {
        "id": "VxXWIADAK1vS"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import string\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from numpy import array\n",
        "from PIL import Image\n",
        "import pickle\n",
        "from keras.utils import to_categorical\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "import tensorflow as tf\n",
        "import os\n",
        "from sklearn.model_selection import train_test_split\n",
        "from torch.optim.lr_scheduler import LambdaLR\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "from model import TransformerModel\n",
        "from torch.utils.data import DataLoader\n",
        "import torch.optim as optim\n",
        "from torch import nn\n",
        "import torch\n",
        "from dataset import DataGenerator\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "import nltk\n",
        "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n",
        "\n",
        "from torch.optim import Optimizer\n",
        "from torch.optim.lr_scheduler import _LRScheduler\n",
        "\n",
        "\n",
        "from utils import *\n",
        "\n",
        "\n",
        "tf.get_logger().setLevel('ERROR')\n",
        "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\"\n"
      ],
      "metadata": {
        "id": "Eu7ocGGiLCC2"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "captions_path = os.path.join('/content/drive/MyDrive/data/captions.txt')"
      ],
      "metadata": {
        "id": "4uxB7NmtLIBg"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "top_k = 5000\n",
        "caption_dict, word_to_index, index_to_word = preprocess_captions(captions_path)\n"
      ],
      "metadata": {
        "id": "qSYIk3SxMedM"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "keys = list(caption_dict.keys())\n",
        "train_keys, test_keys = train_test_split(keys, test_size = 0.2, random_state = 42)\n",
        "train_keys, val_keys = train_test_split(train_keys, test_size = 0.25, random_state = 42)"
      ],
      "metadata": {
        "id": "8hAx8OxtMgUZ"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "features_path = os.path.join('/content/drive/MyDrive/encodings_swin.pkl')\n",
        "with open(features_path, 'rb') as f:\n",
        "    features_dict = pickle.load(f)"
      ],
      "metadata": {
        "id": "zyKJtLy4M6Ub"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_captions = {k: caption_dict[k] for k in train_keys}\n",
        "val_captions = {k: caption_dict[k] for k in val_keys}\n",
        "test_captions = {k: caption_dict[k] for k in test_keys}\n",
        "\n",
        "train_features = {k: features_dict[k] for k in train_keys}\n",
        "val_features = {k: features_dict[k] for k in val_keys}\n",
        "test_features = {k: features_dict[k] for k in test_keys}"
      ],
      "metadata": {
        "id": "pOnJM7CbM71D"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = Tokenizer(num_words = top_k,\n",
        "                      oov_token=\"<unk>\",\n",
        "                      filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ')"
      ],
      "metadata": {
        "id": "T6aoB5IRNGi1"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tc = get_captions(train_captions)\n",
        "tokenizer.fit_on_texts(tc)\n",
        "tokenizer.word_index['<pad>'] = 0\n",
        "tokenizer.index_word[0] = '<pad>'\n",
        "max_seq_length = get_max_length(caption_dict)"
      ],
      "metadata": {
        "id": "_SC8-ouVNHvy"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data_generator = DataGenerator(train_captions, train_features, tokenizer, max_seq_length)\n",
        "train_data = DataLoader(train_data_generator, batch_size=32)\n",
        "val_data_generator = DataGenerator(val_captions, val_features, tokenizer, max_seq_length)\n",
        "val_data = DataLoader(val_data_generator, batch_size=32)\n",
        "test_data_generator = DataGenerator(test_captions, test_features, tokenizer, max_seq_length)\n",
        "test_data = DataLoader(test_data_generator, batch_size=32)"
      ],
      "metadata": {
        "id": "cRAJPuGBRjrL"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = TransformerModel(model_dim = 512, input_dim = 768, num_heads = 8, num_layers = 4, vocab_size = top_k + 1, sequence_length = top_k + 1, height = 7, width = 7)"
      ],
      "metadata": {
        "id": "Rx_jUHZTNLhV"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.load_state_dict(torch.load(os.path.join('/content/drive/MyDrive/model.pth')))"
      ],
      "metadata": {
        "id": "9va7-EM3-QuO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a7afda3e-252f-4eb9-949a-daeb51349cc5"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class LRScheduler(_LRScheduler):\n",
        "    def __init__(self,\n",
        "                 optimizer: Optimizer,\n",
        "                 dim_embed: int,\n",
        "                 warmup_steps: int,\n",
        "                 last_epoch: int=-1,\n",
        "                 verbose: bool=False) -> None:\n",
        "\n",
        "        self.dim_embed = dim_embed\n",
        "        self.warmup_steps = warmup_steps\n",
        "        self.num_param_groups = len(optimizer.param_groups)\n",
        "\n",
        "        super().__init__(optimizer, last_epoch, verbose)\n",
        "\n",
        "    def get_lr(self) -> float:\n",
        "        lr = calc_lr(self._step_count, self.dim_embed, self.warmup_steps)\n",
        "        return [lr] * self.num_param_groups\n",
        "\n",
        "\n",
        "def calc_lr(step, dim_embed, warmup_steps):\n",
        "    return dim_embed**(-0.5) * min(step**(-0.5), step * warmup_steps**(-1.5))"
      ],
      "metadata": {
        "id": "ZntjFBPJMl6q"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_fn = nn.CrossEntropyLoss(reduction = 'none')\n",
        "optimizer = optim.Adam(model.parameters(), lr = 0.001, betas = (0.9, 0.98), eps = 1e-9)\n",
        "lr_mul = 1.0 / 512 ** 0.5\n",
        "scheduler = LRScheduler(optimizer, dim_embed = 512, warmup_steps = 4000)\n"
      ],
      "metadata": {
        "id": "Fssney7fN77j"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "num_epochs = 100\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model = model.to(device)"
      ],
      "metadata": {
        "id": "FyCI4I4POEXr"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def loss_function(real, pred):\n",
        "    mask = torch.logical_not(torch.eq(real, 0))\n",
        "    loss_ = loss_fn(pred.reshape(-1, pred.size(-1)), real.reshape(-1))\n",
        "\n",
        "    mask = mask.float()\n",
        "    loss_ *= mask.view_as(loss_)\n",
        "\n",
        "    return torch.sum(loss_) / torch.sum(mask)"
      ],
      "metadata": {
        "id": "GGxWXLLeOQT_"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_step(img_tensor, tar):\n",
        "    target_in = tar[:, :-1]\n",
        "    target_real = tar[:, 1:]\n",
        "\n",
        "    predictions = model(img_tensor, target_in)\n",
        "    loss = loss_function(target_real, predictions)\n",
        "\n",
        "    _, predicted_ids = torch.max(predictions, dim=-1)\n",
        "    accuracy = torch.sum(predicted_ids == target_real).item() / target_real.numel()\n",
        "\n",
        "    return loss, accuracy"
      ],
      "metadata": {
        "id": "maBsinyXP8br"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(num_epochs):\n",
        "    start = time.time()\n",
        "\n",
        "    model.train()\n",
        "    epoch_loss = 0.0\n",
        "    epoch_accuracy = 0.0\n",
        "\n",
        "    with tqdm(total=len(train_data), desc=f'Epoch {epoch + 1} Training') as pbar:\n",
        "        for batch, (img_tensor, tar) in enumerate(train_data):\n",
        "            img_tensor, tar = img_tensor.to(device), tar.to(device)\n",
        "            img_tensor = img_tensor.float()\n",
        "            tar = tar.long()\n",
        "            optimizer.zero_grad()\n",
        "            loss, batch_accuracy = evaluate_step(img_tensor, tar)\n",
        "            batch_loss = loss.item()\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            epoch_loss += batch_loss\n",
        "            epoch_accuracy += batch_accuracy\n",
        "\n",
        "            pbar.set_postfix({'Loss': batch_loss, 'Accuracy': batch_accuracy})\n",
        "            pbar.update()\n",
        "\n",
        "    mean_epoch_loss = epoch_loss / len(train_data)\n",
        "    mean_epoch_accuracy = epoch_accuracy / len(train_data)\n",
        "\n",
        "    print('Epoch {} Training Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, mean_epoch_loss, mean_epoch_accuracy))\n",
        "\n",
        "    if (epoch + 1) % 5 == 0:\n",
        "        torch.save(model.state_dict(), 'model.pth')\n",
        "\n",
        "    model.eval()\n",
        "    val_loss = 0.0\n",
        "    val_accuracy = 0.0\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for batch, (val_img_tensor, val_tar) in enumerate(val_data):\n",
        "            val_img_tensor, val_tar = val_img_tensor.to(device), val_tar.to(device)\n",
        "            val_img_tensor = val_img_tensor.float()\n",
        "            val_tar = val_tar.long()\n",
        "            val_batch_loss, val_batch_accuracy = evaluate_step(val_img_tensor, val_tar)\n",
        "\n",
        "            val_loss += val_batch_loss\n",
        "            val_accuracy += val_batch_accuracy\n",
        "\n",
        "            pbar.set_postfix({'Loss': val_batch_loss, 'Accuracy': val_batch_accuracy})\n",
        "            pbar.update()\n",
        "\n",
        "    mean_val_loss = val_loss / len(val_data)\n",
        "    mean_val_accuracy = val_accuracy / len(val_data)\n",
        "\n",
        "    print('Epoch {} Validation Loss {:.4f} Validation Accuracy {:.4f}'.format(epoch + 1, mean_val_loss, mean_val_accuracy))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nE3W-SV2QC3P",
        "outputId": "6d9393f9-17fa-487f-c8aa-b54bbe3df7e7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1 Training: 100%|██████████| 152/152 [00:14<00:00, 10.44it/s, Loss=6.29, Accuracy=0.0344]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1 Training Loss 7.2670 Accuracy 0.0265\n",
            "Epoch 1 Validation Loss 6.1868 Validation Accuracy 0.0379\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 2 Training: 100%|██████████| 152/152 [00:14<00:00, 10.34it/s, Loss=5.1, Accuracy=0.0675]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2 Training Loss 5.6075 Accuracy 0.0539\n",
            "Epoch 2 Validation Loss 4.9716 Validation Accuracy 0.0666\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 3 Training: 100%|██████████| 152/152 [00:16<00:00,  9.43it/s, Loss=4.32, Accuracy=0.0799]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3 Training Loss 4.8133 Accuracy 0.0715\n",
            "Epoch 3 Validation Loss 4.4527 Validation Accuracy 0.0802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 4 Training: 100%|██████████| 152/152 [00:14<00:00, 10.28it/s, Loss=4.18, Accuracy=0.0813]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4 Training Loss 4.4005 Accuracy 0.0815\n",
            "Epoch 4 Validation Loss 4.1397 Validation Accuracy 0.0851\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 5 Training: 100%|██████████| 152/152 [00:14<00:00, 10.56it/s, Loss=4.29, Accuracy=0.073]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5 Training Loss 4.1696 Accuracy 0.0847\n",
            "Epoch 5 Validation Loss 3.9696 Validation Accuracy 0.0895\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 6 Training: 100%|██████████| 152/152 [00:14<00:00, 10.51it/s, Loss=4.08, Accuracy=0.0799]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6 Training Loss 3.9694 Accuracy 0.0890\n",
            "Epoch 6 Validation Loss 3.7741 Validation Accuracy 0.0923\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 7 Training: 100%|██████████| 152/152 [00:14<00:00, 10.40it/s, Loss=3.53, Accuracy=0.103]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7 Training Loss 3.7813 Accuracy 0.0938\n",
            "Epoch 7 Validation Loss 3.7420 Validation Accuracy 0.0917\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 8 Training: 100%|██████████| 152/152 [00:14<00:00, 10.39it/s, Loss=3.23, Accuracy=0.107]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8 Training Loss 3.6787 Accuracy 0.0965\n",
            "Epoch 8 Validation Loss 3.6489 Validation Accuracy 0.0940\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 9 Training: 100%|██████████| 152/152 [00:14<00:00, 10.53it/s, Loss=3.33, Accuracy=0.0992]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9 Training Loss 3.5434 Accuracy 0.1010\n",
            "Epoch 9 Validation Loss 3.5812 Validation Accuracy 0.0967\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 10 Training: 100%|██████████| 152/152 [00:14<00:00, 10.52it/s, Loss=3.56, Accuracy=0.0854]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10 Training Loss 3.4748 Accuracy 0.1013\n",
            "Epoch 10 Validation Loss 3.5290 Validation Accuracy 0.0996\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 11 Training: 100%|██████████| 152/152 [00:14<00:00, 10.42it/s, Loss=3.44, Accuracy=0.106]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11 Training Loss 3.3840 Accuracy 0.1040\n",
            "Epoch 11 Validation Loss 3.4472 Validation Accuracy 0.0993\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 12 Training: 100%|██████████| 152/152 [00:14<00:00, 10.55it/s, Loss=3.03, Accuracy=0.116]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12 Training Loss 3.2847 Accuracy 0.1078\n",
            "Epoch 12 Validation Loss 3.4570 Validation Accuracy 0.0993\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 13 Training: 100%|██████████| 152/152 [00:14<00:00, 10.55it/s, Loss=2.89, Accuracy=0.121]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13 Training Loss 3.2421 Accuracy 0.1087\n",
            "Epoch 13 Validation Loss 3.4508 Validation Accuracy 0.0991\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 14 Training: 100%|██████████| 152/152 [00:14<00:00, 10.53it/s, Loss=2.86, Accuracy=0.11]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14 Training Loss 3.1554 Accuracy 0.1123\n",
            "Epoch 14 Validation Loss 3.4300 Validation Accuracy 0.1005\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 15 Training: 100%|██████████| 152/152 [00:14<00:00, 10.52it/s, Loss=3.18, Accuracy=0.103]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15 Training Loss 3.1268 Accuracy 0.1116\n",
            "Epoch 15 Validation Loss 3.3997 Validation Accuracy 0.1027\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 16 Training: 100%|██████████| 152/152 [00:14<00:00, 10.51it/s, Loss=3.06, Accuracy=0.12]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16 Training Loss 3.0576 Accuracy 0.1146\n",
            "Epoch 16 Validation Loss 3.3693 Validation Accuracy 0.1009\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 17 Training: 100%|██████████| 152/152 [00:14<00:00, 10.52it/s, Loss=2.71, Accuracy=0.129]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17 Training Loss 2.9935 Accuracy 0.1166\n",
            "Epoch 17 Validation Loss 3.4250 Validation Accuracy 0.1000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 18 Training: 100%|██████████| 152/152 [00:14<00:00, 10.53it/s, Loss=2.54, Accuracy=0.138]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18 Training Loss 2.9703 Accuracy 0.1176\n",
            "Epoch 18 Validation Loss 3.4356 Validation Accuracy 0.0999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 19 Training: 100%|██████████| 152/152 [00:14<00:00, 10.58it/s, Loss=2.54, Accuracy=0.121]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19 Training Loss 2.9104 Accuracy 0.1198\n",
            "Epoch 19 Validation Loss 3.4171 Validation Accuracy 0.1015\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 20 Training: 100%|██████████| 152/152 [00:14<00:00, 10.60it/s, Loss=2.86, Accuracy=0.112]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20 Training Loss 2.9000 Accuracy 0.1193\n",
            "Epoch 20 Validation Loss 3.4354 Validation Accuracy 0.1028\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 21 Training: 100%|██████████| 152/152 [00:14<00:00, 10.60it/s, Loss=2.9, Accuracy=0.116]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21 Training Loss 2.8545 Accuracy 0.1214\n",
            "Epoch 21 Validation Loss 3.3951 Validation Accuracy 0.1000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 22 Training: 100%|██████████| 152/152 [00:14<00:00, 10.43it/s, Loss=2.56, Accuracy=0.139]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22 Training Loss 2.8228 Accuracy 0.1224\n",
            "Epoch 22 Validation Loss 3.4505 Validation Accuracy 0.1001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 23 Training: 100%|██████████| 152/152 [00:14<00:00, 10.52it/s, Loss=2.4, Accuracy=0.131]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23 Training Loss 2.8207 Accuracy 0.1212\n",
            "Epoch 23 Validation Loss 3.4710 Validation Accuracy 0.0987\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 24 Training: 100%|██████████| 152/152 [00:14<00:00, 10.60it/s, Loss=2.45, Accuracy=0.124]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24 Training Loss 2.7854 Accuracy 0.1234\n",
            "Epoch 24 Validation Loss 3.4595 Validation Accuracy 0.1009\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 25 Training: 100%|██████████| 152/152 [00:14<00:00, 10.60it/s, Loss=2.74, Accuracy=0.123]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25 Training Loss 2.7906 Accuracy 0.1228\n",
            "Epoch 25 Validation Loss 3.5009 Validation Accuracy 0.1011\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 26 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=2.63, Accuracy=0.132]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26 Training Loss 2.7972 Accuracy 0.1215\n",
            "Epoch 26 Validation Loss 3.4682 Validation Accuracy 0.1004\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 27 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=2.51, Accuracy=0.138]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27 Training Loss 2.7713 Accuracy 0.1226\n",
            "Epoch 27 Validation Loss 3.4854 Validation Accuracy 0.1013\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 28 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=2.26, Accuracy=0.149]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28 Training Loss 2.7665 Accuracy 0.1219\n",
            "Epoch 28 Validation Loss 3.4881 Validation Accuracy 0.1007\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 29 Training: 100%|██████████| 152/152 [00:14<00:00, 10.61it/s, Loss=2.4, Accuracy=0.118]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29 Training Loss 2.6860 Accuracy 0.1262\n",
            "Epoch 29 Validation Loss 3.4753 Validation Accuracy 0.1021\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 30 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=2.49, Accuracy=0.135]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30 Training Loss 2.6921 Accuracy 0.1245\n",
            "Epoch 30 Validation Loss 3.4924 Validation Accuracy 0.1007\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 31 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=2.58, Accuracy=0.123]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 31 Training Loss 2.6405 Accuracy 0.1271\n",
            "Epoch 31 Validation Loss 3.4172 Validation Accuracy 0.1013\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 32 Training: 100%|██████████| 152/152 [00:14<00:00, 10.59it/s, Loss=2.33, Accuracy=0.132]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 32 Training Loss 2.5984 Accuracy 0.1283\n",
            "Epoch 32 Validation Loss 3.4552 Validation Accuracy 0.1044\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 33 Training: 100%|██████████| 152/152 [00:14<00:00, 10.58it/s, Loss=2.13, Accuracy=0.142]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 33 Training Loss 2.5696 Accuracy 0.1302\n",
            "Epoch 33 Validation Loss 3.5056 Validation Accuracy 0.1010\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 34 Training: 100%|██████████| 152/152 [00:14<00:00, 10.57it/s, Loss=2.18, Accuracy=0.135]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 34 Training Loss 2.4954 Accuracy 0.1330\n",
            "Epoch 34 Validation Loss 3.4649 Validation Accuracy 0.1041\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 35 Training: 100%|██████████| 152/152 [00:14<00:00, 10.69it/s, Loss=2.27, Accuracy=0.134]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 35 Training Loss 2.4955 Accuracy 0.1322\n",
            "Epoch 35 Validation Loss 3.4965 Validation Accuracy 0.1019\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 36 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=2.25, Accuracy=0.145]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 36 Training Loss 2.4403 Accuracy 0.1343\n",
            "Epoch 36 Validation Loss 3.4067 Validation Accuracy 0.1023\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 37 Training: 100%|██████████| 152/152 [00:14<00:00, 10.62it/s, Loss=2.15, Accuracy=0.163]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 37 Training Loss 2.4179 Accuracy 0.1358\n",
            "Epoch 37 Validation Loss 3.4452 Validation Accuracy 0.1026\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 38 Training: 100%|██████████| 152/152 [00:14<00:00, 10.57it/s, Loss=2.04, Accuracy=0.158]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 38 Training Loss 2.3727 Accuracy 0.1389\n",
            "Epoch 38 Validation Loss 3.5882 Validation Accuracy 0.0994\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 39 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=2.05, Accuracy=0.131]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 39 Training Loss 2.3150 Accuracy 0.1402\n",
            "Epoch 39 Validation Loss 3.4836 Validation Accuracy 0.1044\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 40 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=2.05, Accuracy=0.153]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 40 Training Loss 2.3135 Accuracy 0.1391\n",
            "Epoch 40 Validation Loss 3.5340 Validation Accuracy 0.1020\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 41 Training: 100%|██████████| 152/152 [00:14<00:00, 10.72it/s, Loss=1.94, Accuracy=0.156]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 41 Training Loss 2.2345 Accuracy 0.1430\n",
            "Epoch 41 Validation Loss 3.4505 Validation Accuracy 0.1025\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 42 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=1.94, Accuracy=0.167]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 42 Training Loss 2.2206 Accuracy 0.1447\n",
            "Epoch 42 Validation Loss 3.5160 Validation Accuracy 0.1030\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 43 Training: 100%|██████████| 152/152 [00:14<00:00, 10.61it/s, Loss=1.78, Accuracy=0.165]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 43 Training Loss 2.1776 Accuracy 0.1470\n",
            "Epoch 43 Validation Loss 3.5521 Validation Accuracy 0.1036\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 44 Training: 100%|██████████| 152/152 [00:14<00:00, 10.61it/s, Loss=1.74, Accuracy=0.149]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 44 Training Loss 2.1291 Accuracy 0.1485\n",
            "Epoch 44 Validation Loss 3.5483 Validation Accuracy 0.1043\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 45 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=1.91, Accuracy=0.161]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 45 Training Loss 2.1429 Accuracy 0.1467\n",
            "Epoch 45 Validation Loss 3.5422 Validation Accuracy 0.1026\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 46 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=1.74, Accuracy=0.168]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 46 Training Loss 2.0649 Accuracy 0.1507\n",
            "Epoch 46 Validation Loss 3.4938 Validation Accuracy 0.1023\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 47 Training: 100%|██████████| 152/152 [00:14<00:00, 10.66it/s, Loss=1.94, Accuracy=0.161]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 47 Training Loss 2.0485 Accuracy 0.1525\n",
            "Epoch 47 Validation Loss 3.5720 Validation Accuracy 0.1054\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 48 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=1.59, Accuracy=0.172]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 48 Training Loss 1.9940 Accuracy 0.1550\n",
            "Epoch 48 Validation Loss 3.5987 Validation Accuracy 0.1019\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 49 Training: 100%|██████████| 152/152 [00:14<00:00, 10.67it/s, Loss=1.52, Accuracy=0.167]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 49 Training Loss 1.9494 Accuracy 0.1579\n",
            "Epoch 49 Validation Loss 3.6085 Validation Accuracy 0.1040\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 50 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=1.69, Accuracy=0.161]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 50 Training Loss 1.9573 Accuracy 0.1553\n",
            "Epoch 50 Validation Loss 3.6067 Validation Accuracy 0.1008\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 51 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=1.51, Accuracy=0.18]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 51 Training Loss 1.8951 Accuracy 0.1591\n",
            "Epoch 51 Validation Loss 3.5443 Validation Accuracy 0.1017\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 52 Training: 100%|██████████| 152/152 [00:14<00:00, 10.71it/s, Loss=1.56, Accuracy=0.19]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 52 Training Loss 1.8804 Accuracy 0.1607\n",
            "Epoch 52 Validation Loss 3.6479 Validation Accuracy 0.1039\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 53 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=1.41, Accuracy=0.18]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 53 Training Loss 1.8191 Accuracy 0.1641\n",
            "Epoch 53 Validation Loss 3.7056 Validation Accuracy 0.1015\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 54 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=1.45, Accuracy=0.167]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 54 Training Loss 1.8029 Accuracy 0.1657\n",
            "Epoch 54 Validation Loss 3.6369 Validation Accuracy 0.1055\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 55 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=1.49, Accuracy=0.18]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 55 Training Loss 1.7716 Accuracy 0.1659\n",
            "Epoch 55 Validation Loss 3.7360 Validation Accuracy 0.0992\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 56 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=1.37, Accuracy=0.183]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 56 Training Loss 1.7181 Accuracy 0.1690\n",
            "Epoch 56 Validation Loss 3.6604 Validation Accuracy 0.0995\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 57 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=1.39, Accuracy=0.194]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 57 Training Loss 1.7008 Accuracy 0.1708\n",
            "Epoch 57 Validation Loss 3.7747 Validation Accuracy 0.1028\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 58 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=1.27, Accuracy=0.186]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 58 Training Loss 1.6569 Accuracy 0.1735\n",
            "Epoch 58 Validation Loss 3.8196 Validation Accuracy 0.1017\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 59 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=1.24, Accuracy=0.193]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 59 Training Loss 1.6330 Accuracy 0.1753\n",
            "Epoch 59 Validation Loss 3.7597 Validation Accuracy 0.1012\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 60 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=1.28, Accuracy=0.196]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 60 Training Loss 1.5994 Accuracy 0.1761\n",
            "Epoch 60 Validation Loss 3.8546 Validation Accuracy 0.0979\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 61 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=1.13, Accuracy=0.207]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 61 Training Loss 1.5691 Accuracy 0.1774\n",
            "Epoch 61 Validation Loss 3.7503 Validation Accuracy 0.1000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 62 Training: 100%|██████████| 152/152 [00:14<00:00, 10.69it/s, Loss=1.19, Accuracy=0.211]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 62 Training Loss 1.5545 Accuracy 0.1795\n",
            "Epoch 62 Validation Loss 3.8717 Validation Accuracy 0.1010\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 63 Training: 100%|██████████| 152/152 [00:14<00:00, 10.70it/s, Loss=1.14, Accuracy=0.213]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 63 Training Loss 1.5165 Accuracy 0.1813\n",
            "Epoch 63 Validation Loss 3.9224 Validation Accuracy 0.1017\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 64 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=1.12, Accuracy=0.19]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 64 Training Loss 1.4905 Accuracy 0.1840\n",
            "Epoch 64 Validation Loss 3.8406 Validation Accuracy 0.1025\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 65 Training: 100%|██████████| 152/152 [00:14<00:00, 10.67it/s, Loss=1.17, Accuracy=0.197]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 65 Training Loss 1.4561 Accuracy 0.1846\n",
            "Epoch 65 Validation Loss 3.9370 Validation Accuracy 0.0976\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 66 Training: 100%|██████████| 152/152 [00:14<00:00, 10.66it/s, Loss=1.04, Accuracy=0.215]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 66 Training Loss 1.4196 Accuracy 0.1871\n",
            "Epoch 66 Validation Loss 3.8627 Validation Accuracy 0.0989\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 67 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=1.07, Accuracy=0.233]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 67 Training Loss 1.4069 Accuracy 0.1886\n",
            "Epoch 67 Validation Loss 4.0088 Validation Accuracy 0.1006\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 68 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=1.04, Accuracy=0.212]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 68 Training Loss 1.3707 Accuracy 0.1903\n",
            "Epoch 68 Validation Loss 4.0408 Validation Accuracy 0.0998\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 69 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=0.938, Accuracy=0.207]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 69 Training Loss 1.3511 Accuracy 0.1929\n",
            "Epoch 69 Validation Loss 3.9591 Validation Accuracy 0.0995\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 70 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=0.909, Accuracy=0.23]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 70 Training Loss 1.3298 Accuracy 0.1933\n",
            "Epoch 70 Validation Loss 4.0722 Validation Accuracy 0.0971\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 71 Training: 100%|██████████| 152/152 [00:14<00:00, 10.66it/s, Loss=0.849, Accuracy=0.226]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 71 Training Loss 1.2864 Accuracy 0.1964\n",
            "Epoch 71 Validation Loss 4.0002 Validation Accuracy 0.1001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 72 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=1.05, Accuracy=0.231]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 72 Training Loss 1.2723 Accuracy 0.1973\n",
            "Epoch 72 Validation Loss 4.1511 Validation Accuracy 0.0981\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 73 Training: 100%|██████████| 152/152 [00:14<00:00, 10.66it/s, Loss=0.808, Accuracy=0.231]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 73 Training Loss 1.2396 Accuracy 0.1997\n",
            "Epoch 73 Validation Loss 4.1726 Validation Accuracy 0.0994\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 74 Training: 100%|██████████| 152/152 [00:14<00:00, 10.66it/s, Loss=0.89, Accuracy=0.209]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 74 Training Loss 1.2401 Accuracy 0.2009\n",
            "Epoch 74 Validation Loss 4.1325 Validation Accuracy 0.0993\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 75 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=0.872, Accuracy=0.231]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 75 Training Loss 1.2083 Accuracy 0.2015\n",
            "Epoch 75 Validation Loss 4.1961 Validation Accuracy 0.0956\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 76 Training: 100%|██████████| 152/152 [00:14<00:00, 10.67it/s, Loss=0.858, Accuracy=0.223]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 76 Training Loss 1.1922 Accuracy 0.2025\n",
            "Epoch 76 Validation Loss 4.1116 Validation Accuracy 0.1000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 77 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=0.878, Accuracy=0.241]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 77 Training Loss 1.1621 Accuracy 0.2059\n",
            "Epoch 77 Validation Loss 4.2813 Validation Accuracy 0.0996\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 78 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=0.861, Accuracy=0.222]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 78 Training Loss 1.1434 Accuracy 0.2071\n",
            "Epoch 78 Validation Loss 4.3142 Validation Accuracy 0.0974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 79 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=0.812, Accuracy=0.219]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 79 Training Loss 1.1283 Accuracy 0.2083\n",
            "Epoch 79 Validation Loss 4.2617 Validation Accuracy 0.0992\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 80 Training: 100%|██████████| 152/152 [00:14<00:00, 10.72it/s, Loss=0.792, Accuracy=0.233]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 80 Training Loss 1.0933 Accuracy 0.2107\n",
            "Epoch 80 Validation Loss 4.3258 Validation Accuracy 0.0963\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 81 Training: 100%|██████████| 152/152 [00:14<00:00, 10.72it/s, Loss=0.681, Accuracy=0.249]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 81 Training Loss 1.1030 Accuracy 0.2090\n",
            "Epoch 81 Validation Loss 4.2222 Validation Accuracy 0.0984\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 82 Training: 100%|██████████| 152/152 [00:14<00:00, 10.66it/s, Loss=0.774, Accuracy=0.263]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 82 Training Loss 1.0805 Accuracy 0.2118\n",
            "Epoch 82 Validation Loss 4.4332 Validation Accuracy 0.0983\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 83 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=0.739, Accuracy=0.227]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 83 Training Loss 1.0508 Accuracy 0.2138\n",
            "Epoch 83 Validation Loss 4.4158 Validation Accuracy 0.0979\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 84 Training: 100%|██████████| 152/152 [00:14<00:00, 10.63it/s, Loss=0.837, Accuracy=0.225]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 84 Training Loss 1.0331 Accuracy 0.2165\n",
            "Epoch 84 Validation Loss 4.3703 Validation Accuracy 0.0963\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 85 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=0.79, Accuracy=0.23]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 85 Training Loss 1.0059 Accuracy 0.2175\n",
            "Epoch 85 Validation Loss 4.4024 Validation Accuracy 0.0965\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 86 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=0.791, Accuracy=0.231]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 86 Training Loss 0.9933 Accuracy 0.2178\n",
            "Epoch 86 Validation Loss 4.3491 Validation Accuracy 0.0987\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 87 Training: 100%|██████████| 152/152 [00:14<00:00, 10.64it/s, Loss=0.69, Accuracy=0.256]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 87 Training Loss 0.9955 Accuracy 0.2190\n",
            "Epoch 87 Validation Loss 4.5593 Validation Accuracy 0.0987\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 88 Training: 100%|██████████| 152/152 [00:14<00:00, 10.65it/s, Loss=0.71, Accuracy=0.238]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 88 Training Loss 0.9764 Accuracy 0.2197\n",
            "Epoch 88 Validation Loss 4.5670 Validation Accuracy 0.0992\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 89 Training: 100%|██████████| 152/152 [00:14<00:00, 10.67it/s, Loss=0.787, Accuracy=0.219]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 89 Training Loss 0.9561 Accuracy 0.2229\n",
            "Epoch 89 Validation Loss 4.4948 Validation Accuracy 0.0965\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 90 Training: 100%|██████████| 152/152 [00:14<00:00, 10.72it/s, Loss=0.753, Accuracy=0.238]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 90 Training Loss 0.9377 Accuracy 0.2229\n",
            "Epoch 90 Validation Loss 4.5708 Validation Accuracy 0.0917\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 91 Training: 100%|██████████| 152/152 [00:14<00:00, 10.73it/s, Loss=0.656, Accuracy=0.249]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 91 Training Loss 0.9292 Accuracy 0.2231\n",
            "Epoch 91 Validation Loss 4.4741 Validation Accuracy 0.0966\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 92 Training: 100%|██████████| 152/152 [00:14<00:00, 10.72it/s, Loss=0.744, Accuracy=0.244]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 92 Training Loss 0.9216 Accuracy 0.2247\n",
            "Epoch 92 Validation Loss 4.6817 Validation Accuracy 0.0968\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 93 Training: 100%|██████████| 152/152 [00:14<00:00, 10.72it/s, Loss=0.628, Accuracy=0.241]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 93 Training Loss 0.8945 Accuracy 0.2263\n",
            "Epoch 93 Validation Loss 4.6764 Validation Accuracy 0.0961\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 94 Training: 100%|██████████| 152/152 [00:14<00:00, 10.72it/s, Loss=0.634, Accuracy=0.236]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 94 Training Loss 0.8927 Accuracy 0.2279\n",
            "Epoch 94 Validation Loss 4.6066 Validation Accuracy 0.0966\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 95 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=0.645, Accuracy=0.242]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 95 Training Loss 0.8682 Accuracy 0.2293\n",
            "Epoch 95 Validation Loss 4.6816 Validation Accuracy 0.0920\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 96 Training: 100%|██████████| 152/152 [00:14<00:00, 10.69it/s, Loss=0.618, Accuracy=0.251]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 96 Training Loss 0.8618 Accuracy 0.2284\n",
            "Epoch 96 Validation Loss 4.5859 Validation Accuracy 0.0965\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 97 Training: 100%|██████████| 152/152 [00:14<00:00, 10.69it/s, Loss=0.746, Accuracy=0.244]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 97 Training Loss 0.8531 Accuracy 0.2300\n",
            "Epoch 97 Validation Loss 4.7531 Validation Accuracy 0.0950\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 98 Training: 100%|██████████| 152/152 [00:14<00:00, 10.69it/s, Loss=0.628, Accuracy=0.245]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 98 Training Loss 0.8363 Accuracy 0.2313\n",
            "Epoch 98 Validation Loss 4.7661 Validation Accuracy 0.0956\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 99 Training: 100%|██████████| 152/152 [00:14<00:00, 10.68it/s, Loss=0.623, Accuracy=0.238]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 99 Training Loss 0.8202 Accuracy 0.2336\n",
            "Epoch 99 Validation Loss 4.7362 Validation Accuracy 0.0972\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 100 Training: 100%|██████████| 152/152 [00:14<00:00, 10.66it/s, Loss=0.649, Accuracy=0.247]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 100 Training Loss 0.8069 Accuracy 0.2339\n",
            "Epoch 100 Validation Loss 4.7815 Validation Accuracy 0.0913\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def beam_search(model, img_tensor, start, end, k=3, max_len=50):\n",
        "    batch_size = img_tensor.size(0)\n",
        "    sequences_batch = [[[(start,), 1.0]] for _ in range(batch_size)]\n",
        "\n",
        "    for _ in range(max_len):\n",
        "        all_candidates_batch = [[] for _ in range(batch_size)]\n",
        "        all_sequences_complete = True\n",
        "\n",
        "        for batch in range(batch_size):\n",
        "            sequences = sequences_batch[batch]\n",
        "            all_candidates = all_candidates_batch[batch]\n",
        "            for seq_and_score in sequences:\n",
        "                seq, score = seq_and_score[0], seq_and_score[1]\n",
        "                if len(seq) >= max_len or seq[-1] == end:\n",
        "                    all_candidates.append(seq_and_score)\n",
        "                    continue\n",
        "\n",
        "                all_sequences_complete = False\n",
        "\n",
        "                dec_input = torch.tensor([seq], dtype=torch.long, device=device)\n",
        "\n",
        "                predictions = model(img_tensor[batch].unsqueeze(0), dec_input)\n",
        "                predictions = predictions[:, -1:, :]\n",
        "                predictions = torch.softmax(predictions, dim=-1)\n",
        "\n",
        "                top_k_probs, top_k_tokens = torch.topk(predictions, k)\n",
        "                top_k_log_probs = top_k_probs.log().squeeze()\n",
        "\n",
        "                for j in range(k):\n",
        "                    next_token = top_k_tokens[0][0][j].item()\n",
        "                    if next_token == end:\n",
        "                        all_candidates.append((seq + (next_token,), score + top_k_log_probs[j].item()))\n",
        "                        continue\n",
        "                    else:\n",
        "                        candidate = (seq + (next_token,), score + top_k_log_probs[j].item())\n",
        "                        all_candidates.append(candidate)\n",
        "\n",
        "            ordered = sorted(all_candidates, key=lambda tup: tup[1], reverse=True)\n",
        "\n",
        "            sequences_batch[batch] = ordered[:k]\n",
        "\n",
        "        if all_sequences_complete:\n",
        "            break\n",
        "\n",
        "    return sequences_batch\n"
      ],
      "metadata": {
        "id": "yWwF03WsJ52Y"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_bleu(references, hypothesis, weights):\n",
        "    bleu_scores = []\n",
        "    for ref in references:\n",
        "        ref_tokens = [token for token in ref.split()[1:-1] if token != '<unk>']\n",
        "        hyp_tokens = [token for token in hypothesis.split()[1:-1] if token != '<unk>']\n",
        "        bleu_score = sentence_bleu([ref_tokens], hyp_tokens, weights=weights, smoothing_function=SmoothingFunction().method1)\n",
        "        bleu_scores.append(bleu_score * 100)\n",
        "    avg_bleu_score = sum(bleu_scores) / len(bleu_scores)\n",
        "    return avg_bleu_score\n",
        "\n",
        "\n",
        "\n",
        "weights_list = [(1.0, 0, 0, 0), (0.5, 0.5, 0, 0), (0.3, 0.3, 0.3, 0), (0.25, 0.25, 0.25, 0.25)]\n",
        "\n",
        "\n",
        "total_bleu_scores = {weights: [] for weights in weights_list}\n",
        "\n",
        "for batch, (img_tensor, tar) in enumerate(test_data):\n",
        "    img_tensor, tar = img_tensor.to(device), tar.to(device)\n",
        "    img_tensor = img_tensor.float()\n",
        "    tar = tar.long()\n",
        "    start = tokenizer.word_index['<start>']\n",
        "    end = tokenizer.word_index['<end>']\n",
        "\n",
        "    batch_bleu_scores = {weights: [] for weights in weights_list}\n",
        "\n",
        "    output_sequences_batch = beam_search(model, img_tensor, start, end, k=3, max_len=50)\n",
        "\n",
        "    for i, (output_sequences, target) in enumerate(zip(output_sequences_batch, tar)):\n",
        "        target_sentence = tokenizer.sequences_to_texts([target.cpu().numpy()])[0]\n",
        "\n",
        "        best_generated_sentences = {weights: None for weights in weights_list}\n",
        "        best_scores = {weights: -1 for weights in weights_list}\n",
        "\n",
        "        for j, (seq, _) in enumerate(output_sequences):\n",
        "            generated_sentence = tokenizer.sequences_to_texts([seq])[0]\n",
        "\n",
        "            for weights in weights_list:\n",
        "                bleu_score = evaluate_bleu([target_sentence], generated_sentence, weights)\n",
        "\n",
        "                if bleu_score > best_scores[weights]:\n",
        "                    best_generated_sentences[weights] = generated_sentence\n",
        "                    best_scores[weights] = bleu_score\n",
        "\n",
        "        for weights in weights_list:\n",
        "            batch_bleu_scores[weights].append(best_scores[weights])\n",
        "\n",
        "    for weights in weights_list:\n",
        "        avg_batch_bleu_score = np.mean(batch_bleu_scores[weights])\n",
        "        total_bleu_scores[weights].append(avg_batch_bleu_score)\n",
        "    break\n",
        "\n",
        "overall_avg_bleu_scores = {weights: np.mean(scores) for weights, scores in total_bleu_scores.items()}\n",
        "\n",
        "for weights, avg_score in overall_avg_bleu_scores.items():\n",
        "    print(f\"Average BLEU Score for weights {weights}: {avg_score}\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V6U5DKSpuvds",
        "outputId": "0e30a1f0-f2f2-49b2-b1ab-9493f4e3ee83"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average BLEU Score for weights (1.0, 0, 0, 0): 3.8530133027160964\n",
            "Average BLEU Score for weights (0.5, 0.5, 0, 0): 2.050459568302278\n",
            "Average BLEU Score for weights (0.3, 0.3, 0.3, 0): 1.6340909479885535\n",
            "Average BLEU Score for weights (0.25, 0.25, 0.25, 0.25): 0.9452908998030194\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(img_tensor, target):\n",
        "    start_token = tokenizer.word_index['<start>']\n",
        "    end_token = tokenizer.word_index['<end>']\n",
        "\n",
        "    beam_width = 3\n",
        "    beam_result = beam_search(model, img_tensor, start_token, end_token, k=beam_width)\n",
        "\n",
        "    output_sequence = beam_result[0][0][0][1:-1]\n",
        "    output_sequence = [tokenizer.index_word[idx] for idx in output_sequence]\n",
        "\n",
        "    target_seq = [tokenizer.index_word[int(idx)] for idx in target if idx not in [1, 2, 0, 3]]\n",
        "\n",
        "    print('Target sequence: ', ' '.join(target_seq))\n",
        "    print('Predicted sequence: ', ' '.join(output_sequence))\n",
        "\n",
        "    bleu_score = sentence_bleu([target_seq], output_sequence,\n",
        "                               weights=(0.25, 0.25, 0.25, 0.25),\n",
        "                               smoothing_function=SmoothingFunction().method1) * 100\n",
        "\n",
        "    print('Bleu Score:', bleu_score)\n"
      ],
      "metadata": {
        "id": "P-p8xROBW-gd"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "i = 0\n",
        "for img_tensors, targets in test_data:\n",
        "  target = targets[i]\n",
        "  img_tensor = img_tensors[i]\n",
        "  img_tensor = img_tensor[None, :, :]\n",
        "  img_tensor, target = img_tensor.to(device), target.to(device)\n",
        "  evaluate(img_tensor, target)\n",
        "  i += 1\n",
        "  if (i == 5):\n",
        "    break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CrIp0SNkk2DO",
        "outputId": "25851d0f-6724-4e72-e236-66bcc4fd4823"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target sequence:  black and white dog is jumping through field of brown grass\n",
            "Predicted sequence:  black and white dog running through field\n",
            "Bleu Score: 27.610369103579487\n",
            "Target sequence:  little girl in bathing suit leaps up in the water\n",
            "Predicted sequence:  young boy jumping into the ocean\n",
            "Bleu Score: 2.096016611399374\n",
            "Target sequence:  the two children watch the small yellow dog on the tree\n",
            "Predicted sequence:  four dogs are playing around tree\n",
            "Bleu Score: 1.774239756616722\n",
            "Target sequence:  boy jumping rail on his skateboard\n",
            "Predicted sequence:  young man wearing gray shirt and jeans skateboards down street\n",
            "Bleu Score: 0\n",
            "Target sequence:  two people on grassy plain are gathering parachute that one of the people just used\n",
            "Predicted sequence:  two people are in the air above the grass\n",
            "Bleu Score: 3.096277640217347\n"
          ]
        }
      ]
    }
  ]
}