Models/qa-with-nq-dataset-and-bert-base.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"name":"qa-with-nq-dataset-and-bert-base.ipynb","provenance":[],"collapsed_sections":[]}},"nbformat_minor":0,"nbformat":4,"cells":[{"cell_type":"markdown","source":["# Question Answering with NQ dataset and BERT-Base\n","\n","\n","In this notebook i train again bert-base-uncased for qa tasks but this time with the [NQ](https://ai.google.com/research/NaturalQuestions/download) dataset. I use the same model parameters with the SQuAD 2.0 model.\n","\n","**Note:** due to limitations on the free version of google colab, the experiments on this assignment were executed on [Kaggle](https://www.kaggle.com/)."],"metadata":{"id":"is-BQ_NxNsO0"}},{"cell_type":"markdown","source":["## First Steps"],"metadata":{"id":"jTfBhfsHN7-a"}},{"cell_type":"markdown","source":["**GPU Info**\n","\n","To get the same results you should have a Tesla P100 GPU. This GPU is available for free usage on [Kaggle](https://www.kaggle.com/)."],"metadata":{"id":"8GvtY-GhN-vi"}},{"cell_type":"code","source":["!nvidia-smi"],"metadata":{"id":"p2mc9E6eNa0R","outputId":"562617ae-c729-438b-97fd-e989fc77de0b","executionInfo":{"status":"ok","timestamp":1646593693702,"user_tz":-120,"elapsed":245,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:29:44.102396Z","iopub.execute_input":"2022-03-09T11:29:44.102919Z","iopub.status.idle":"2022-03-09T11:29:44.855081Z","shell.execute_reply.started":"2022-03-09T11:29:44.102811Z","shell.execute_reply":"2022-03-09T11:29:44.854159Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Wed Mar  9 11:29:44 2022       \n+-----------------------------------------------------------------------------+\n| NVIDIA-SMI 450.119.04   Driver Version: 450.119.04   CUDA Version: 11.0     |\n|-------------------------------+----------------------+----------------------+\n| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n|                               |                      |               MIG M. |\n|===============================+======================+======================|\n|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n| N/A   38C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n|                               |                      |                  N/A |\n+-------------------------------+----------------------+----------------------+\n                                                                               \n+-----------------------------------------------------------------------------+\n| Processes:                                                                  |\n|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n|        ID   ID                                                   Usage      |\n|=============================================================================|\n|  No running processes found                                                 |\n+-----------------------------------------------------------------------------+\n","output_type":"stream"}]},{"cell_type":"markdown","source":["**Install pytorch interface for pre-trained BERT**\n","\n","Then, we will install the interface to use the pre-trained BERT model."],"metadata":{"id":"_pSfqxmEPx88"}},{"cell_type":"code","source":["!pip install transformers"],"metadata":{"id":"0rZyAPuhPySz","outputId":"3fe2671b-41ae-4fd3-f652-03fb21a9c2f4","executionInfo":{"status":"ok","timestamp":1646593712064,"user_tz":-120,"elapsed":16066,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:29:49.203192Z","iopub.execute_input":"2022-03-09T11:29:49.203639Z","iopub.status.idle":"2022-03-09T11:29:58.176752Z","shell.execute_reply.started":"2022-03-09T11:29:49.203598Z","shell.execute_reply":"2022-03-09T11:29:58.175915Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Requirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (4.15.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.2.1)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from transformers) (4.11.2)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (1.20.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (6.0)\nRequirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers) (0.0.47)\nRequirement already satisfied: tokenizers<0.11,>=0.10.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.10.3)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.4.2)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (21.3)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.26.0)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.62.3)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2021.11.10)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=20.0->transformers) (3.0.6)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.6.0)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2021.10.8)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.26.7)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (3.1)\nRequirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.0.9)\nRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.16.0)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.1.0)\nRequirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (8.0.3)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n","output_type":"stream"}]},{"cell_type":"markdown","source":["**Import necessary libraries**\n","\n","Now, it's time to import the necessary libraries for this notebook."],"metadata":{"id":"VRmYivZrQPrR"}},{"cell_type":"code","source":["import torch\n","import json\n","import pandas as pd\n","import numpy as np\n","import random\n","import matplotlib.pyplot as plt\n","import re\n","import string\n","import collections\n","from transformers import BertTokenizerFast, BertForQuestionAnswering\n","from transformers.tokenization_utils_base import BatchEncoding\n","from torch.utils.data import Dataset, DataLoader\n","from tqdm import tqdm\n","\n","pd.set_option('max_colwidth', 500)\n","%matplotlib inline"],"metadata":{"id":"V9xaR0zaQP_g","execution":{"iopub.status.busy":"2022-03-09T11:30:44.420038Z","iopub.execute_input":"2022-03-09T11:30:44.420654Z","iopub.status.idle":"2022-03-09T11:30:46.166874Z","shell.execute_reply.started":"2022-03-09T11:30:44.420615Z","shell.execute_reply":"2022-03-09T11:30:46.166176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["**Enable CUDA**\n","\n","Enable CUDA for GPU utilization by our model. This makes calculations and thus the training of our models faster."],"metadata":{"id":"xUz3L_c8QVXR"}},{"cell_type":"code","source":["device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n","device"],"metadata":{"id":"doKXFdk8QVoD","outputId":"8f33ad68-3224-4e14-bf58-989696447dc4","executionInfo":{"status":"ok","timestamp":1646593723427,"user_tz":-120,"elapsed":249,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:30:50.129016Z","iopub.execute_input":"2022-03-09T11:30:50.129795Z","iopub.status.idle":"2022-03-09T11:30:50.176022Z","shell.execute_reply.started":"2022-03-09T11:30:50.129750Z","shell.execute_reply":"2022-03-09T11:30:50.175240Z"},"trusted":true},"execution_count":null,"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"device(type='cuda', index=0)"},"metadata":{}}]},{"cell_type":"markdown","source":["**Set random seed**\n","\n","Set a constant random seed in order to get the same (deterministic) outputs every time we run our models.\n","\n"],"metadata":{"id":"HKOXPf5XQX3Z"}},{"cell_type":"code","source":["seed = 17064\n","def reset_seed():\n","  random.seed(seed)\n","  np.random.seed(seed)\n","  torch.manual_seed(seed)\n","  torch.cuda.manual_seed_all(seed)\n","reset_seed()"],"metadata":{"id":"QnruhHSWQcGO","execution":{"iopub.status.busy":"2022-03-09T11:30:52.137145Z","iopub.execute_input":"2022-03-09T11:30:52.137791Z","iopub.status.idle":"2022-03-09T11:30:52.145663Z","shell.execute_reply.started":"2022-03-09T11:30:52.137755Z","shell.execute_reply":"2022-03-09T11:30:52.144978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["**BERT model definition**\n","\n","And here, we define the name of the pre-trained bert model from [Hugging Face](https://huggingface.co/models) which we'll use."],"metadata":{"id":"Y-Hb3sovQwHa"}},{"cell_type":"code","source":["BERT_MODEL_NAME = 'bert-base-uncased'"],"metadata":{"id":"wqtitofaQdb3","execution":{"iopub.status.busy":"2022-03-09T11:30:54.035440Z","iopub.execute_input":"2022-03-09T11:30:54.036202Z","iopub.status.idle":"2022-03-09T11:30:54.039856Z","shell.execute_reply.started":"2022-03-09T11:30:54.036161Z","shell.execute_reply":"2022-03-09T11:30:54.038963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Dataset loading\n","\n","Now, let's load our datasets:"],"metadata":{"id":"mtvSPZfoQ5f4"}},{"cell_type":"markdown","source":["**Define json preprocessing function**\n","\n","First, we will define a function that takes a squad json dataset file path as argument and returns a dataframe with the questions, their answers in the form (start_position, end_position, answer_text) and the corpus that contains the answer to the corresponding question."],"metadata":{"id":"Jnby8mS-gXoz"}},{"cell_type":"code","source":["def squad_load_from_json(json_file_path: str):\n","  with open(json_file_path, \"r\") as f:\n","    json_data = json.load(f)['data']\n","    questions = []\n","    answers = []\n","    corpuses = []\n","    for category in json_data:\n","      for paragraph in category['paragraphs']:\n","        context = paragraph['context']\n","        for qa in paragraph['qas']:\n","          corpuses.append(context)\n","          question = qa['question']\n","          questions.append(question)\n","          ans_list = qa['answers']\n","          ans_set = set()\n","          for idx, ans in enumerate(ans_list):\n","            ans_set.add((ans['answer_start'], ans['answer_start']+len(ans['text']), ans['text']))\n","          answers.append(list(ans_set))\n","    return pd.DataFrame(data={'question':questions, 'answer':answers, 'corpus':corpuses})"],"metadata":{"id":"mWhOlZ69gcKr","execution":{"iopub.status.busy":"2022-03-09T11:32:16.043094Z","iopub.execute_input":"2022-03-09T11:32:16.043391Z","iopub.status.idle":"2022-03-09T11:32:16.052447Z","shell.execute_reply.started":"2022-03-09T11:32:16.043358Z","shell.execute_reply":"2022-03-09T11:32:16.051691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Then, we will load the datasets converted to SQuAD json format."],"metadata":{"id":"Fny3tu46U60P"}},{"cell_type":"markdown","source":["**Train Dataset**"],"metadata":{"id":"ObZ_q6UXQ8v3"}},{"cell_type":"code","source":["train_dataset = squad_load_from_json(\"../input/nq-squad-format/nq_train.json\")\n","train_dataset"],"metadata":{"id":"YIUz68o5g78D","outputId":"69da4e90-8a7f-46c9-e99e-262342eaa377","executionInfo":{"status":"ok","timestamp":1646593731851,"user_tz":-120,"elapsed":1484,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:32:18.226824Z","iopub.execute_input":"2022-03-09T11:32:18.227396Z","iopub.status.idle":"2022-03-09T11:32:22.299439Z","shell.execute_reply.started":"2022-03-09T11:32:18.227354Z","shell.execute_reply":"2022-03-09T11:32:22.298743Z"},"trusted":true},"execution_count":null,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"                                                 question  \\\n0                           who sings it's going to be me   \n1                   when does the clown movie it come out   \n2                      who sang the song come fly with me   \n3           what is the brightest star in corona borealis   \n4            who owns the land under this navigable river   \n...                                                   ...   \n110860            who wrote the music for game of thrones   \n110861  where did the tradition of santa clause come from   \n110862      what is the population of amherst nova scotia   \n110863       where are villi found in the small intestine   \n110864                  who bats in the top of the inning   \n\n                                                                                                                                                                                                                                              answer  \\\n0                                                                                                                                                                                                                                  [(50, 55, NSYNC)]   \n1                                                                                                                                                                                                                                                 []   \n2                                                                                                                                                                                                                  [(0, 22, Francis Albert Sinatra)]   \n3                                                                                                                                                                                               [(22, 62, the magnitude 2.2 Alpha Coronae Borealis)]   \n4                                                                                                                                                                                                                                [(188, 193, state)]   \n...                                                                                                                                                                                                                                              ...   \n110860                                                                                                                                                                                                                     [(67, 80, Ramin Djawadi)]   \n110861  [(368, 590, traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure of Father Christmas and the Dutch figure of Sinterklaas (himself also based on Saint Nicholas))]   \n110862                                                                                                                                                                                                                           [(110, 115, 9,413)]   \n110863                                                                                                                                                                                                                         [(88, 97, the lumen)]   \n110864                                                                                                                                                                                                               [(239, 256, The visiting team)]   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     corpus  \n0               \"It's Gonna Be Me\" is a song by American boy band NSYNC. It was released on June 13, 2000 as the second single in the United States and third in Europe from their second studio album No Strings Attached. The song was NSYNC's only Billboard Hot 100 number-one hit, making it their highest-charting single. It topped the chart for two consecutive weeks and has been certified Gold by the RIAA. The song serves as the opening track of the 2000 compilation album Now That's What I Call Music! 5.  \n1                    It was released in North America on September 8, 2017.[197] In Europe, the film was released in Belgium on September 6, 2017, Denmark and the Netherlands on September 7, 2017,[198][199] and Norway and Finland on September 8, 2017.[200] On March 7, 2017, the alternate title of the film was announced by Stephen King as Part 1 – The Losers’ Club.[201][202] In addition to the conventional 2D format, It was also released across 615 IMAX screens globally, including 389 domestically.[203]  \n2       Francis Albert Sinatra (/sɪˈnɑːtrə/; Italian: [siˈnaːtra]; December 12, 1915 – May 14, 1998) was an American singer, actor, and producer who was one of the most popular and influential musical artists of the 20th century. He is one of the best-selling music artists of all time, having sold more than 150 million records worldwide.[2] Born in Hoboken, New Jersey, to Italian immigrants, Sinatra began his musical career in the swing era with bandleaders Harry James and Tommy Dorsey. Sinatra foun...  \n3       The brightest star is the magnitude 2.2 Alpha Coronae Borealis. The yellow supergiant R Coronae Borealis is the prototype of a rare class of giant stars—the R Coronae Borealis variables—that are extremely hydrogen deficient, and thought to result from the merger of two white dwarfs. T Coronae Borealis, also known as the Blaze Star, is another unusual type of variable star known as a recurrent nova. Normally of magnitude 10, it last flared up to magnitude 2 in 1946. ADS 9731 and Sigma Coronae...  \n4       In determining the contours of riparian rights, there is a clear distinction between navigable (public) waters and non-navigable waters. The land below navigable waters is the property of state,[3] and subject to all the public land laws and in most states public trust rights. Navigable waters are treated as public highways with any exclusive riparian right ending at the ordinary high water mark. Like a road, any riparian right is subordinate to the public's right to travel on the river, but...  \n...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     ...  \n110860                                                                                                          The music for the fantasy TV series Game of Thrones is composed by Ramin Djawadi. The music is primarily instrumental with the occasional vocal performances, and is created to support musically the characters and plots of the show. It features various theme, the most prominent is its Main Title that accompanies the series' title sequence. In every season, a soundtrack album would be released.  \n110861  Santa Claus, also known as Saint Nicholas, Kris Kringle, Father Christmas, or simply Santa, is a legendary figure originating in Western Christian culture who is said to bring gifts to the homes of well-behaved (\"good\" or \"nice\") children on Christmas Eve (24 December) and the early morning hours of Christmas Day (25 December).[1] The modern Santa Claus grew out of traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure ...  \n110862                                                                                                                                                  In the 2016 Census of Population conducted by Statistics Canada, the Town of Amherst recorded a population of 9,413 living in 4,372 of its 4,745 total private dwellings, a change of 2999690000000000000♠−3.1% from its 2011 population of 9,717. With a land area of 12.07 km2 (4.66 sq mi), it had a population density of 779.9/km2 (2,019.8/sq mi) in 2016.[1]  \n110863                                   Intestinal villi (singular: villus) are small, finger-like projections that extend into the lumen of the small intestine. Each villus is approximately 0.5–1.6 mm in length (in humans), and has many microvilli projecting from the enterocytes of its epithelium which collectively form the striated or brush border. Each of these microvilli are much smaller than a single villus. The intestinal villi are much smaller than any of the circular folds in the intestine. \\n  \n110864  Each half-inning formally starts when the umpire calls \"Play\" or \"Play ball\". A full inning consists of six outs, three for each team; and, in Major League Baseball and most other adult leagues, a regulation game consists of nine innings. The visiting team bats in the first half-inning, the top of the inning, derived from the position of the visiting team at the top line of a baseball line score. The home team's half of an inning is the bottom of the inning, and the break between halves of a...  \n\n[110865 rows x 3 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>question</th>\n      <th>answer</th>\n      <th>corpus</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>who sings it's going to be me</td>\n      <td>[(50, 55, NSYNC)]</td>\n      <td>\"It's Gonna Be Me\" is a song by American boy band NSYNC. It was released on June 13, 2000 as the second single in the United States and third in Europe from their second studio album No Strings Attached. The song was NSYNC's only Billboard Hot 100 number-one hit, making it their highest-charting single. It topped the chart for two consecutive weeks and has been certified Gold by the RIAA. The song serves as the opening track of the 2000 compilation album Now That's What I Call Music! 5.</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>when does the clown movie it come out</td>\n      <td>[]</td>\n      <td>It was released in North America on September 8, 2017.[197] In Europe, the film was released in Belgium on September 6, 2017, Denmark and the Netherlands on September 7, 2017,[198][199] and Norway and Finland on September 8, 2017.[200] On March 7, 2017, the alternate title of the film was announced by Stephen King as Part 1 – The Losers’ Club.[201][202] In addition to the conventional 2D format, It was also released across 615 IMAX screens globally, including 389 domestically.[203]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>who sang the song come fly with me</td>\n      <td>[(0, 22, Francis Albert Sinatra)]</td>\n      <td>Francis Albert Sinatra (/sɪˈnɑːtrə/; Italian: [siˈnaːtra]; December 12, 1915 – May 14, 1998) was an American singer, actor, and producer who was one of the most popular and influential musical artists of the 20th century. He is one of the best-selling music artists of all time, having sold more than 150 million records worldwide.[2] Born in Hoboken, New Jersey, to Italian immigrants, Sinatra began his musical career in the swing era with bandleaders Harry James and Tommy Dorsey. Sinatra foun...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>what is the brightest star in corona borealis</td>\n      <td>[(22, 62, the magnitude 2.2 Alpha Coronae Borealis)]</td>\n      <td>The brightest star is the magnitude 2.2 Alpha Coronae Borealis. The yellow supergiant R Coronae Borealis is the prototype of a rare class of giant stars—the R Coronae Borealis variables—that are extremely hydrogen deficient, and thought to result from the merger of two white dwarfs. T Coronae Borealis, also known as the Blaze Star, is another unusual type of variable star known as a recurrent nova. Normally of magnitude 10, it last flared up to magnitude 2 in 1946. ADS 9731 and Sigma Coronae...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>who owns the land under this navigable river</td>\n      <td>[(188, 193, state)]</td>\n      <td>In determining the contours of riparian rights, there is a clear distinction between navigable (public) waters and non-navigable waters. The land below navigable waters is the property of state,[3] and subject to all the public land laws and in most states public trust rights. Navigable waters are treated as public highways with any exclusive riparian right ending at the ordinary high water mark. Like a road, any riparian right is subordinate to the public's right to travel on the river, but...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>110860</th>\n      <td>who wrote the music for game of thrones</td>\n      <td>[(67, 80, Ramin Djawadi)]</td>\n      <td>The music for the fantasy TV series Game of Thrones is composed by Ramin Djawadi. The music is primarily instrumental with the occasional vocal performances, and is created to support musically the characters and plots of the show. It features various theme, the most prominent is its Main Title that accompanies the series' title sequence. In every season, a soundtrack album would be released.</td>\n    </tr>\n    <tr>\n      <th>110861</th>\n      <td>where did the tradition of santa clause come from</td>\n      <td>[(368, 590, traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure of Father Christmas and the Dutch figure of Sinterklaas (himself also based on Saint Nicholas))]</td>\n      <td>Santa Claus, also known as Saint Nicholas, Kris Kringle, Father Christmas, or simply Santa, is a legendary figure originating in Western Christian culture who is said to bring gifts to the homes of well-behaved (\"good\" or \"nice\") children on Christmas Eve (24 December) and the early morning hours of Christmas Day (25 December).[1] The modern Santa Claus grew out of traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure ...</td>\n    </tr>\n    <tr>\n      <th>110862</th>\n      <td>what is the population of amherst nova scotia</td>\n      <td>[(110, 115, 9,413)]</td>\n      <td>In the 2016 Census of Population conducted by Statistics Canada, the Town of Amherst recorded a population of 9,413 living in 4,372 of its 4,745 total private dwellings, a change of 2999690000000000000♠−3.1% from its 2011 population of 9,717. With a land area of 12.07 km2 (4.66 sq mi), it had a population density of 779.9/km2 (2,019.8/sq mi) in 2016.[1]</td>\n    </tr>\n    <tr>\n      <th>110863</th>\n      <td>where are villi found in the small intestine</td>\n      <td>[(88, 97, the lumen)]</td>\n      <td>Intestinal villi (singular: villus) are small, finger-like projections that extend into the lumen of the small intestine. Each villus is approximately 0.5–1.6 mm in length (in humans), and has many microvilli projecting from the enterocytes of its epithelium which collectively form the striated or brush border. Each of these microvilli are much smaller than a single villus. The intestinal villi are much smaller than any of the circular folds in the intestine. \\n</td>\n    </tr>\n    <tr>\n      <th>110864</th>\n      <td>who bats in the top of the inning</td>\n      <td>[(239, 256, The visiting team)]</td>\n      <td>Each half-inning formally starts when the umpire calls \"Play\" or \"Play ball\". A full inning consists of six outs, three for each team; and, in Major League Baseball and most other adult leagues, a regulation game consists of nine innings. The visiting team bats in the first half-inning, the top of the inning, derived from the position of the visiting team at the top line of a baseball line score. The home team's half of an inning is the bottom of the inning, and the break between halves of a...</td>\n    </tr>\n  </tbody>\n</table>\n<p>110865 rows × 3 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":["**Validation Dataset**"],"metadata":{"id":"YnNmSit1RoDl"}},{"cell_type":"code","source":["validation_dataset = squad_load_from_json(\"../input/nq-squad-format/nq_dev.json\")\n","validation_dataset"],"metadata":{"id":"UHM-6PXxRr3t","outputId":"0a38841c-33b9-41ef-fff5-356b67a35be8","executionInfo":{"status":"ok","timestamp":1646593734549,"user_tz":-120,"elapsed":252,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:33:47.661239Z","iopub.execute_input":"2022-03-09T11:33:47.661545Z","iopub.status.idle":"2022-03-09T11:33:47.742005Z","shell.execute_reply.started":"2022-03-09T11:33:47.661514Z","shell.execute_reply":"2022-03-09T11:33:47.741323Z"},"trusted":true},"execution_count":null,"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"                                                             question  \\\n0                       who is the owner of the mandalay bay in vegas   \n1                   who kicks the ball first to start a football game   \n2                               mount everest how did it get its name   \n3                              who votes in the baseball hall of fame   \n4                         who played taylor on the bold and beautiful   \n...                                                               ...   \n3364                         where are red blood cells made in adults   \n3365  who was the pinkerton detective agency's first female detective   \n3366                     how many episodes are there in modern family   \n3367                  who built the first temple for god in jerusalem   \n3368                         what is a dropped pin on google maps for   \n\n                                                                                                                                                                                                      answer  \\\n0                                                                                                                                                                    [(124, 149, MGM Resorts International)]   \n1                                                                                  [(138, 179, the team that lost the pre-game coin toss), (127, 179, awarded to the team that lost the pre-game coin toss)]   \n2                                                                                                                                                                                                         []   \n3     [(80, 162, the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee), (73, 163, either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,)]   \n4                                                                                                                                                                                  [(112, 123, Hunter Tylo)]   \n...                                                                                                                                                                                                      ...   \n3364                                                                                                                                                                        [(357, 375, in the bone marrow)]   \n3365                                                                                                                                                                                   [(0, 10, Kate Warne)]   \n3366                                                                                                                                  [(18, 21, 201), (0, 44, As of January 17, 2018,[update] 201 episodes)]   \n3367                                                                                                                 [(62, 117, Solomon, king of the United Kingdom of Israel and Judah), (62, 69, Solomon)]   \n3368                                                                                                                                   [(58, 88, marks locations in Google Maps), (58, 73, marks locations)]   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   corpus  \n0                                                                                                                                                                                                 Mandalay Bay is a 43-story luxury resort and casino on the Las Vegas Strip in Paradise, Nevada. It is owned and operated by MGM Resorts International. One of the property's towers operates as the Delano; the Four Seasons Hotel is independently operated within the Mandalay Bay tower, occupying 5 floors (35–39).  \n1                                           A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.  \n2                                                                                                                                                            In 1865, Everest was given its official English name by the Royal Geographical Society, upon a recommendation by Andrew Waugh, the British Surveyor General of India. As there appeared to be several different local names, Waugh chose to name the mountain after his predecessor in the post, Sir George Everest, despite George Everest's objections.[6]  \n3     Players are currently inducted into the Hall of Fame through election by either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,[8] which now consists of four subcommittees, each of which considers and votes for candidates from a separate era of baseball. Five years after retirement, any player with 10 years of major league experience who passes a screening committee (which removes from consideration players of clearly lesser qualification) is eligible to b...  \n4     Taylor Hayes is a fictional character from the American CBS soap opera The Bold and the Beautiful, portrayed by Hunter Tylo. The character was created by William J. Bell and debuted during the episode dated June 6, 1990. Tylo appeared as a regular continuously until 1994 when she took a hiatus for a few months before being written back into the series. In 1996, she left the serial after being cast on Melrose Place, where she was soon fired on the grounds of being pregnant, and returned short...  \n...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   ...  \n3364  In humans, mature red blood cells are flexible and oval biconcave disks. They lack a cell nucleus and most organelles, in order to accommodate maximum space for hemoglobin; they can be viewed as sacks of hemoglobin, with a plasma membrane as the sack. Approximately 2.4 million new erythrocytes are produced per second in human adults.[2] The cells develop in the bone marrow and circulate for about 100–120 days in the body before their components are recycled by macrophages. Each circulation t...  \n3365                                                                                                                                                                                                                                                                                                                                                                            Kate Warne (1833 – January 28, 1868)[1] was the first female detective, in 1856, in the Pinkerton Detective Agency and the United States.  \n3366                                                                                                                                                                                                                                                                                                                                                                                                                                            As of January 17, 2018,[update] 201 episodes of Modern Family have aired.  \n3367  The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...  \n3368                                                                                                                                                                                                                                  The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.  \n\n[3369 rows x 3 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>question</th>\n      <th>answer</th>\n      <th>corpus</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>who is the owner of the mandalay bay in vegas</td>\n      <td>[(124, 149, MGM Resorts International)]</td>\n      <td>Mandalay Bay is a 43-story luxury resort and casino on the Las Vegas Strip in Paradise, Nevada. It is owned and operated by MGM Resorts International. One of the property's towers operates as the Delano; the Four Seasons Hotel is independently operated within the Mandalay Bay tower, occupying 5 floors (35–39).</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>who kicks the ball first to start a football game</td>\n      <td>[(138, 179, the team that lost the pre-game coin toss), (127, 179, awarded to the team that lost the pre-game coin toss)]</td>\n      <td>A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>mount everest how did it get its name</td>\n      <td>[]</td>\n      <td>In 1865, Everest was given its official English name by the Royal Geographical Society, upon a recommendation by Andrew Waugh, the British Surveyor General of India. As there appeared to be several different local names, Waugh chose to name the mountain after his predecessor in the post, Sir George Everest, despite George Everest's objections.[6]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>who votes in the baseball hall of fame</td>\n      <td>[(80, 162, the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee), (73, 163, either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,)]</td>\n      <td>Players are currently inducted into the Hall of Fame through election by either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,[8] which now consists of four subcommittees, each of which considers and votes for candidates from a separate era of baseball. Five years after retirement, any player with 10 years of major league experience who passes a screening committee (which removes from consideration players of clearly lesser qualification) is eligible to b...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>who played taylor on the bold and beautiful</td>\n      <td>[(112, 123, Hunter Tylo)]</td>\n      <td>Taylor Hayes is a fictional character from the American CBS soap opera The Bold and the Beautiful, portrayed by Hunter Tylo. The character was created by William J. Bell and debuted during the episode dated June 6, 1990. Tylo appeared as a regular continuously until 1994 when she took a hiatus for a few months before being written back into the series. In 1996, she left the serial after being cast on Melrose Place, where she was soon fired on the grounds of being pregnant, and returned short...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>3364</th>\n      <td>where are red blood cells made in adults</td>\n      <td>[(357, 375, in the bone marrow)]</td>\n      <td>In humans, mature red blood cells are flexible and oval biconcave disks. They lack a cell nucleus and most organelles, in order to accommodate maximum space for hemoglobin; they can be viewed as sacks of hemoglobin, with a plasma membrane as the sack. Approximately 2.4 million new erythrocytes are produced per second in human adults.[2] The cells develop in the bone marrow and circulate for about 100–120 days in the body before their components are recycled by macrophages. Each circulation t...</td>\n    </tr>\n    <tr>\n      <th>3365</th>\n      <td>who was the pinkerton detective agency's first female detective</td>\n      <td>[(0, 10, Kate Warne)]</td>\n      <td>Kate Warne (1833 – January 28, 1868)[1] was the first female detective, in 1856, in the Pinkerton Detective Agency and the United States.</td>\n    </tr>\n    <tr>\n      <th>3366</th>\n      <td>how many episodes are there in modern family</td>\n      <td>[(18, 21, 201), (0, 44, As of January 17, 2018,[update] 201 episodes)]</td>\n      <td>As of January 17, 2018,[update] 201 episodes of Modern Family have aired.</td>\n    </tr>\n    <tr>\n      <th>3367</th>\n      <td>who built the first temple for god in jerusalem</td>\n      <td>[(62, 117, Solomon, king of the United Kingdom of Israel and Judah), (62, 69, Solomon)]</td>\n      <td>The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...</td>\n    </tr>\n    <tr>\n      <th>3368</th>\n      <td>what is a dropped pin on google maps for</td>\n      <td>[(58, 88, marks locations in Google Maps), (58, 73, marks locations)]</td>\n      <td>The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.</td>\n    </tr>\n  </tbody>\n</table>\n<p>3369 rows × 3 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":["**Exploding datasets on multiple answers**\n","\n","Now, we will explode the datasets on the answers list field because it will help later."],"metadata":{"id":"ViYC5HkjYtEA"}},{"cell_type":"code","source":["train_dataset = train_dataset.explode('answer').reset_index()\n","train_dataset"],"metadata":{"id":"nvJXORnjY-Ji","executionInfo":{"status":"ok","timestamp":1646593743068,"user_tz":-120,"elapsed":246,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"outputId":"b9463689-b098-402e-e6b2-4700b277e394","execution":{"iopub.status.busy":"2022-03-09T11:33:55.140503Z","iopub.execute_input":"2022-03-09T11:33:55.141334Z","iopub.status.idle":"2022-03-09T11:33:55.269629Z","shell.execute_reply.started":"2022-03-09T11:33:55.141292Z","shell.execute_reply":"2022-03-09T11:33:55.268813Z"},"trusted":true},"execution_count":null,"outputs":[{"execution_count":17,"output_type":"execute_result","data":{"text/plain":"         index                                           question  \\\n0            0                      who sings it's going to be me   \n1            1              when does the clown movie it come out   \n2            2                 who sang the song come fly with me   \n3            3      what is the brightest star in corona borealis   \n4            4       who owns the land under this navigable river   \n...        ...                                                ...   \n110860  110860            who wrote the music for game of thrones   \n110861  110861  where did the tradition of santa clause come from   \n110862  110862      what is the population of amherst nova scotia   \n110863  110863       where are villi found in the small intestine   \n110864  110864                  who bats in the top of the inning   \n\n                                                                                                                                                                                                                                            answer  \\\n0                                                                                                                                                                                                                                  (50, 55, NSYNC)   \n1                                                                                                                                                                                                                                              NaN   \n2                                                                                                                                                                                                                  (0, 22, Francis Albert Sinatra)   \n3                                                                                                                                                                                               (22, 62, the magnitude 2.2 Alpha Coronae Borealis)   \n4                                                                                                                                                                                                                                (188, 193, state)   \n...                                                                                                                                                                                                                                            ...   \n110860                                                                                                                                                                                                                     (67, 80, Ramin Djawadi)   \n110861  (368, 590, traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure of Father Christmas and the Dutch figure of Sinterklaas (himself also based on Saint Nicholas))   \n110862                                                                                                                                                                                                                           (110, 115, 9,413)   \n110863                                                                                                                                                                                                                         (88, 97, the lumen)   \n110864                                                                                                                                                                                                               (239, 256, The visiting team)   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     corpus  \n0               \"It's Gonna Be Me\" is a song by American boy band NSYNC. It was released on June 13, 2000 as the second single in the United States and third in Europe from their second studio album No Strings Attached. The song was NSYNC's only Billboard Hot 100 number-one hit, making it their highest-charting single. It topped the chart for two consecutive weeks and has been certified Gold by the RIAA. The song serves as the opening track of the 2000 compilation album Now That's What I Call Music! 5.  \n1                    It was released in North America on September 8, 2017.[197] In Europe, the film was released in Belgium on September 6, 2017, Denmark and the Netherlands on September 7, 2017,[198][199] and Norway and Finland on September 8, 2017.[200] On March 7, 2017, the alternate title of the film was announced by Stephen King as Part 1 – The Losers’ Club.[201][202] In addition to the conventional 2D format, It was also released across 615 IMAX screens globally, including 389 domestically.[203]  \n2       Francis Albert Sinatra (/sɪˈnɑːtrə/; Italian: [siˈnaːtra]; December 12, 1915 – May 14, 1998) was an American singer, actor, and producer who was one of the most popular and influential musical artists of the 20th century. He is one of the best-selling music artists of all time, having sold more than 150 million records worldwide.[2] Born in Hoboken, New Jersey, to Italian immigrants, Sinatra began his musical career in the swing era with bandleaders Harry James and Tommy Dorsey. Sinatra foun...  \n3       The brightest star is the magnitude 2.2 Alpha Coronae Borealis. The yellow supergiant R Coronae Borealis is the prototype of a rare class of giant stars—the R Coronae Borealis variables—that are extremely hydrogen deficient, and thought to result from the merger of two white dwarfs. T Coronae Borealis, also known as the Blaze Star, is another unusual type of variable star known as a recurrent nova. Normally of magnitude 10, it last flared up to magnitude 2 in 1946. ADS 9731 and Sigma Coronae...  \n4       In determining the contours of riparian rights, there is a clear distinction between navigable (public) waters and non-navigable waters. The land below navigable waters is the property of state,[3] and subject to all the public land laws and in most states public trust rights. Navigable waters are treated as public highways with any exclusive riparian right ending at the ordinary high water mark. Like a road, any riparian right is subordinate to the public's right to travel on the river, but...  \n...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     ...  \n110860                                                                                                          The music for the fantasy TV series Game of Thrones is composed by Ramin Djawadi. The music is primarily instrumental with the occasional vocal performances, and is created to support musically the characters and plots of the show. It features various theme, the most prominent is its Main Title that accompanies the series' title sequence. In every season, a soundtrack album would be released.  \n110861  Santa Claus, also known as Saint Nicholas, Kris Kringle, Father Christmas, or simply Santa, is a legendary figure originating in Western Christian culture who is said to bring gifts to the homes of well-behaved (\"good\" or \"nice\") children on Christmas Eve (24 December) and the early morning hours of Christmas Day (25 December).[1] The modern Santa Claus grew out of traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure ...  \n110862                                                                                                                                                  In the 2016 Census of Population conducted by Statistics Canada, the Town of Amherst recorded a population of 9,413 living in 4,372 of its 4,745 total private dwellings, a change of 2999690000000000000♠−3.1% from its 2011 population of 9,717. With a land area of 12.07 km2 (4.66 sq mi), it had a population density of 779.9/km2 (2,019.8/sq mi) in 2016.[1]  \n110863                                   Intestinal villi (singular: villus) are small, finger-like projections that extend into the lumen of the small intestine. Each villus is approximately 0.5–1.6 mm in length (in humans), and has many microvilli projecting from the enterocytes of its epithelium which collectively form the striated or brush border. Each of these microvilli are much smaller than a single villus. The intestinal villi are much smaller than any of the circular folds in the intestine. \\n  \n110864  Each half-inning formally starts when the umpire calls \"Play\" or \"Play ball\". A full inning consists of six outs, three for each team; and, in Major League Baseball and most other adult leagues, a regulation game consists of nine innings. The visiting team bats in the first half-inning, the top of the inning, derived from the position of the visiting team at the top line of a baseball line score. The home team's half of an inning is the bottom of the inning, and the break between halves of a...  \n\n[110865 rows x 4 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>question</th>\n      <th>answer</th>\n      <th>corpus</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>who sings it's going to be me</td>\n      <td>(50, 55, NSYNC)</td>\n      <td>\"It's Gonna Be Me\" is a song by American boy band NSYNC. It was released on June 13, 2000 as the second single in the United States and third in Europe from their second studio album No Strings Attached. The song was NSYNC's only Billboard Hot 100 number-one hit, making it their highest-charting single. It topped the chart for two consecutive weeks and has been certified Gold by the RIAA. The song serves as the opening track of the 2000 compilation album Now That's What I Call Music! 5.</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>when does the clown movie it come out</td>\n      <td>NaN</td>\n      <td>It was released in North America on September 8, 2017.[197] In Europe, the film was released in Belgium on September 6, 2017, Denmark and the Netherlands on September 7, 2017,[198][199] and Norway and Finland on September 8, 2017.[200] On March 7, 2017, the alternate title of the film was announced by Stephen King as Part 1 – The Losers’ Club.[201][202] In addition to the conventional 2D format, It was also released across 615 IMAX screens globally, including 389 domestically.[203]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>who sang the song come fly with me</td>\n      <td>(0, 22, Francis Albert Sinatra)</td>\n      <td>Francis Albert Sinatra (/sɪˈnɑːtrə/; Italian: [siˈnaːtra]; December 12, 1915 – May 14, 1998) was an American singer, actor, and producer who was one of the most popular and influential musical artists of the 20th century. He is one of the best-selling music artists of all time, having sold more than 150 million records worldwide.[2] Born in Hoboken, New Jersey, to Italian immigrants, Sinatra began his musical career in the swing era with bandleaders Harry James and Tommy Dorsey. Sinatra foun...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>what is the brightest star in corona borealis</td>\n      <td>(22, 62, the magnitude 2.2 Alpha Coronae Borealis)</td>\n      <td>The brightest star is the magnitude 2.2 Alpha Coronae Borealis. The yellow supergiant R Coronae Borealis is the prototype of a rare class of giant stars—the R Coronae Borealis variables—that are extremely hydrogen deficient, and thought to result from the merger of two white dwarfs. T Coronae Borealis, also known as the Blaze Star, is another unusual type of variable star known as a recurrent nova. Normally of magnitude 10, it last flared up to magnitude 2 in 1946. ADS 9731 and Sigma Coronae...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>who owns the land under this navigable river</td>\n      <td>(188, 193, state)</td>\n      <td>In determining the contours of riparian rights, there is a clear distinction between navigable (public) waters and non-navigable waters. The land below navigable waters is the property of state,[3] and subject to all the public land laws and in most states public trust rights. Navigable waters are treated as public highways with any exclusive riparian right ending at the ordinary high water mark. Like a road, any riparian right is subordinate to the public's right to travel on the river, but...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>110860</th>\n      <td>110860</td>\n      <td>who wrote the music for game of thrones</td>\n      <td>(67, 80, Ramin Djawadi)</td>\n      <td>The music for the fantasy TV series Game of Thrones is composed by Ramin Djawadi. The music is primarily instrumental with the occasional vocal performances, and is created to support musically the characters and plots of the show. It features various theme, the most prominent is its Main Title that accompanies the series' title sequence. In every season, a soundtrack album would be released.</td>\n    </tr>\n    <tr>\n      <th>110861</th>\n      <td>110861</td>\n      <td>where did the tradition of santa clause come from</td>\n      <td>(368, 590, traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure of Father Christmas and the Dutch figure of Sinterklaas (himself also based on Saint Nicholas))</td>\n      <td>Santa Claus, also known as Saint Nicholas, Kris Kringle, Father Christmas, or simply Santa, is a legendary figure originating in Western Christian culture who is said to bring gifts to the homes of well-behaved (\"good\" or \"nice\") children on Christmas Eve (24 December) and the early morning hours of Christmas Day (25 December).[1] The modern Santa Claus grew out of traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure ...</td>\n    </tr>\n    <tr>\n      <th>110862</th>\n      <td>110862</td>\n      <td>what is the population of amherst nova scotia</td>\n      <td>(110, 115, 9,413)</td>\n      <td>In the 2016 Census of Population conducted by Statistics Canada, the Town of Amherst recorded a population of 9,413 living in 4,372 of its 4,745 total private dwellings, a change of 2999690000000000000♠−3.1% from its 2011 population of 9,717. With a land area of 12.07 km2 (4.66 sq mi), it had a population density of 779.9/km2 (2,019.8/sq mi) in 2016.[1]</td>\n    </tr>\n    <tr>\n      <th>110863</th>\n      <td>110863</td>\n      <td>where are villi found in the small intestine</td>\n      <td>(88, 97, the lumen)</td>\n      <td>Intestinal villi (singular: villus) are small, finger-like projections that extend into the lumen of the small intestine. Each villus is approximately 0.5–1.6 mm in length (in humans), and has many microvilli projecting from the enterocytes of its epithelium which collectively form the striated or brush border. Each of these microvilli are much smaller than a single villus. The intestinal villi are much smaller than any of the circular folds in the intestine. \\n</td>\n    </tr>\n    <tr>\n      <th>110864</th>\n      <td>110864</td>\n      <td>who bats in the top of the inning</td>\n      <td>(239, 256, The visiting team)</td>\n      <td>Each half-inning formally starts when the umpire calls \"Play\" or \"Play ball\". A full inning consists of six outs, three for each team; and, in Major League Baseball and most other adult leagues, a regulation game consists of nine innings. The visiting team bats in the first half-inning, the top of the inning, derived from the position of the visiting team at the top line of a baseball line score. The home team's half of an inning is the bottom of the inning, and the break between halves of a...</td>\n    </tr>\n  </tbody>\n</table>\n<p>110865 rows × 4 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":["validation_dataset = validation_dataset.explode('answer').reset_index()\n","validation_dataset"],"metadata":{"id":"8XBWvjFZYkth","executionInfo":{"status":"ok","timestamp":1646593745568,"user_tz":-120,"elapsed":238,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"outputId":"8bc24a53-1da2-41b9-fd09-901ae1bdc58f","execution":{"iopub.status.busy":"2022-03-09T11:33:58.240638Z","iopub.execute_input":"2022-03-09T11:33:58.241276Z","iopub.status.idle":"2022-03-09T11:33:58.272092Z","shell.execute_reply.started":"2022-03-09T11:33:58.241217Z","shell.execute_reply":"2022-03-09T11:33:58.271325Z"},"trusted":true},"execution_count":null,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"      index                                           question  \\\n0         0      who is the owner of the mandalay bay in vegas   \n1         1  who kicks the ball first to start a football game   \n2         1  who kicks the ball first to start a football game   \n3         2              mount everest how did it get its name   \n4         3             who votes in the baseball hall of fame   \n...     ...                                                ...   \n4664   3366       how many episodes are there in modern family   \n4665   3367    who built the first temple for god in jerusalem   \n4666   3367    who built the first temple for god in jerusalem   \n4667   3368           what is a dropped pin on google maps for   \n4668   3368           what is a dropped pin on google maps for   \n\n                                                                                             answer  \\\n0                                                             (124, 149, MGM Resorts International)   \n1                                             (138, 179, the team that lost the pre-game coin toss)   \n2                                  (127, 179, awarded to the team that lost the pre-game coin toss)   \n3                                                                                               NaN   \n4     (80, 162, the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee)   \n...                                                                                             ...   \n4664                                          (0, 44, As of January 17, 2018,[update] 201 episodes)   \n4665                             (62, 117, Solomon, king of the United Kingdom of Israel and Judah)   \n4666                                                                              (62, 69, Solomon)   \n4667                                                       (58, 88, marks locations in Google Maps)   \n4668                                                                      (58, 73, marks locations)   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   corpus  \n0                                                                                                                                                                                                 Mandalay Bay is a 43-story luxury resort and casino on the Las Vegas Strip in Paradise, Nevada. It is owned and operated by MGM Resorts International. One of the property's towers operates as the Delano; the Four Seasons Hotel is independently operated within the Mandalay Bay tower, occupying 5 floors (35–39).  \n1                                           A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.  \n2                                           A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.  \n3                                                                                                                                                            In 1865, Everest was given its official English name by the Royal Geographical Society, upon a recommendation by Andrew Waugh, the British Surveyor General of India. As there appeared to be several different local names, Waugh chose to name the mountain after his predecessor in the post, Sir George Everest, despite George Everest's objections.[6]  \n4     Players are currently inducted into the Hall of Fame through election by either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,[8] which now consists of four subcommittees, each of which considers and votes for candidates from a separate era of baseball. Five years after retirement, any player with 10 years of major league experience who passes a screening committee (which removes from consideration players of clearly lesser qualification) is eligible to b...  \n...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   ...  \n4664                                                                                                                                                                                                                                                                                                                                                                                                                                            As of January 17, 2018,[update] 201 episodes of Modern Family have aired.  \n4665  The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...  \n4666  The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...  \n4667                                                                                                                                                                                                                                  The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.  \n4668                                                                                                                                                                                                                                  The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.  \n\n[4669 rows x 4 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>question</th>\n      <th>answer</th>\n      <th>corpus</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>who is the owner of the mandalay bay in vegas</td>\n      <td>(124, 149, MGM Resorts International)</td>\n      <td>Mandalay Bay is a 43-story luxury resort and casino on the Las Vegas Strip in Paradise, Nevada. It is owned and operated by MGM Resorts International. One of the property's towers operates as the Delano; the Four Seasons Hotel is independently operated within the Mandalay Bay tower, occupying 5 floors (35–39).</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>who kicks the ball first to start a football game</td>\n      <td>(138, 179, the team that lost the pre-game coin toss)</td>\n      <td>A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>who kicks the ball first to start a football game</td>\n      <td>(127, 179, awarded to the team that lost the pre-game coin toss)</td>\n      <td>A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2</td>\n      <td>mount everest how did it get its name</td>\n      <td>NaN</td>\n      <td>In 1865, Everest was given its official English name by the Royal Geographical Society, upon a recommendation by Andrew Waugh, the British Surveyor General of India. As there appeared to be several different local names, Waugh chose to name the mountain after his predecessor in the post, Sir George Everest, despite George Everest's objections.[6]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>3</td>\n      <td>who votes in the baseball hall of fame</td>\n      <td>(80, 162, the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee)</td>\n      <td>Players are currently inducted into the Hall of Fame through election by either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,[8] which now consists of four subcommittees, each of which considers and votes for candidates from a separate era of baseball. Five years after retirement, any player with 10 years of major league experience who passes a screening committee (which removes from consideration players of clearly lesser qualification) is eligible to b...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4664</th>\n      <td>3366</td>\n      <td>how many episodes are there in modern family</td>\n      <td>(0, 44, As of January 17, 2018,[update] 201 episodes)</td>\n      <td>As of January 17, 2018,[update] 201 episodes of Modern Family have aired.</td>\n    </tr>\n    <tr>\n      <th>4665</th>\n      <td>3367</td>\n      <td>who built the first temple for god in jerusalem</td>\n      <td>(62, 117, Solomon, king of the United Kingdom of Israel and Judah)</td>\n      <td>The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...</td>\n    </tr>\n    <tr>\n      <th>4666</th>\n      <td>3367</td>\n      <td>who built the first temple for god in jerusalem</td>\n      <td>(62, 69, Solomon)</td>\n      <td>The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...</td>\n    </tr>\n    <tr>\n      <th>4667</th>\n      <td>3368</td>\n      <td>what is a dropped pin on google maps for</td>\n      <td>(58, 88, marks locations in Google Maps)</td>\n      <td>The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.</td>\n    </tr>\n    <tr>\n      <th>4668</th>\n      <td>3368</td>\n      <td>what is a dropped pin on google maps for</td>\n      <td>(58, 73, marks locations)</td>\n      <td>The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.</td>\n    </tr>\n  </tbody>\n</table>\n<p>4669 rows × 4 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":["**Download BERT tokenizer**\n","\n","After loading the datasets, before we use them for the evaluation process, we must first tokenize them to calculate the gold answer start and end token positions."],"metadata":{"id":"qx0bT8qWC2W2"}},{"cell_type":"code","source":["tokenizer = BertTokenizerFast.from_pretrained(BERT_MODEL_NAME)"],"metadata":{"id":"uwfHVB8fC2w7","executionInfo":{"status":"ok","timestamp":1646593748992,"user_tz":-120,"elapsed":707,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"outputId":"8349adc2-8c93-4791-9c65-8b9b20bce455","execution":{"iopub.status.busy":"2022-03-09T11:34:00.543536Z","iopub.execute_input":"2022-03-09T11:34:00.543806Z","iopub.status.idle":"2022-03-09T11:34:05.496209Z","shell.execute_reply.started":"2022-03-09T11:34:00.543774Z","shell.execute_reply":"2022-03-09T11:34:05.495448Z"},"trusted":true,"colab":{"referenced_widgets":["629a312082c14cf7a5857bfb647758b2","bd5f127b445b4430880efa2c49bdf8b5","3ac4ff2a3987442a94bf7f1c35896fc3","c7f72cf138664dd48da11f46c0b775d6"]}},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"629a312082c14cf7a5857bfb647758b2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bd5f127b445b4430880efa2c49bdf8b5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3ac4ff2a3987442a94bf7f1c35896fc3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c7f72cf138664dd48da11f46c0b775d6"}},"metadata":{}}]},{"cell_type":"markdown","source":["**Calculate start and end tokens of the answers**"],"metadata":{"id":"EvwlQ6BOh9CS"}},{"cell_type":"code","source":["def calculate_tokenized_ans_indices(dataset: pd.DataFrame):\n","  ans_tok_start = []\n","  ans_tok_end = []\n","  ans_tok_text = []\n","  for idx, ans in enumerate(dataset['answer'].values):\n","    if not pd.isna(ans):\n","      ans_text_start = ans[0]\n","      ans_text_end = ans[1]\n","      ans_text = ans[2]\n","      encoding = tokenizer.encode_plus(text=dataset['corpus'].values[idx], text_pair=dataset['question'].values[idx], max_length=512, padding='max_length', truncation=True)\n","      ans_start = encoding.char_to_token(0, ans_text_start)\n","      ans_end = encoding.char_to_token(0, ans_text_end-1)\n","      # Τruncated answer\n","      if ans_start is None:\n","        ans_start = ans_end = tokenizer.model_max_length\n","      elif ans_end is None:\n","        ans_end = [i for i, inp in enumerate(encoding['input_ids']) if inp == tokenizer.sep_token_id][0]\n","      # else:\n","      ans_text_tok = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(encoding['input_ids'][ans_start:ans_end+1]))\n","      #   if (normalize_answer(ans_text) != normalize_answer(ans_tok_text)):\n","      #     print(idx, normalize_answer(ans_text), \"|||||\", normalize_answer(ans_tok_text))\n","    else:\n","      ans_start = ans_end = tokenizer.model_max_length\n","      ans_text_tok = \"\"\n","    ans_tok_start.append(ans_start)\n","    ans_tok_end.append(ans_end)\n","    ans_tok_text.append(ans_text_tok)\n","  dataset['ans_start_tok'] = ans_tok_start\n","  dataset['ans_end_tok'] = ans_tok_end\n","  dataset['ans_tok_text'] = ans_tok_text\n","  return dataset"],"metadata":{"id":"gBuRHPaUiFp-","execution":{"iopub.status.busy":"2022-03-09T11:34:12.379619Z","iopub.execute_input":"2022-03-09T11:34:12.380392Z","iopub.status.idle":"2022-03-09T11:34:12.390319Z","shell.execute_reply.started":"2022-03-09T11:34:12.380342Z","shell.execute_reply":"2022-03-09T11:34:12.389500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_dataset = calculate_tokenized_ans_indices(train_dataset)\n","train_dataset"],"metadata":{"id":"n_JjLTfglsGv","outputId":"900136de-0c37-47f8-e082-66dbfe2995cb","executionInfo":{"status":"ok","timestamp":1646593824144,"user_tz":-120,"elapsed":68894,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:34:25.105009Z","iopub.execute_input":"2022-03-09T11:34:25.105752Z","iopub.status.idle":"2022-03-09T11:35:17.352830Z","shell.execute_reply.started":"2022-03-09T11:34:25.105715Z","shell.execute_reply":"2022-03-09T11:35:17.351992Z"},"trusted":true},"execution_count":null,"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"         index                                           question  \\\n0            0                      who sings it's going to be me   \n1            1              when does the clown movie it come out   \n2            2                 who sang the song come fly with me   \n3            3      what is the brightest star in corona borealis   \n4            4       who owns the land under this navigable river   \n...        ...                                                ...   \n110860  110860            who wrote the music for game of thrones   \n110861  110861  where did the tradition of santa clause come from   \n110862  110862      what is the population of amherst nova scotia   \n110863  110863       where are villi found in the small intestine   \n110864  110864                  who bats in the top of the inning   \n\n                                                                                                                                                                                                                                            answer  \\\n0                                                                                                                                                                                                                                  (50, 55, NSYNC)   \n1                                                                                                                                                                                                                                              NaN   \n2                                                                                                                                                                                                                  (0, 22, Francis Albert Sinatra)   \n3                                                                                                                                                                                               (22, 62, the magnitude 2.2 Alpha Coronae Borealis)   \n4                                                                                                                                                                                                                                (188, 193, state)   \n...                                                                                                                                                                                                                                            ...   \n110860                                                                                                                                                                                                                     (67, 80, Ramin Djawadi)   \n110861  (368, 590, traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure of Father Christmas and the Dutch figure of Sinterklaas (himself also based on Saint Nicholas))   \n110862                                                                                                                                                                                                                           (110, 115, 9,413)   \n110863                                                                                                                                                                                                                         (88, 97, the lumen)   \n110864                                                                                                                                                                                                               (239, 256, The visiting team)   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     corpus  \\\n0               \"It's Gonna Be Me\" is a song by American boy band NSYNC. It was released on June 13, 2000 as the second single in the United States and third in Europe from their second studio album No Strings Attached. The song was NSYNC's only Billboard Hot 100 number-one hit, making it their highest-charting single. It topped the chart for two consecutive weeks and has been certified Gold by the RIAA. The song serves as the opening track of the 2000 compilation album Now That's What I Call Music! 5.   \n1                    It was released in North America on September 8, 2017.[197] In Europe, the film was released in Belgium on September 6, 2017, Denmark and the Netherlands on September 7, 2017,[198][199] and Norway and Finland on September 8, 2017.[200] On March 7, 2017, the alternate title of the film was announced by Stephen King as Part 1 – The Losers’ Club.[201][202] In addition to the conventional 2D format, It was also released across 615 IMAX screens globally, including 389 domestically.[203]   \n2       Francis Albert Sinatra (/sɪˈnɑːtrə/; Italian: [siˈnaːtra]; December 12, 1915 – May 14, 1998) was an American singer, actor, and producer who was one of the most popular and influential musical artists of the 20th century. He is one of the best-selling music artists of all time, having sold more than 150 million records worldwide.[2] Born in Hoboken, New Jersey, to Italian immigrants, Sinatra began his musical career in the swing era with bandleaders Harry James and Tommy Dorsey. Sinatra foun...   \n3       The brightest star is the magnitude 2.2 Alpha Coronae Borealis. The yellow supergiant R Coronae Borealis is the prototype of a rare class of giant stars—the R Coronae Borealis variables—that are extremely hydrogen deficient, and thought to result from the merger of two white dwarfs. T Coronae Borealis, also known as the Blaze Star, is another unusual type of variable star known as a recurrent nova. Normally of magnitude 10, it last flared up to magnitude 2 in 1946. ADS 9731 and Sigma Coronae...   \n4       In determining the contours of riparian rights, there is a clear distinction between navigable (public) waters and non-navigable waters. The land below navigable waters is the property of state,[3] and subject to all the public land laws and in most states public trust rights. Navigable waters are treated as public highways with any exclusive riparian right ending at the ordinary high water mark. Like a road, any riparian right is subordinate to the public's right to travel on the river, but...   \n...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     ...   \n110860                                                                                                          The music for the fantasy TV series Game of Thrones is composed by Ramin Djawadi. The music is primarily instrumental with the occasional vocal performances, and is created to support musically the characters and plots of the show. It features various theme, the most prominent is its Main Title that accompanies the series' title sequence. In every season, a soundtrack album would be released.   \n110861  Santa Claus, also known as Saint Nicholas, Kris Kringle, Father Christmas, or simply Santa, is a legendary figure originating in Western Christian culture who is said to bring gifts to the homes of well-behaved (\"good\" or \"nice\") children on Christmas Eve (24 December) and the early morning hours of Christmas Day (25 December).[1] The modern Santa Claus grew out of traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure ...   \n110862                                                                                                                                                  In the 2016 Census of Population conducted by Statistics Canada, the Town of Amherst recorded a population of 9,413 living in 4,372 of its 4,745 total private dwellings, a change of 2999690000000000000♠−3.1% from its 2011 population of 9,717. With a land area of 12.07 km2 (4.66 sq mi), it had a population density of 779.9/km2 (2,019.8/sq mi) in 2016.[1]   \n110863                                   Intestinal villi (singular: villus) are small, finger-like projections that extend into the lumen of the small intestine. Each villus is approximately 0.5–1.6 mm in length (in humans), and has many microvilli projecting from the enterocytes of its epithelium which collectively form the striated or brush border. Each of these microvilli are much smaller than a single villus. The intestinal villi are much smaller than any of the circular folds in the intestine. \\n   \n110864  Each half-inning formally starts when the umpire calls \"Play\" or \"Play ball\". A full inning consists of six outs, three for each team; and, in Major League Baseball and most other adult leagues, a regulation game consists of nine innings. The visiting team bats in the first half-inning, the top of the inning, derived from the position of the visiting team at the top line of a baseball line score. The home team's half of an inning is the bottom of the inning, and the break between halves of a...   \n\n        ans_start_tok  ans_end_tok  \\\n0                  16           18   \n1                 512          512   \n2                   1            3   \n3                   5           14   \n4                  38           38   \n...               ...          ...   \n110860             15           19   \n110861             84          128   \n110862             20           23   \n110863             21           23   \n110864             53           55   \n\n                                                                                                                                                                                                                                  ans_tok_text  \n0                                                                                                                                                                                                                                        nsync  \n1                                                                                                                                                                                                                                               \n2                                                                                                                                                                                                                       francis albert sinatra  \n3                                                                                                                                                                                                    the magnitude 2. 2 alpha coronae borealis  \n4                                                                                                                                                                                                                                        state  \n...                                                                                                                                                                                                                                        ...  \n110860                                                                                                                                                                                                                           ramin djawadi  \n110861  traditions surrounding the historical saint nicholas ( a fourth - century greek bishop and gift - giver of myra ), the british figure of father christmas and the dutch figure of sinterklaas ( himself also based on saint nicholas )  \n110862                                                                                                                                                                                                                                  9, 413  \n110863                                                                                                                                                                                                                               the lumen  \n110864                                                                                                                                                                                                                       the visiting team  \n\n[110865 rows x 7 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>question</th>\n      <th>answer</th>\n      <th>corpus</th>\n      <th>ans_start_tok</th>\n      <th>ans_end_tok</th>\n      <th>ans_tok_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>who sings it's going to be me</td>\n      <td>(50, 55, NSYNC)</td>\n      <td>\"It's Gonna Be Me\" is a song by American boy band NSYNC. It was released on June 13, 2000 as the second single in the United States and third in Europe from their second studio album No Strings Attached. The song was NSYNC's only Billboard Hot 100 number-one hit, making it their highest-charting single. It topped the chart for two consecutive weeks and has been certified Gold by the RIAA. The song serves as the opening track of the 2000 compilation album Now That's What I Call Music! 5.</td>\n      <td>16</td>\n      <td>18</td>\n      <td>nsync</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>when does the clown movie it come out</td>\n      <td>NaN</td>\n      <td>It was released in North America on September 8, 2017.[197] In Europe, the film was released in Belgium on September 6, 2017, Denmark and the Netherlands on September 7, 2017,[198][199] and Norway and Finland on September 8, 2017.[200] On March 7, 2017, the alternate title of the film was announced by Stephen King as Part 1 – The Losers’ Club.[201][202] In addition to the conventional 2D format, It was also released across 615 IMAX screens globally, including 389 domestically.[203]</td>\n      <td>512</td>\n      <td>512</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>who sang the song come fly with me</td>\n      <td>(0, 22, Francis Albert Sinatra)</td>\n      <td>Francis Albert Sinatra (/sɪˈnɑːtrə/; Italian: [siˈnaːtra]; December 12, 1915 – May 14, 1998) was an American singer, actor, and producer who was one of the most popular and influential musical artists of the 20th century. He is one of the best-selling music artists of all time, having sold more than 150 million records worldwide.[2] Born in Hoboken, New Jersey, to Italian immigrants, Sinatra began his musical career in the swing era with bandleaders Harry James and Tommy Dorsey. Sinatra foun...</td>\n      <td>1</td>\n      <td>3</td>\n      <td>francis albert sinatra</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>what is the brightest star in corona borealis</td>\n      <td>(22, 62, the magnitude 2.2 Alpha Coronae Borealis)</td>\n      <td>The brightest star is the magnitude 2.2 Alpha Coronae Borealis. The yellow supergiant R Coronae Borealis is the prototype of a rare class of giant stars—the R Coronae Borealis variables—that are extremely hydrogen deficient, and thought to result from the merger of two white dwarfs. T Coronae Borealis, also known as the Blaze Star, is another unusual type of variable star known as a recurrent nova. Normally of magnitude 10, it last flared up to magnitude 2 in 1946. ADS 9731 and Sigma Coronae...</td>\n      <td>5</td>\n      <td>14</td>\n      <td>the magnitude 2. 2 alpha coronae borealis</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>who owns the land under this navigable river</td>\n      <td>(188, 193, state)</td>\n      <td>In determining the contours of riparian rights, there is a clear distinction between navigable (public) waters and non-navigable waters. The land below navigable waters is the property of state,[3] and subject to all the public land laws and in most states public trust rights. Navigable waters are treated as public highways with any exclusive riparian right ending at the ordinary high water mark. Like a road, any riparian right is subordinate to the public's right to travel on the river, but...</td>\n      <td>38</td>\n      <td>38</td>\n      <td>state</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>110860</th>\n      <td>110860</td>\n      <td>who wrote the music for game of thrones</td>\n      <td>(67, 80, Ramin Djawadi)</td>\n      <td>The music for the fantasy TV series Game of Thrones is composed by Ramin Djawadi. The music is primarily instrumental with the occasional vocal performances, and is created to support musically the characters and plots of the show. It features various theme, the most prominent is its Main Title that accompanies the series' title sequence. In every season, a soundtrack album would be released.</td>\n      <td>15</td>\n      <td>19</td>\n      <td>ramin djawadi</td>\n    </tr>\n    <tr>\n      <th>110861</th>\n      <td>110861</td>\n      <td>where did the tradition of santa clause come from</td>\n      <td>(368, 590, traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure of Father Christmas and the Dutch figure of Sinterklaas (himself also based on Saint Nicholas))</td>\n      <td>Santa Claus, also known as Saint Nicholas, Kris Kringle, Father Christmas, or simply Santa, is a legendary figure originating in Western Christian culture who is said to bring gifts to the homes of well-behaved (\"good\" or \"nice\") children on Christmas Eve (24 December) and the early morning hours of Christmas Day (25 December).[1] The modern Santa Claus grew out of traditions surrounding the historical Saint Nicholas (a fourth-century Greek bishop and gift-giver of Myra), the British figure ...</td>\n      <td>84</td>\n      <td>128</td>\n      <td>traditions surrounding the historical saint nicholas ( a fourth - century greek bishop and gift - giver of myra ), the british figure of father christmas and the dutch figure of sinterklaas ( himself also based on saint nicholas )</td>\n    </tr>\n    <tr>\n      <th>110862</th>\n      <td>110862</td>\n      <td>what is the population of amherst nova scotia</td>\n      <td>(110, 115, 9,413)</td>\n      <td>In the 2016 Census of Population conducted by Statistics Canada, the Town of Amherst recorded a population of 9,413 living in 4,372 of its 4,745 total private dwellings, a change of 2999690000000000000♠−3.1% from its 2011 population of 9,717. With a land area of 12.07 km2 (4.66 sq mi), it had a population density of 779.9/km2 (2,019.8/sq mi) in 2016.[1]</td>\n      <td>20</td>\n      <td>23</td>\n      <td>9, 413</td>\n    </tr>\n    <tr>\n      <th>110863</th>\n      <td>110863</td>\n      <td>where are villi found in the small intestine</td>\n      <td>(88, 97, the lumen)</td>\n      <td>Intestinal villi (singular: villus) are small, finger-like projections that extend into the lumen of the small intestine. Each villus is approximately 0.5–1.6 mm in length (in humans), and has many microvilli projecting from the enterocytes of its epithelium which collectively form the striated or brush border. Each of these microvilli are much smaller than a single villus. The intestinal villi are much smaller than any of the circular folds in the intestine. \\n</td>\n      <td>21</td>\n      <td>23</td>\n      <td>the lumen</td>\n    </tr>\n    <tr>\n      <th>110864</th>\n      <td>110864</td>\n      <td>who bats in the top of the inning</td>\n      <td>(239, 256, The visiting team)</td>\n      <td>Each half-inning formally starts when the umpire calls \"Play\" or \"Play ball\". A full inning consists of six outs, three for each team; and, in Major League Baseball and most other adult leagues, a regulation game consists of nine innings. The visiting team bats in the first half-inning, the top of the inning, derived from the position of the visiting team at the top line of a baseball line score. The home team's half of an inning is the bottom of the inning, and the break between halves of a...</td>\n      <td>53</td>\n      <td>55</td>\n      <td>the visiting team</td>\n    </tr>\n  </tbody>\n</table>\n<p>110865 rows × 7 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":["validation_dataset = calculate_tokenized_ans_indices(validation_dataset)\n","validation_dataset"],"metadata":{"id":"yfp7_F4y7xc-","outputId":"d5ecc67c-146f-4aa3-8025-649a9f0606a4","executionInfo":{"status":"ok","timestamp":1646593852361,"user_tz":-120,"elapsed":9445,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:35:25.356904Z","iopub.execute_input":"2022-03-09T11:35:25.357531Z","iopub.status.idle":"2022-03-09T11:35:27.937413Z","shell.execute_reply.started":"2022-03-09T11:35:25.357493Z","shell.execute_reply":"2022-03-09T11:35:27.936752Z"},"trusted":true},"execution_count":null,"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"      index                                           question  \\\n0         0      who is the owner of the mandalay bay in vegas   \n1         1  who kicks the ball first to start a football game   \n2         1  who kicks the ball first to start a football game   \n3         2              mount everest how did it get its name   \n4         3             who votes in the baseball hall of fame   \n...     ...                                                ...   \n4664   3366       how many episodes are there in modern family   \n4665   3367    who built the first temple for god in jerusalem   \n4666   3367    who built the first temple for god in jerusalem   \n4667   3368           what is a dropped pin on google maps for   \n4668   3368           what is a dropped pin on google maps for   \n\n                                                                                             answer  \\\n0                                                             (124, 149, MGM Resorts International)   \n1                                             (138, 179, the team that lost the pre-game coin toss)   \n2                                  (127, 179, awarded to the team that lost the pre-game coin toss)   \n3                                                                                               NaN   \n4     (80, 162, the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee)   \n...                                                                                             ...   \n4664                                          (0, 44, As of January 17, 2018,[update] 201 episodes)   \n4665                             (62, 117, Solomon, king of the United Kingdom of Israel and Judah)   \n4666                                                                              (62, 69, Solomon)   \n4667                                                       (58, 88, marks locations in Google Maps)   \n4668                                                                      (58, 73, marks locations)   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   corpus  \\\n0                                                                                                                                                                                                 Mandalay Bay is a 43-story luxury resort and casino on the Las Vegas Strip in Paradise, Nevada. It is owned and operated by MGM Resorts International. One of the property's towers operates as the Delano; the Four Seasons Hotel is independently operated within the Mandalay Bay tower, occupying 5 floors (35–39).   \n1                                           A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.   \n2                                           A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.   \n3                                                                                                                                                            In 1865, Everest was given its official English name by the Royal Geographical Society, upon a recommendation by Andrew Waugh, the British Surveyor General of India. As there appeared to be several different local names, Waugh chose to name the mountain after his predecessor in the post, Sir George Everest, despite George Everest's objections.[6]   \n4     Players are currently inducted into the Hall of Fame through election by either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,[8] which now consists of four subcommittees, each of which considers and votes for candidates from a separate era of baseball. Five years after retirement, any player with 10 years of major league experience who passes a screening committee (which removes from consideration players of clearly lesser qualification) is eligible to b...   \n...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   ...   \n4664                                                                                                                                                                                                                                                                                                                                                                                                                                            As of January 17, 2018,[update] 201 episodes of Modern Family have aired.   \n4665  The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...   \n4666  The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...   \n4667                                                                                                                                                                                                                                  The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.   \n4668                                                                                                                                                                                                                                  The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.   \n\n      ans_start_tok  ans_end_tok  \\\n0                29           31   \n1                34           43   \n2                32           43   \n3               512          512   \n4                14           31   \n...             ...          ...   \n4664              1           12   \n4665             11           21   \n4666             11           11   \n4667             14           18   \n4668             14           15   \n\n                                                                             ans_tok_text  \n0                                                               mgm resorts international  \n1                                             the team that lost the pre - game coin toss  \n2                                  awarded to the team that lost the pre - game coin toss  \n3                                                                                          \n4     the baseball writers'association of america ( or bbwaa ), or the veterans committee  \n...                                                                                   ...  \n4664                                      as of january 17, 2018, [ update ] 201 episodes  \n4665                              solomon, king of the united kingdom of israel and judah  \n4666                                                                              solomon  \n4667                                                       marks locations in google maps  \n4668                                                                      marks locations  \n\n[4669 rows x 7 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>question</th>\n      <th>answer</th>\n      <th>corpus</th>\n      <th>ans_start_tok</th>\n      <th>ans_end_tok</th>\n      <th>ans_tok_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>who is the owner of the mandalay bay in vegas</td>\n      <td>(124, 149, MGM Resorts International)</td>\n      <td>Mandalay Bay is a 43-story luxury resort and casino on the Las Vegas Strip in Paradise, Nevada. It is owned and operated by MGM Resorts International. One of the property's towers operates as the Delano; the Four Seasons Hotel is independently operated within the Mandalay Bay tower, occupying 5 floors (35–39).</td>\n      <td>29</td>\n      <td>31</td>\n      <td>mgm resorts international</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>who kicks the ball first to start a football game</td>\n      <td>(138, 179, the team that lost the pre-game coin toss)</td>\n      <td>A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.</td>\n      <td>34</td>\n      <td>43</td>\n      <td>the team that lost the pre - game coin toss</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>who kicks the ball first to start a football game</td>\n      <td>(127, 179, awarded to the team that lost the pre-game coin toss)</td>\n      <td>A kick-off is used to start each half of play, and each period of extra time where applicable. The kick-off to start a game is awarded to the team that lost the pre-game coin toss (the team that won the coin-toss chooses which direction they wish to play). The kick-off begins when the referee blows the whistle. The kick-off to start the second half is taken by the other team. If extra time is played another coin-toss is used at the beginning of this period.</td>\n      <td>32</td>\n      <td>43</td>\n      <td>awarded to the team that lost the pre - game coin toss</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2</td>\n      <td>mount everest how did it get its name</td>\n      <td>NaN</td>\n      <td>In 1865, Everest was given its official English name by the Royal Geographical Society, upon a recommendation by Andrew Waugh, the British Surveyor General of India. As there appeared to be several different local names, Waugh chose to name the mountain after his predecessor in the post, Sir George Everest, despite George Everest's objections.[6]</td>\n      <td>512</td>\n      <td>512</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>3</td>\n      <td>who votes in the baseball hall of fame</td>\n      <td>(80, 162, the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee)</td>\n      <td>Players are currently inducted into the Hall of Fame through election by either the Baseball Writers' Association of America (or BBWAA), or the Veterans Committee,[8] which now consists of four subcommittees, each of which considers and votes for candidates from a separate era of baseball. Five years after retirement, any player with 10 years of major league experience who passes a screening committee (which removes from consideration players of clearly lesser qualification) is eligible to b...</td>\n      <td>14</td>\n      <td>31</td>\n      <td>the baseball writers'association of america ( or bbwaa ), or the veterans committee</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4664</th>\n      <td>3366</td>\n      <td>how many episodes are there in modern family</td>\n      <td>(0, 44, As of January 17, 2018,[update] 201 episodes)</td>\n      <td>As of January 17, 2018,[update] 201 episodes of Modern Family have aired.</td>\n      <td>1</td>\n      <td>12</td>\n      <td>as of january 17, 2018, [ update ] 201 episodes</td>\n    </tr>\n    <tr>\n      <th>4665</th>\n      <td>3367</td>\n      <td>who built the first temple for god in jerusalem</td>\n      <td>(62, 117, Solomon, king of the United Kingdom of Israel and Judah)</td>\n      <td>The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...</td>\n      <td>11</td>\n      <td>21</td>\n      <td>solomon, king of the united kingdom of israel and judah</td>\n    </tr>\n    <tr>\n      <th>4666</th>\n      <td>3367</td>\n      <td>who built the first temple for god in jerusalem</td>\n      <td>(62, 69, Solomon)</td>\n      <td>The Hebrew Bible states that the temple was constructed under Solomon, king of the United Kingdom of Israel and Judah and that during the Kingdom of Judah, the temple was dedicated to Yahweh, and is said to have housed the Ark of the Covenant. Jewish historian Josephus says that \"the temple was burnt four hundred and seventy years, six months, and ten days after it was built\",[1] although rabbinic sources state that the First Temple stood for 410 years and, based on the 2nd-century work Sede...</td>\n      <td>11</td>\n      <td>11</td>\n      <td>solomon</td>\n    </tr>\n    <tr>\n      <th>4667</th>\n      <td>3368</td>\n      <td>what is a dropped pin on google maps for</td>\n      <td>(58, 88, marks locations in Google Maps)</td>\n      <td>The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.</td>\n      <td>14</td>\n      <td>18</td>\n      <td>marks locations in google maps</td>\n    </tr>\n    <tr>\n      <th>4668</th>\n      <td>3368</td>\n      <td>what is a dropped pin on google maps for</td>\n      <td>(58, 73, marks locations)</td>\n      <td>The Google Maps pin is the inverted-drop-shaped icon that marks locations in Google Maps. The pin is protected under a U.S. design patent as \"teardrop-shaped marker icon including a shadow.\"[1][2] Google has used the pin in various graphics, games, and promotional materials.</td>\n      <td>14</td>\n      <td>15</td>\n      <td>marks locations</td>\n    </tr>\n  </tbody>\n</table>\n<p>4669 rows × 7 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":["**Create custom dataset class**\n","\n","Now let's create the custom dataset class which also handle the tokenization part. This way, the tokenization happens during training and validation and we don't consume ram to save all the tokenizations in prior."],"metadata":{"id":"40jHaO2LiFX4"}},{"cell_type":"code","source":["class NQ_Dataset(Dataset):\n","  def __init__(self, data: pd.DataFrame):\n","    self.data = data\n","\n","  def __len__(self):\n","    return len(self.data)\n","\n","  def __getitem__(self, idx):\n","    encoding = tokenizer(text=self.data['corpus'].values[idx], text_pair=self.data['question'].values[idx], max_length=512, padding='max_length', truncation=True, return_tensors='pt').to(device)\n","    return {\n","        'ans_start_tok': torch.tensor(self.data['ans_start_tok'].values[idx], dtype=torch.long, device=device),\n","        'ans_end_tok': torch.tensor(self.data['ans_end_tok'].values[idx], dtype=torch.long, device=device),\n","        'input_ids': encoding['input_ids'][0], \n","        'attention_mask': encoding['attention_mask'][0],\n","        'token_type_ids': encoding['token_type_ids'][0]\n","    }"],"metadata":{"id":"Lg8jIWz0iFmx","execution":{"iopub.status.busy":"2022-03-09T11:35:35.141637Z","iopub.execute_input":"2022-03-09T11:35:35.142071Z","iopub.status.idle":"2022-03-09T11:35:35.149540Z","shell.execute_reply.started":"2022-03-09T11:35:35.142033Z","shell.execute_reply":"2022-03-09T11:35:35.148905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_squad_dataset = NQ_Dataset(data=train_dataset)"],"metadata":{"id":"V1Iwzgz4mgil","execution":{"iopub.status.busy":"2022-03-09T11:35:43.315338Z","iopub.execute_input":"2022-03-09T11:35:43.316095Z","iopub.status.idle":"2022-03-09T11:35:43.321324Z","shell.execute_reply.started":"2022-03-09T11:35:43.316047Z","shell.execute_reply":"2022-03-09T11:35:43.319765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["val_squad_dataset = NQ_Dataset(data=validation_dataset)"],"metadata":{"id":"LC7cg4hGoosl","execution":{"iopub.status.busy":"2022-03-09T11:35:44.518753Z","iopub.execute_input":"2022-03-09T11:35:44.519031Z","iopub.status.idle":"2022-03-09T11:35:44.523570Z","shell.execute_reply.started":"2022-03-09T11:35:44.518999Z","shell.execute_reply":"2022-03-09T11:35:44.522436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["**Create Dataloaders**"],"metadata":{"id":"EgABQ9XAp6aW"}},{"cell_type":"code","source":["BATCH_SIZE=8"],"metadata":{"id":"cl4-YeWGp_MW","execution":{"iopub.status.busy":"2022-03-09T11:35:46.256302Z","iopub.execute_input":"2022-03-09T11:35:46.257195Z","iopub.status.idle":"2022-03-09T11:35:46.261071Z","shell.execute_reply.started":"2022-03-09T11:35:46.257134Z","shell.execute_reply":"2022-03-09T11:35:46.260353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_dataloader = DataLoader(train_squad_dataset, batch_size=BATCH_SIZE, shuffle=False)"],"metadata":{"id":"FpV37IAZp6p5","execution":{"iopub.status.busy":"2022-03-09T11:35:47.415113Z","iopub.execute_input":"2022-03-09T11:35:47.415650Z","iopub.status.idle":"2022-03-09T11:35:47.419584Z","shell.execute_reply.started":"2022-03-09T11:35:47.415611Z","shell.execute_reply":"2022-03-09T11:35:47.418951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["validation_dataloader = DataLoader(val_squad_dataset, batch_size=1, shuffle=False)"],"metadata":{"id":"Fit_x6PmqHIt","execution":{"iopub.status.busy":"2022-03-09T11:35:48.302860Z","iopub.execute_input":"2022-03-09T11:35:48.303634Z","iopub.status.idle":"2022-03-09T11:35:48.308173Z","shell.execute_reply.started":"2022-03-09T11:35:48.303593Z","shell.execute_reply":"2022-03-09T11:35:48.307176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["On the validation dataloader, i gave 1 as batch size and i'll explain why beyond."],"metadata":{"id":"mrfc8-O_VOL0"}},{"cell_type":"markdown","source":["**Train & Evaluation methods**\n","\n","Now, let's define the training function. It follows the same logic as the previous ones. The only major difference is the way the validation loss is calculated. In particular, for each question in the validation dataset, i take the minimum loss among all the answers because the model's goal is to approach any valid answer as good as possible, and if i consider all answer losses (and even those from answers far from the one that gives the minimum loss), this will add extra noisy error for no reason and confuse the train-validation loss plot. To calculate the validation loss this way, we must take the answers one at a time, so this is the reason i set batch size=1 on the validation dataloader."],"metadata":{"id":"OnxB1D3DqYIM"}},{"cell_type":"code","source":["def train(model: BertForQuestionAnswering, train_dataloader: DataLoader, validation_dataloader: DataLoader, learning_rate: float, epochs: int):\n","  # Define the optimizer\n","  opt = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n","  # Initialize train and validation losses lists\n","  train_losses = []\n","  validation_losses = []\n","  # Train for given # of epochs\n","  for epoch in range(epochs):\n","    model.train()\n","    t_losses = []\n","    for batch in tqdm(train_dataloader):\n","      # Delete previously stored gradients\n","      opt.zero_grad()\n","      # Get loss and outputs from the model\n","      start_positions = batch['ans_start_tok'] # we can do this because we know taht squad trai dataset questions always have 1 question\n","      end_positions = batch['ans_end_tok']\n","      out = model(input_ids=batch['input_ids'], attention_mask=batch['attention_mask'], token_type_ids=batch['token_type_ids'], start_positions=start_positions, end_positions=end_positions)\n","      # Perform backpropagation starting from the loss returned in this epoch\n","      loss = out[0]\n","      loss.backward()\n","      # Update model's weights based on the gradients calculated during backprop\n","      opt.step()\n","      t_losses.append(loss.item())\n","\n","    # Calculate train loss in current epoch\n","    train_loss = np.mean(t_losses) \n","    train_losses.append(train_loss)\n","    with torch.no_grad():\n","      model.eval()\n","      v_losses = []\n","      tmp_losses = []\n","      cur_idx = -1\n","      for val_idx, val_batch in enumerate(tqdm(validation_dataloader)):\n","        start_positions = val_batch['ans_start_tok'] # we can do this because we know taht squad trai dataset questions always have 1 question\n","        end_positions = val_batch['ans_end_tok']\n","        out = model(input_ids=val_batch['input_ids'], attention_mask=val_batch['attention_mask'], token_type_ids=val_batch['token_type_ids'], start_positions=start_positions, end_positions=end_positions)\n","        loss = out[0]\n","        batch_idx = validation_dataset['index'].values[val_idx]\n","        if cur_idx != batch_idx:\n","          cur_idx = batch_idx\n","          if len(tmp_losses) > 0:\n","            v_losses.append(min(tmp_losses))\n","          tmp_losses = []\n","        tmp_losses.append(loss.item())\n","      if len(tmp_losses) > 0:\n","        v_losses.append(min(tmp_losses))\n","      val_loss = np.mean(v_losses) \n","      validation_losses.append(val_loss)\n","\n","    # Print current epoch status\n","    print(f\"Epoch {epoch:3}: Loss = {train_loss:.5f} Val_loss = {val_loss:.5f}\")\n","\n","  return train_losses, validation_losses"],"metadata":{"id":"7xp2O95NSt51","execution":{"iopub.status.busy":"2022-03-09T11:35:50.656243Z","iopub.execute_input":"2022-03-09T11:35:50.656502Z","iopub.status.idle":"2022-03-09T11:35:50.668668Z","shell.execute_reply.started":"2022-03-09T11:35:50.656473Z","shell.execute_reply":"2022-03-09T11:35:50.667993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["And let's also define a function to plot the train and validation losses."],"metadata":{"id":"3TK8ewIiVfY8"}},{"cell_type":"code","source":["def plot_loss_vs_epochs(train_losses, val_losses):\n","  assert(len(train_losses) == len(val_losses))\n","  plt.grid()\n","  plt.plot(list(range(len(train_losses))), train_losses, \"o-\", label=\"Train\")\n","  plt.plot(list(range(len(train_losses))), val_losses, \"o-\", label=\"Validation\")\n","  plt.xlabel(\"Epochs\")\n","  plt.ylabel(\"Loss\")\n","  plt.legend()\n","  plt.show()"],"metadata":{"id":"Hq8DRSvLgevj","execution":{"iopub.status.busy":"2022-03-09T11:35:54.015329Z","iopub.execute_input":"2022-03-09T11:35:54.015590Z","iopub.status.idle":"2022-03-09T11:35:54.021187Z","shell.execute_reply.started":"2022-03-09T11:35:54.015563Z","shell.execute_reply":"2022-03-09T11:35:54.020451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["**Model Definition**\n","\n","Now let's load the pre-trained bert-base-uncased model."],"metadata":{"id":"4yvh_2pnSn1x"}},{"cell_type":"code","source":["reset_seed()"],"metadata":{"id":"2zGtAdQnSqiV","execution":{"iopub.status.busy":"2022-03-09T11:35:56.959097Z","iopub.execute_input":"2022-03-09T11:35:56.959526Z","iopub.status.idle":"2022-03-09T11:35:56.966010Z","shell.execute_reply.started":"2022-03-09T11:35:56.959491Z","shell.execute_reply":"2022-03-09T11:35:56.965275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["model = BertForQuestionAnswering.from_pretrained(BERT_MODEL_NAME).to(device)\n","model"],"metadata":{"id":"MyKrefB-SsHn","outputId":"09505609-7c10-43b2-84e5-745c119359f5","executionInfo":{"status":"ok","timestamp":1646594801054,"user_tz":-120,"elapsed":1645,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:36:00.647403Z","iopub.execute_input":"2022-03-09T11:36:00.648295Z","iopub.status.idle":"2022-03-09T11:36:16.138525Z","shell.execute_reply.started":"2022-03-09T11:36:00.648257Z","shell.execute_reply":"2022-03-09T11:36:16.137788Z"},"trusted":true,"colab":{"referenced_widgets":["fcf6ee6dc8da459191731acc797f693d"]}},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/420M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fcf6ee6dc8da459191731acc797f693d"}},"metadata":{}},{"name":"stderr","text":"Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForQuestionAnswering: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight']\n- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\nSome weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"},{"execution_count":33,"output_type":"execute_result","data":{"text/plain":"BertForQuestionAnswering(\n  (bert): BertModel(\n    (embeddings): BertEmbeddings(\n      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n      (position_embeddings): Embedding(512, 768)\n      (token_type_embeddings): Embedding(2, 768)\n      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (encoder): BertEncoder(\n      (layer): ModuleList(\n        (0): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (1): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (2): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (3): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (4): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (5): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (6): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (7): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (8): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (9): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (10): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (11): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n  )\n  (qa_outputs): Linear(in_features=768, out_features=2, bias=True)\n)"},"metadata":{}}]},{"cell_type":"markdown","source":["**Training**\n","\n","And now, time for training..."],"metadata":{"id":"BezmBjQBVnr6"}},{"cell_type":"code","source":["train_losses, validation_losses = train(model, train_dataloader, validation_dataloader, learning_rate=1e-5, epochs=3)"],"metadata":{"id":"21VOfQ1Pg1zp","outputId":"12c2b610-2ace-4c43-b642-61a9f25f8d10","executionInfo":{"status":"ok","timestamp":1646595124195,"user_tz":-120,"elapsed":320126,"user":{"displayName":"Βασίλης Κυριακόπουλος","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjLJ843-mdA-XYMgrr2K7G5eg-3pPcT6fAhtDjZiw=s64","userId":"10189451843812858163"}},"execution":{"iopub.status.busy":"2022-03-09T11:36:53.039242Z","iopub.execute_input":"2022-03-09T11:36:53.039530Z","iopub.status.idle":"2022-03-09T17:01:33.598764Z","shell.execute_reply.started":"2022-03-09T11:36:53.039497Z","shell.execute_reply":"2022-03-09T17:01:33.598076Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stderr","text":"100%|██████████| 13859/13859 [1:46:25<00:00,  2.17it/s]\n100%|██████████| 4669/4669 [01:47<00:00, 43.50it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch   0: Loss = 1.46427 Val_loss = 0.60050\n","output_type":"stream"},{"name":"stderr","text":"100%|██████████| 13859/13859 [1:46:24<00:00,  2.17it/s]\n100%|██████████| 4669/4669 [01:47<00:00, 43.48it/s]\n","output_type":"stream"},{"name":"stdout","text":"Epoch   1: Loss = 0.93841 Val_loss = 0.55722\n","output_type":"stream"},{"name":"stderr","text":"100%|██████████| 13859/13859 [1:46:28<00:00,  2.17it/s]\n100%|██████████| 4669/4669 [01:47<00:00, 43.62it/s]","output_type":"stream"},{"name":"stdout","text":"Epoch   2: Loss = 0.67777 Val_loss = 0.58197\n","output_type":"stream"},{"name":"stderr","text":"\n","output_type":"stream"}]},{"cell_type":"code","source":["plot_loss_vs_epochs(train_losses, validation_losses)"],"metadata":{"id":"TXx0EVLwmLem","execution":{"iopub.status.busy":"2022-03-09T17:01:33.600681Z","iopub.execute_input":"2022-03-09T17:01:33.601430Z","iopub.status.idle":"2022-03-09T17:01:33.816551Z","shell.execute_reply.started":"2022-03-09T17:01:33.601388Z","shell.execute_reply":"2022-03-09T17:01:33.815867Z"},"trusted":true,"outputId":"33a1130a-0bc0-436b-d8c7-c6cf1245cf72"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAzsElEQVR4nO3dd3hUZdrH8e+dSe+EQIAMEEAIPSShEwRkXRCkKBFFV2UtrK6CblHXdVesu7rsrq9Y1+5ayGoo0qwUBRGEhN6rkoB0EgIBkvC8f8yAQ0iZJDOZSeb+XNe55sxzyvxyGM495zlnzogxBqWUUr7Lz9MBlFJKeZYWAqWU8nFaCJRSysdpIVBKKR+nhUAppXycv6cDVFVsbKxJSEio1rInT54kLCzMtYFcwFtzgfdm01xVo7mqpj7mysrKOmyMaVTmRGNMnRpSU1NNdS1atKjay7qTt+Yyxnuzaa6q0VxVUx9zAatMOftV7RpSSikfp4VAKaV8nBYCpZTycXXuZLFSqv4QEXbv3s3p06c9HeUiUVFRbN682dMxLuFMruDgYKxWKwEBAU6vVwuBUspjwsLCiIiIICEhARHxdJwLTpw4QUREhKdjXKKyXMYYjhw5Qk5ODq1atXJ6vT7RNTRrdS79nlnI+M9O0u+ZhcxanevpSEopwGKx0LBhQ68qAnWZiNCwYcMqH2HV+yOCWatzeXjGegqLSgDIPV7IwzPWAzA6Od6T0ZRSoEXAxaqzPev9EcGUz7deKALnFRaVMOXzrR5KpJRS3qXeF4J9xwur1K6U8h1HjhyhW7dudOvWjSZNmhAfH0+3bt3o168fZ8+erXDZVatWMWnSpFpK6l71vmuoWXQIuWXs9KNCnT+jrpTyDrNW5zLl863sO15Is+gQHhiSWKMu3oYNG7JmzRoAHnvsMcLDw/njH//IiRMnCAwMpLi4GH//sneT3bt3p3v37tV+bW9S748IHhiSSEiA5aI2P4Hjp4r4w0drOXW22EPJlFJVcf58X+7xQgw/n+9z9cUf48eP5/7776dXr148+OCDfP/99/Tp04fk5GT69u3L1q22buXFixdz9dVXA7YicttttzFw4EBat27N1KlTXZrJ3er9EcH5TwtTPt9K7vFC4qND+MOV7dhz9BQvLNzOupzjvHxTCm3jvO9SMaV8yeNzNrJpX36501f/eJyzJecuaissKuHBzHVM+/7HMpfp2CySySM6VTlLbm4uy5Ytw2KxkJ+fz5IlS/D39+err77iz3/+M9OnT79kmS1btrBo0SJOnDhBYmIid999d5Wu5fekel8IwFYMRifHs3jxYgYOHHihvUdCA+7PWMPIF7/l6Ws6c22K1XMhlVIVKl0EKmuvidGjR2Ox2HoS8vLyuPXWW9m+fTsiQlFRUZnLDB8+nKCgIIKCgmjcuDEHDhzAaq0b+xSfKATl6d+2EfPv68/Eaav5/UdrWb7rCI+P7ExIoKXyhZVSLlXZJ/d+zyws83xffHQI//tNH5dmcbzV81//+lcGDRrEzJkz2bNnz0UfJh0FBQVdGLdYLBQX151u53p/jqAycZHBfHhHL+4ddBkfrcph9EvfsuNggadjKaVKKet8X0iAhQeGJLr1dfPy8oiPt3Uxv/POO259LU/x+UIA4G/x449DEnn3tp4cKjjDyBeX8ska/faxUt5kdHI8f7+2C/HRIQi2I4G/X9vF7V8MffDBB3n44YdJTk6uU5/yq8Knu4ZKG9CuEfMmpTFp2mruy1jD8l1HmDyiE8EB2lWklDc4f77PHR577LEL4ydOnLgw3qdPH7Zt23bh+VNPPQXAwIEDL3QTOS4LsGHDBrdkdBc9IiilaVQI0+7szd0D2zDt+71c8/Iydh3SriKlVP2lhaAM/hY/HhranrfH92B/XiEjXljKnLX7PB1LKaXcwm2FQETeEpGDIlLhMZKI9BCRYhFJd1eW6hrUvjHzJ/UnsUkEE6et5pGZ6zld6r5FSilV17nziOAdYGhFM4iIBXgW+MKNOWqkmf3StN9c3poPVvzImFeWsefwSU/HUkopl3FbITDGfAMcrWS2icB04KC7crhCgMWPh4d14I1bupNzrJCrX1jKvHX7PR1LKaVcQowx7lu5SAIw1xjTuYxp8cCHwCDgLft8meWsZwIwASAuLi41IyOjWnkKCgoIDw+v1rLnHS48x8trzrAr7xyDW/hzQ/tAAvxqdj91V+RyF2/NprmqxltzRUZG0rZtW0/HuERJScmFbxZ7E2dz7dixg7y8vIvaBg0alGWMKfsuecYYtw1AArChnGkfA73t4+8A6c6sMzU11VTXokWLqr2sozNFJebJORtNy4fmmqunLjE/HD5Zo/W5Kpc7eGs2zVU13porOzvbo68/cOBA89lnn13U9txzz5nbb7+9zPkHDBhgVq5caYwx5qqrrjLHjh27ZJ7JkyebKVOmVPi6M2fONBs3brzw/K9//av58ssvK82bn59f6TzGGLNp06ZL2oBVppz9qievGuoOZIjIHiAdeFlERnswj9MC/f34y9Udee3mVH44cpLhLyzhsw0/eTqWUvXfuo/guc7wWLTtcd1HNVrduHHjKN3DkJGRQXp65deuzJ8/n+jo6Gq97qxZs9i0adOF50888QS/+MUvqrUuV/BYITDGtDLGJBhjEoBM4LfGmFmeylMdv+zUhHmT+tM6Noy73s/i8TkbOVvs+htgKaWw7fTnTIK8vYCxPc6ZVKNikJ6ezrx58y78CM2ePXvYt28fmZmZdO/enU6dOjF58uQyl01ISODw4cMAPP3007Rr1460tLQLt6kGeP311+nRowdJSUmMGTOGU6dOsWzZMmbPns0DDzxAt27d2LlzJ+PHjycz09YzvmDBApKTk+nSpQu33XYbZ86cufB6Tz/9NCkpKXTp0oUtW7ZU++8uzW3fLBaRacBAIFZEcoDJQACAMeZVd71ubWseE8rHd/Xl759u5u1v95D943FeHJdM85hQT0dTqm759E/w0/ryp+eshJIzF7cVFcIn90LWu2Uv06QLXPVMuauMiYmhZ8+efPrpp4waNYqMjAzGjh3LxIkTadmyJSUlJQwePJh169bRtWvXMteRlZVFRkYGa9asobi4mJSUFFJTUwG49tprufPOOwH4y1/+wptvvsnEiRMZOXIkV1999SVHHqdPn2b8+PEsWLCAdu3accstt/DKK69w//33A7Yf0snOzubll1/mn//8J2+88Ub526sK3HnV0DhjTFNjTIAxxmqMedMY82pZRcAYM96Uc6K4Lgj092PyiE68+qsUdh0qYPjUJXyxUbuKlHKp0kWgsnYnOXYPZWRkMG7cOGbOnElKSgrJycls3Ljxom6c0pYsWcI111xDaGgokZGRjBw58sK0DRs20L9/f7p06cIHH3zAxo0bK8yydetWWrVqRbt27QC49dZb+eabby5MP7/u1NRU9uzZU90/+RJ6ryEXGtq5KR2bRnHPh9lMeC+LO9Ja8dBV7Qmw6Be4lapUBZ/cAds5gby9l7ZHNYdfz6v2y44aNYrf/e53ZGdnc+rUKWJiYpg6dSpZWVk0aNCA8ePHc/r06Wqte/z48cyaNYukpCTeeecdFi9eXO2c8POtrl19m2vdQ7lYi4ahZN7dh1v7tOSNpbsZ+5/vyryHulKqigY/CgEhF7cFhNjaayA8PJxBgwZx2223MW7cOPLz8wkLCyMqKooDBw7w6aefVrj85ZdfzqxZsygsLOTEiRPMmTPnwrQTJ07QtGlTioqK+OCDDy60R0REXHRju/MSExPZs2cPO3bsAOC9995jwIABNfr7nKGFwA2C/C08PqozL92YwvYDBQx7fgkLNh/wdCyl6rauY2HEVNsRAGJ7HDHV1l5D48aNY+3atYwbN46kpCS6du1K+/btufHGG+nXr1+Fy6akpHD99deTlJTEVVddRY8ePS5Me/LJJ+nVqxf9+vWjffv2F9pvuOEGpkyZQnJyMjt37rzQHhwczNtvv811111Hly5d8PPz46677qrx31ep8q4r9dbBG75HUBW7DxWYq/7vG9Pyobnmb/M2mbPFJV6Ry1nemk1zVY235vL09wjK4+z1+rWtPn6PwCckxIYx47d9+VXvFvznm13c8Npy9mlXkVLKi2ghqAXBARaeGt2FqeOS2bI/n+FTl7Boq1ffXkkp5UO0ENSikUnNmDMxjbjIYH799kqe/WwLxSX6BTTl24wb73fmi6qzPbUQ1LLWjcKZdU8/xvVsziuLd3Lj6ys4dlqLgfJNJSUlHDlyRIuBixhjOHLkCMHBwVVaTr9H4AHBARb+fm1XerVqyJ9nrufR3BJiWh9iQLtGno6mVK06efIkJ06c4NChQ56OcpHTp09XeWdaG5zJFRwcjNVqrdJ6tRB40OjkeDrHRzH+tW8Y//b33DPwMu7/RVv89QtoykcYY2jVqpWnY1xi8eLFJCcnezrGJdyVS/c4HnZZ43D+2ieE61KtvLhoBze9sYKD+dX7FqNSSlWHFgIvEGQR/pGexL+uS2JdTh7Dpi5h6fbDno6llPIRWgi8yJhUK7Pv7UeD0EBufmsF//5yGyXn9CSaUsq9tBB4mbZxEXxybz+uSY5n6oLt3PzmCg6e0K4ipZT7aCHwQqGB/vx7bDf+kd6V7B+PMez5pSzboV1FSin30ELgxcZ2b84n96QRFeLPr95cwdQF27WrSCnlcloIvFxikwhm35vGyKRm/PvLbYx/+3sOF9TshziUUsqRFoI6ICzIn+eu78Yz13bh+91HGfb8EpbvOuLpWEqpekILQR0hItzQswWz7ulHeJA/N76+nJcW7eCcdhUppWpIC0Ed06FpJLMnpjG8azOmfL6V8e+s5Ih2FSmlakALQR0UHuTP1Bu68fQ1nVm+6wjDpy5l5Z6jno6llKqjtBDUUSLCTb1aMuPuvgQH+HHDa8t5ZfFO7SpSSlWZFoI6rnN8FHMmpjG0UxOe/WwLt7+7kmMnz3o6llKqDtFCUA9EBAfw4o3JPDmqE9/uOMKwqUvI+kG7ipRSztFCUE+ICDf3SWD63X0JsPhx/X+W89o3O/UHP5RSldJCUM90sUYxd1IaV3aM42/zt3Dnf1dx/JR2FSmlyqeFoB6KDA7g5ZtSeGxER77edojhU5eS/eMxT8dSSnkpLQT1lIgwvl8rMu/qiwiMffU73liyS7uKlFKX0EJQzyU1j2bexP4Mat+Yp+ZtZsJ7WeSdKvJ0LKWUF9FC4AOiQgN47eZU/jK8A4u2HGT4C0tYu/e4p2MppbyEFgIfISLc0b81H93VB2Mg/dVlvP3tbu0qUkppIfA1KS0aMG9SGgPaNeLxOZu4+/1s8gq1q0gpX6aFwAdFhwby+i3d+fOw9ny5+QAjXljK+pw8T8dSSnmIFgIfJSJMuLwNH/2mN0Ul5xjzyjL++90e7SpSygdpIfBxqS1jmD+pP/0ua8ijn2zk3g9Xk39au4qU8iVuKwQi8paIHBSRDeVMv0lE1onIehFZJiJJ7sqiKtYgLJA3b+3BQ0Pb89nGnxj5wlI27tOuIqV8hTuPCN4BhlYwfTcwwBjTBXgSeM2NWVQl/PyEuwe2IWNCb04XneOal5fxwYoftKtIKR/gtkJgjPkGKPcWmMaYZcaY8/c9WA5Y3ZVFOa9HQgzzJqXRu3VDHpm5gf+sO0PBmWJPx1JKuZG48xOfiCQAc40xnSuZ749Ae2PMHeVMnwBMAIiLi0vNyMioVp6CggLCw8Ortaw7eWOuc8Ywb1cRM7afJS7Uj3uSg2ke4T2nlLxxm4HmqirNVTU1yTVo0KAsY0z3MicaY9w2AAnAhkrmGQRsBho6s87U1FRTXYsWLar2su7krbmMMeaV6V+ZHk99ado9Mt9MW/GDOXfunKcjGWO8d5tprqrRXFVTk1zAKlPOftWjH/FEpCvwBjDKGHPEk1lU2drHWJh/X396torhTzPW8/uP1nJSu4qUqlc8VghEpAUwA7jZGLPNUzlU5WLDg3jn1z35/ZXt+GRNLiNfXMrWn054OpZSykXcefnoNOA7IFFEckTkdhG5S0Tuss/yKNAQeFlE1ojIKndlUTVn8RMmDW7L+3f0Iq+wmFEvLeWjVXs9HUsp5QL+7lqxMWZcJdPvAMo8Oay8V982scy/L437M9bwYOY6Vuw6ypOjOxEa6La3klLKzbznMhBVZzSOCOa923tx3+C2zFidw6gXv2X7Ae0qUqqu0kKgqsXiJ/zuyna8d1svjp06y8gXv2V6Vo6nYymlqkELgaqRtLaxzJ/Un6TmUfzh47U8mLmWwrMlno6llKoCLQSqxhpHBvP+7b2YeMVlfJyVw+iXvmXHwQJPx1JKOUkLgXIJf4sff/hlIu/+uieHCs4w8sWlzFqd6+lYSiknaCFQLnV5u0bMn9Sfzs2iuP9/a3h4xjpOF2lXkVLeTAuBcrkmUcF8eGcvfjuwDdO+38vol75l1yHtKlLKW2khUG7hb/HjwaHtefvXPTiQf5oRLyxl9tp9no6llCqDFgLlVoMSGzNvUn/aN41k0rTVPDJzvXYVKeVltBAot2sWHULGhN785vLWfLDiR659eRl7Dp/0dCyllJ0WAlUrAix+PDysA2/e2p3c44Vc/cJS5q3b7+lYSim0EKhaNrhDHPPv60/buHDu+TCbRz/ZwJli7SpSypO0EKhaFx8dwv8m9OGOtFb897sfSH/lO348csrTsZTyWVoIlEcE+vvxl6s78trNqfxw5CTDX1jCZxu0q0gpT9BCoDzql52aMG9Sf1o3Cueu97N5bPZGzhaf83QspXyKFgLlcc1jQvn4N334db8E3lm2h+teXcbeo9pVpFRt0UKgvEKgvx+TR3Ti1V+lsuvwSYZPXcIXG3/ydCylfIIWAuVVhnZuwryJ/WnZMIwJ72Xx5NxN2lWklJtpIVBep0XDUDLv7sOtfVry5tLdjP3Pd+Qc064ipdxFC4HySkH+Fh4f1ZmXb0ph58EChk9dyoLNBzwdS6l6SQuB8mrDujRlzsQ0rA1CuP3dVfxt/maKSrSrSClX0kKgvF5CbBjT7+7Lr3q34LVvdnHDa8s5UqjFQClX0UKg6oTgAAtPje7CC+OS2frTCSYvK2TRloOejqVUvaCFQNUpI5KaMWdiGg2C/fj1Oyt55tMtFGtXkVI1ooVA1TmtYsP4a+9gxvVswatf72Tc68v5Ke+0p2MpVWdpIVB1UqBF+Pu1XXj+hm5s3JfPsKlL+HrbIU/HUqpO0kKg6rRR3eKZMzGNxhFB3PrW90z5XLuKlKoqpwqBiISJiJ99vJ2IjBSRAPdGU8o5bRqFM/O3/bi+e3NeWrSTm95YwYF87SpSylnOHhF8AwSLSDzwBXAz8I67QilVVSGBFp5N78q/rktiXU4ew55fwpLt2lWklDOcLQRijDkFXAu8bIy5DujkvlhKVc+YVCuz7+1HTFggt7z1Pf/+chsl54ynYynl1ZwuBCLSB7gJmGdvs7gnklI10zYugk/u7ce1yVamLtjOzW+u4OAJ7SpSqjzOFoL7gYeBmcaYjSLSGljktlRK1VBooD//GpvElPSuZP94jGHPL2XZjsOejqWUV3KqEBhjvjbGjDTGPGs/aXzYGDPJzdmUqrHrujfnk3vSiArx51dvruD5r7ZrV5FSpTh71dCHIhIpImHABmCTiDzg3mhKuUZikwhm35vGqG7xPPfVNm5963sOF5zxdCylvIazXUMdjTH5wGjgU6AVtiuHyiUib4nIQRHZUM50EZGpIrJDRNaJSEpVgitVFWFB/vx7bBLPjunCyj1HGfb8EpbvOuLpWEp5BWcLQYD9ewOjgdnGmCKgsuPrd4ChFUy/CmhrHyYArziZRalqERGu79GCWff0IzzInxtfX86LC7dzTruKlI9zthD8B9gDhAHfiEhLIL+iBYwx3wBHK5hlFPBfY7MciBaRpk7mUaraOjSNZPbENK7u2ox/frGN8e+s5Ih2FSkfJsZU79OQiPgbY4ormScBmGuM6VzGtLnAM8aYpfbnC4CHjDGryph3ArajBuLi4lIzMjKqlbmgoIDw8PBqLetO3poLvDebK3IZY/g6p5j3N58lPEC4OymIxJiaXRVdn7eXO2iuqqlJrkGDBmUZY7qXOdEYU+kARAH/BlbZh38BUU4slwBsKGfaXCDN4fkCoHtl60xNTTXVtWjRomov607emssY783mylwbco+bAf9YaFo/PM+8tGi7KSk55xW5XElzVU19zAWsMuXsV53tGnoLOAGMtQ/5wNvVKEqOcoHmDs+t9jalalWnZlHMmZjG0M5N+MdnW7n93ZUcPXnW07GUqjXOFoI2xpjJxphd9uFxoHUNX3s2cIv96qHeQJ4xZn8N16lUtUQEB/DiuGSeHN2Zb3ccYfjUJazaU9EpLqXqD2cLQaGIpJ1/IiL9gMKKFhCRacB3QKKI5IjI7SJyl4jcZZ9lPrAL2AG8Dvy2yumVciER4ebeLZnx274EWPy4/rXl/OfrnXpVkar3/J2c7y7gvyISZX9+DLi1ogWMMeMqmW6Ae5x8faVqTef4KOZOSuOhzHX8/dMtfL/7KP8am0R0aKCnoynlFs7eYmKtMSYJ6Ap0NcYkA1e4NZlSHhQZHMDLN6Xw+MhOfLP9EMOnLiX7x2OejqWUW1TpF8qMMfnG9g1jgN+7IY9SXkNEuLVvAtPv7oufH4x99TveWLLr/FVuStUbNfmpSnFZCqW8WFdrNHMn9mdwh8Y8NW8zE97LIu9UkadjKeUyNSkE+rFI+YyokABe/VUqj17dkcVbDzJs6hLW7D3u6VhKuUSFhUBETohIfhnDCaBZLWVUyiuICLelteLju/oCcN2ry3hr6W7tKlJ1XoVXDRljImoriFJ1Rbfm0cyblMYfP17HE3M38f3uo1yeGMtLC3eSe7yQ+OULeWBIIqOT4z0dVSmnOHv5qFLKQXRoIK/fksobS3bzt/mb+XzjTxf6SnOPF/LwjPUAWgxUnVCTcwRK+TQR4c7LW9MwPOiSE2aFRSVM+XyrR3IpVVVaCJSqofJuYZ17vJAfjpys5TRKVZ0WAqVqqFl0SLnTBkxZzNhXv+OjlXspOFPhXduV8hgtBErV0ANDEgkJuPh3DEICLEwe0ZEHhiRyuOAMD05fR/envuR3/1vD0u2H9f5FyqvoyWKlauj8CeEpn2+1XTUUHXLRVUO/HdiG1XuPk5mVw5y1+5i5OpdmUcFcm2JlTKqVVrFhnoyvlBYCpVxhdHI8o5PjWbx4MQMHDrxomoiQ0qIBKS0a8OjVHfly0wEys3J4efEOXly0g9SWDUhPtTK8a1MigwM88wcon6aFQKlaFBxgYURSM0YkNeNA/mlmrs5lelYOD89Yz2OzNzKkUxPGpFpJuywWi5/exUXVDi0ESnlIXGQwdw1ow28ub826nDymZ+fwyZp9zF67j7jIIK5JtpKeGs9ljfV7ncq9tBAo5WEiQlLzaJKaR/PI8A4s3HyQzKwcXl+yi1e/3klS82jSU62M7NqMqFDtOlKup4VAKS8S5G/hqi5NuapLUw6dOMMna3LJzMrhr7M28OScTVzZMY4xqfFc3rYR/ha96E+5hhYCpbxUo4gg7ujfmtvTWrFxXz6ZWTnMXruPeev30ygiiNHdmpGe2pzEJtp1pGpGC4FSXk5E6BwfRef4KP48rAOLttq6jt7+dg+vL9lNl/goxqTEM7JbPDFh+nOaquq0EChVhwT6+zGkUxOGdGrCkYIzfLJmH9Ozc3hsziaenr+ZK9o3Jj21OQMTGxGgXUfKSVoIlKqjGoYHcVtaK25La8Xm/flMz8ph1ppcPt94gIZhgYzqFk96qpWOzSI9HVV5OS0EStUDHZpG8perO/LQVe35ZtshMrNyeH/5D7z17W46NI2kW1QRnQvOEBse5OmoygtpIVCqHgmw+DG4QxyDO8Rx7ORZ5qzbx/SsHKZtOcvHf1vAwMTGpKfGc0X7OAL9tetI2WghUKqeahAWyC19ErilTwIfzF3Ij5amzMzO5avNB2gQGsDIJNtVR53jIxHRbzH7Mi0ESvmA+HA/bhrYgQd+mciSHYdtRwkr9/Ludz+QGBfBmFTbvZIaRwR7OqryAC0ESvkQf4sfgxIbMyixMXmnipi7fh+ZWTn8bf4Wnv1sK5e3jSU9tTmDOzQmuNSttVX9pYVAKR8VFRrATb1aclOvluw8VMD0rBxmZOdyz4fZRAb7M9L+hbUka5R2HdVzWgiUUrRpFM6DQ9vzh18msmznYTKzcvh4VQ7vL/+RNo3CSE9tzjXJ8TSJ0q6j+kgLgVLqAouf0L9tI/q3bUT+6SLmr9vP9Owcnv1sC1M+30Ja20aMSYlnSKcm2nVUj2ghUEqVKTI4gBt6tuCGni3Yc/gkM7JzmJ6dy30Za4gI8ufqpKakp1pJadFAu47qOC0ESqlKJcSG8ftfJnL/L9qxfPcRMrNymLV6H9O+30ur2DDGpMRzTYqV+OgQT0dV1aCFQCnlND8/oW+bWPq2ieWJUcV8un4/mVk5/POLbfzry230bdOQ9FQrQzo1ITRQdy91hf5LKaWqJTzIn+u6N+e67s3Ze/QU07NzmJ6dw+/+t5awwA0M79qUMSlWeraK0a4jL6eFQClVY81jQrn/F+2YdEVbVu45yvTsHOat289Hq3JoERPKmBQr16bE0zwm1NNRVRm0ECilXMbPT+jVuiG9WjfksZGd+HzjT2Rm5fB/C7bx3Ffb6N06hjEpVoZ1aUpYkO5+vIVb/yVEZCjwPGAB3jDGPFNqegvgXSDaPs+fjDHz3ZlJKVU7QgP9uSbZyjXJVnKPFzLTftXRA5nrmDx7I0M7N6GtpYTLzxn8/LTryJPcVghExAK8BFwJ5AArRWS2MWaTw2x/AT4yxrwiIh2B+UCCuzIppTwjPjqEe69oyz2DLiP7x2NkZuUwd+1+Zpwp5v3tixiTEs+1KVYSYsM8HdUnufOIoCewwxizC0BEMoBRgGMhMMD5X82IAva5MY9SysNEhNSWMaS2jGHyiE7838eL2HQ6nBcW7WDqwh30SGjAmBQrw7s2JSI4wNNxfYYYY9yzYpF0YKgx5g7785uBXsaYex3maQp8ATQAwoBfGGOyyljXBGACQFxcXGpGRka1MhUUFBAeHl6tZd3JW3OB92bTXFXj7bmOnj7Hsn3FfJtbzP6ThkA/SI2z0C8+gI4N/fCr5auOvH17VcegQYOyjDHdy5xojHHLAKRjOy9w/vnNwIul5vk98Af7eB9sRwt+Fa03NTXVVNeiRYuqvaw7eWsuY7w3m+aqmrqS69y5cyb7h6PmkZnrTJfJn5mWD801vf/2lXn2081mx8ETHsvlLWqSC1hlytmvurNrKBdo7vDcam9zdDswFMAY852IBAOxwEE35lJKeSkRIblFA5JbNOAvwzuyYPNBMrP28urXO3l58U6SW0STnmrl6q7NiArRriNXcWchWAm0FZFW2ArADcCNpeb5ERgMvCMiHYBg4JAbMyml6ojgAAvDuzZleNemHMw/zaw1uWRm5fDIzA08PmcTv+wYR3qqlf5tG2HRq45qxG2FwBhTLCL3Ap9juzT0LWPMRhF5AtshymzgD8DrIvI7bCeOx9sPYZRS6oLGkcFMuLwNd/ZvzYbcfKZn5zBrTS5z1+2ncUQQ16TEk55ipW1chKej1klu/R6BsX0nYH6ptkcdxjcB/dyZQSlVf4gIXaxRdLFG8edhHVi45SCZWTm8uWQ3//l6F0nWKMakWhmZ1Izo0EBPx60z9Kt9Sqk6KdDfj6GdmzC0cxMOF5zhkzW2n9189JONPDV3M7/o2JgxKVYGtGuEv8XP03G9mhYCpVSdFxsexO1prbg9rRUb9+UxPSuXT9bkMn/9T8SGBzG6WzPSu1tp3ySy8pX5IC0ESql6pVOzKDo1i+LhYe1ZvPUQmVl7efe7PbyxdDedmkWSnmplVLd4YsK06+g8LQRKqXopwOLHlR3juLJjHEdPnmX2mlymZ+fy+JxNPD1vM1e0b0x6qpWBiY0J9PftriMtBEqpei8mLJDx/Voxvl8rtv50gunZOczIzuWLTQeICQtkVLdmjEmx0qlZpE/+doIWAqWUT0lsEsGfh3XgwSGJLNl+mMysHD5Y/iNvf7uH9k0iSE+10uiMb13FroVAKeWT/C1+DGrfmEHtG3P81FnmrLP97OZT8zbjJzB7/0rSU61c0aExQf4WT8d1Ky0ESimfFx0ayM29W3Jz75bsOHiCf8/8jqx9eSzYcpDo0ABGJjUjPdVKl/ioetl1pIVAKaUcXNY4grGJgbxw+QCW7jjM9Kwc/rdyL//97gfaNg4nPdXKNcnxNI4M9nRUl9FCoJRSZbD4CQPaNWJAu0bkny5inr3r6O+fbuHZz7ZwebtGjEmxcmXHOIID6nbXkRYCpZSqRGRwAON6tmBczxbsOlTAjOxcZmTnMHHaaiKD/RmR1IwxqVaSm0fXya4jLQRKKVUFrRuF88chifz+ynZ8t+sImVk5TM/O4YMVP9K6URhjUqxcmxJP06gQT0d1mhYCpZSqBj8/od9lsfS7LJYnRnXi0/U/kZmdw5TPt/LPL7aSdlks6alWftmxCSGB3t11pIVAKaVqKCI4gLE9mjO2R3N+PHKK6dm2o4T7MtYQHuTP1V2bkp5qJbVlA6/sOtJCoJRSLtSiYSi/u7Id9w1uy4rdR5mencPstfvIWLmXhIahjEmxck1KPNYGoZ6OeoEWAqWUcgM/P6FPm4b0adOQx0d24rMNP5GZlcO/vtzGv77cRt82DRmTYuWqLk0IDfTsrlgLgVJKuVlYkD9jUq2MSbWy9+gpZq7OZXp2Dn/4eC2PfrKBq7rYuo56JsTg54Gf3dRCoJRStah5TCiTBrdl4hWXseqHY0zPymGu/TsK1gYhjEmxMibFSouGtdd1pIVAKaU8QETokRBDj4QYJo/oxBebbF1HUxdu5/kF2+nZKob0VCvDujTlq00HmPL5VnKPFxK/fCEPDElkdHK8y7JoIVBKKQ8LCbQwqls8o7rFs+94oa3rKCuHBzPX8ciM9ZwzUGJsd0TNPV7IwzPWA7isGPj2rzEopZSXaRYdwj2DLmPBHwYw47d9CfD3u1AEzissKmHK51td9ppaCJRSyguJCCktGlB4tqTM6fuOF7rstbQQKKWUF2sWXfatKsprrw4tBEop5cUeGJJISKm7m4YEWHhgSKLLXkNPFiullBc7f0L4wlVD0SF61ZBSSvma0cnxjE6OZ/HixQwcONDl69euIaWU8nFaCJRSysdpIVBKKR+nhUAppXycFgKllPJxWgiUUsrHaSFQSikfp4VAKaV8nFsLgYgMFZGtIrJDRP5UzjxjRWSTiGwUkQ/dEmTdR/BcZwYsHg3PdbY9V0opBbjxm8UiYgFeAq4EcoCVIjLbGLPJYZ62wMNAP2PMMRFp7PIg6z6COZOgqBAByNtrew7QdazLX04ppeoadx4R9AR2GGN2GWPOAhnAqFLz3Am8ZIw5BmCMOejyFAuegKJSt2stKoSvJkNJkctfTiml6hp33msoHtjr8DwH6FVqnnYAIvItYAEeM8Z85tIUeTllt+fvgydjISAMgqNKDZFltDkMQQ7j/oEujauUUrVNTKlfvnHZikXSgaHGmDvsz28Gehlj7nWYZy5QBIwFrMA3QBdjzPFS65oATACIi4tLzcjIcDpH7+/uIPjMoUvai/zDybGOxL/4ZIWDcK7C9Zf4BVLsH+bUUGK5tO2cJZCCggLCw8Od/ptqk7dm01xVo7mqpj7mGjRoUJYxpntZ09x5RJALNHd4brW3OcoBVhhjioDdIrINaAusdJzJGPMa8BpA9+7dTZXuvhfztwvnCC4ICCFgxHO0quwcgTFw9iSczoMz+bbHi4bjWE7nYTmdT9BF7fugwD5+rpLuJ0sQZ/1CCIyMreAIJBKCo8ueFhACIs5vjypy190Oa0pzVY3mqhqvy7XuI1jwBCYvB4mywuBHXXqO052FYCXQVkRaYSsANwA3lppnFjAOeFtEYrF1Fe1yaYrzG6s6G1EEgsJtA9W497cxtgJUThGxPeZzeNdmmsWE/Tzt+N6fi0/x6Ypfwy+g4q6soAq6uIIjITDcrYVEKVVF50ps+43i07bHjTNh0VNQfMZtF7y4rRAYY4pF5F7gc2z9/28ZYzaKyBPAKmPMbPu0X4rIJqAEeMAYc8TlYbqOha5j+bq2q7wIBIbahogm5c62bfFimpWXq+h0qUJy3GG8rAKTB/n7f16m6FQlGS1lFxJ7AWn501FYvqX8cyiBEeCnX0dR9VhJse3/0fkdc1UfnZ73NBQXQsnZyjMVFdouhPH2QgBgjJkPzC/V9qjDuAF+bx9UWQKCbUN4Na+sLT7rUEiOV1xAzg9Hdl4Yb3W2APZMq+AF5OLCUOERSBkn4oMiwc9SwfqVcmCM7Wq/4sLq7WSdWuY0fU/lwbJztmXOFVczrNi6bv2Dy34Mb1z+NMfH85/+SyvvQphq0F8oq+/8A8E/FsJiq7X41wsXMKB3snNF5HzBOf7DxW2VCYp0OBfiXAEJLtwPp47alrHo29hjjIHiM2V8Yj6/46348bLd2yF/RiU771KPpuILOMolljJ2tsEQEGobD2lwYdrhg0dp1rJNJTtqh2XLmscS6Jpu12+m2LqDSouy1nzddvo/SFXI+FkgNMY2VMe5EocjEieKyOk8yM+Bgxt/XoZLr2zrDbDC/qTMS4AruxQ4+ufC44pLgO0n8wbk5cBq15/Mc9q5c7adaPFp287ZvvOMzNsKu/2c2EGX9Ym5omVOU9a/j1P8AmgiAXA83LZT9Q+x7UADQmznrsIalb8D9g8p4zGk4k/XlgCno1XYXVvbBj9a5gUvDH60/GWqSAuBci8/i+2TVkiD6i1/7hycLbikcGxes5wOCc3K6PbKg4Kf4PDWnwuJKan4NfxDql9EgqNg8+zyv73eeYzDDvWU05+UL3osKnR+3pIzZf6JKQCrK9nWlqCLd66On3aDoyGish2xkzvk849+FpZ629U53qgmF7w4SQuB8m5+fvYdciSOVyMf+CmUDr0HVr684yXAZR6FHL+0/dRhOLrTfgRzvHp9xEWFMONO21BdFe1cQ2Od2CH/vOy6zdvpmtKz/B2zf7Ce9Pdmbr7gRQuBqt8cLwGOqsElwBUVkQVPlL/8wD+X8ym7kp23f5BLL+s9enAxtOrvsvWp+kULgVIVcbwEOLJp2fOseruck3nNYeBD7s2nlAvosaBSNTX4UVsXiyMXn8xTyp20EChVU13HwoipENUcg9iOBEZM1ducqzpDu4aUcgVPfXtdKRfQIwKllPJxWgiUUsrHaSFQSikfp4VAKaV8nBYCpZTycW77qUp3EZFDwA/VXDwWOOzCOK7irbnAe7NprqrRXFVTH3O1NMY0KmtCnSsENSEiq8r7zU5P8tZc4L3ZNFfVaK6q8bVc2jWklFI+TguBUkr5OF8rBK95OkA5vDUXeG82zVU1mqtqfCqXT50jUEopdSlfOyJQSilVihYCpZTycfWmEIjIUBHZKiI7RORPZUwPEpH/2aevEJEEh2kP29u3isiQWs71exHZJCLrRGSBiLR0mFYiImvsw+xazjVeRA45vP4dDtNuFZHt9uHWWs71nEOmbSJy3GGaO7fXWyJyUEQ2lDNdRGSqPfc6EUlxmObO7VVZrpvsedaLyDIRSXKYtsfevkZEVtVyroEikufw7/Wow7QK3wNuzvWAQ6YN9vdUjH2aW7aXiDQXkUX2/cBGEbmvjHnc+/4yxtT5AbAAO4HWQCCwFuhYap7fAq/ax28A/mcf72ifPwhoZV+PpRZzDQJC7eN3n89lf17gwe01HnixjGVjgF32xwb28Qa1lavU/BOBt9y9vezrvhzbb8BvKGf6MOBTQIDewAp3by8nc/U9/3rAVedz2Z/vAWI9tL0GAnNr+h5wda5S844AFrp7ewFNgRT7eASwrYz/j259f9WXI4KewA5jzC5jzFkgAxhVap5RwLv28UxgsIiIvT3DGHPGGLMb2GFfX63kMsYsMsacsj9dDlhd9No1ylWBIcCXxpijxphjwJfAUA/lGgdMc9FrV8gY8w1wtIJZRgH/NTbLgWgRaYp7t1eluYwxy+yvC7X3/nJme5WnJu9NV+eqlfeXMWa/MSbbPn4C2AyU/oFtt76/6kshiAccfzQ2h0s35IV5jDHFQB7Q0Mll3ZnL0e3Yqv55wSKySkSWi8hoF2WqSq4x9sPQTBFpXsVl3ZkLexdaK2ChQ7O7tpczysvuzu1VVaXfXwb4QkSyRGSCB/L0EZG1IvKpiHSyt3nF9hKRUGw71OkOzW7fXmLrsk4GVpSa5Nb3l/5CmZcQkV8B3YEBDs0tjTG5ItIaWCgi640xO2sp0hxgmjHmjIj8BtvR1BW19NrOuAHINMaUOLR5cnt5NREZhK0QpDk0p9m3V2PgSxHZYv/EXBuysf17FYjIMGAW0LaWXtsZI4BvjTGORw9u3V4iEo6t8NxvjMl31XqdUV+OCHKB5g7Prfa2MucREX8gCjji5LLuzIWI/AJ4BBhpjDlzvt0Yk2t/3AUsxvZJoVZyGWOOOGR5A0h1dll35nJwA6UO2924vZxRXnZ3bi+niEhXbP+Go4wxR863O2yvg8BMXNclWiljTL4xpsA+Ph8IEJFYvGB72VX0/nL59hKRAGxF4ANjzIwyZnHv+8vVJz48MWA7stmFravg/AmmTqXmuYeLTxZ/ZB/vxMUni3fhupPFzuRKxnZyrG2p9gZAkH08FtiOi06aOZmrqcP4NcBy8/PJqd32fA3s4zG1lcs+X3tsJ+6kNraXw2skUP7Jz+FcfDLve3dvLydztcB23qtvqfYwIMJhfBkwtBZzNTn/74dth/qjfds59R5wVy779Chs5xHCamN72f/u/wL/V8E8bn1/uWzjenrAdlZ9G7ad6iP2tiewfcoGCAY+tv+n+B5o7bDsI/bltgJX1XKur4ADwBr7MNve3hdYb/+PsB64vZZz/R3YaH/9RUB7h2Vvs2/HHcCvazOX/fljwDOllnP39poG7AeKsPXD3g7cBdxlny7AS/bc64HutbS9Ksv1BnDM4f21yt7e2r6t1tr/nR+p5Vz3Ory/luNQqMp6D9RWLvs847FdQOK4nNu2F7buOgOsc/h3Glab7y+9xYRSSvm4+nKOQCmlVDVpIVBKKR+nhUAppXycFgKllPJxWgiUUsrHaSFQyq7U3UvXuPLOlyKSUN4dL5XyNL3FhFI/KzTGdPN0CKVqmx4RKFUJ+33o/2G/F/33InKZvT1BRBbKz78l0cLeHiciM+03VFsrIn3tq7KIyOv2e85/ISIh9vknyc+/SZHhoT9T+TAtBEr9LKRU19D1DtPyjDFdgBeB/7O3vQC8a4zpCnwATLW3TwW+NsYkYbv3/UZ7e1vgJWNMJ+A4MMbe/icg2b6eu9zzpylVPv1msVJ2IlJgjAkvo30PcIUxZpf95mA/GWMaishhbPdkKrK37zfGxIrIIcBqHG4gaL+98JfGmLb25w8BAcaYp0TkM6AA2x04Zxn7zdiUqi16RKCUc0w541VxxmG8hJ/P0Q3Hdh+ZFGCl/e64StUaLQRKOed6h8fv7OPLsN3JFuAmYIl9fAG2nx1FRCwiElXeSkXED2hujFkEPITtzpeXHJUo5U76yUOpn4WIyBqH558ZY85fQtpARNZh+1Q/zt42EXhbRB4ADgG/trffB7wmIrdj++R/N7Y7XpbFArxvLxYCTDXGHHfR36OUU/QcgVKVsJ8j6G6MOezpLEq5g3YNKaWUj9MjAqWU8nF6RKCUUj5OC4FSSvk4LQRKKeXjtBAopZSP00KglFI+7v8BffD4Y9HTalYAAAAASUVORK5CYII=\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["**Save the model**"],"metadata":{"id":"WHGtr7vI8hAN"}},{"cell_type":"code","source":["torch.save(model.state_dict(), \"model.pt\")"],"metadata":{"id":"4NYOtGL-mMod","execution":{"iopub.status.busy":"2022-03-09T17:01:33.817661Z","iopub.execute_input":"2022-03-09T17:01:33.820098Z","iopub.status.idle":"2022-03-09T17:01:34.448425Z","shell.execute_reply.started":"2022-03-09T17:01:33.820054Z","shell.execute_reply":"2022-03-09T17:01:34.447677Z"},"trusted":true},"execution_count":null,"outputs":[]}]}