Jianyu Shi_W266_Final-Project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "09c904b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "from transformers import AutoTokenizer, BertForSequenceClassification, AlbertForSequenceClassification, AdamW\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from fever.scorer import fever_score\n",
    "from prettytable import PrettyTable\n",
    "from collections import Counter\n",
    "\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "623c7491",
   "metadata": {},
   "source": [
    "# doc retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6754dd17",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load training data with claim, label, and predicted sentences from the wiki articles  \n",
    "# data preprocessed by UKPLab: https://github.com/UKPLab/fever-2018-team-athene\n",
    "\n",
    "all_train_list = []\n",
    "with open('data/all_train.json', 'r') as all_train:\n",
    "    for example in all_train:\n",
    "        instance = json.loads(example.strip())\n",
    "        all_train_list.append(instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ad80e017",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 111134,\n",
       " 'claim': 'Chinese people are people associated with Ireland.',\n",
       " 'evidence': [['People',\n",
       "   0,\n",
       "   'A people is a plurality of persons considered as a whole , as is the case with an ethnic group or nation .',\n",
       "   0],\n",
       "  ['People',\n",
       "   1,\n",
       "   'Collectively , for example , the contemporary Frisians and Danes are two related Germanic peoples , while various Middle Eastern ethnic groups are often linguistically categorized as Semitic peoples .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   0,\n",
       "   'Ireland LRB LSB ˈaɪərlənd RSB Éire LSB ˈeːɾʲə RSB ; Ulster Scots : Airlann LSB scoˈɑːrlən RSB RRB is an island in the North Atlantic .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   1,\n",
       "   \"It is separated from Great Britain to its east by the North Channel , the Irish Sea , and St George 's Channel .\",\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   2,\n",
       "   'Ireland is the second largest island of the British Isles , the third largest in Europe , and the twentieth largest on Earth .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   5,\n",
       "   'Politically , Ireland is divided between the Republic of Ireland LRB officially named Ireland RRB , which covers five sixths of the island , and Northern Ireland , which is part of the United Kingdom , in the northeast of the island .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   6,\n",
       "   'In 2011 , the population of Ireland was about 6.4 million , ranking it the second most populous island in Europe after Great Britain .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   7,\n",
       "   'Just under 4.6 million live in the Republic of Ireland and just over 1.8 million live in Northern Ireland .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   10,\n",
       "   \"The island 's geography comprises relatively low lying mountains surrounding a central plain , with several navigable rivers extending inland .\",\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   11,\n",
       "   'The island has lush vegetation , a product of its mild but changeable climate which avoids extremes in temperature .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   12,\n",
       "   'Thick woodlands covered the island until the Middle Ages .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   13,\n",
       "   'As of 2013 , the amount of land that is wooded in Ireland is about 11 % of the total , compared with a European average of 35 % .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   14,\n",
       "   'There are twenty six extant mammal species native to Ireland .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   15,\n",
       "   'The Irish climate is very moderate and classified as oceanic .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   16,\n",
       "   'As a result , winters are milder than expected for such a northerly area .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   17,\n",
       "   'However , summers are cooler than those in Continental Europe .',\n",
       "   0],\n",
       "  ['Ireland', 18, 'Rainfall and cloud cover are abundant .', 0],\n",
       "  ['Ireland',\n",
       "   21,\n",
       "   'The earliest evidence of human presence in Ireland is dated at 10,500 BC .',\n",
       "   0],\n",
       "  ['Ireland', 22, 'Gaelic Ireland had emerged by the 1st century CE .', 0],\n",
       "  ['Ireland',\n",
       "   23,\n",
       "   'The island was Christianised from the 5th century onward .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   24,\n",
       "   'Following the Norman invasion in the 12th century , England claimed sovereignty over Ireland .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   25,\n",
       "   'However , English rule did not extend over the whole island until the 16th 17th century Tudor conquest , which led to colonisation by settlers from Britain .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   26,\n",
       "   'In the 1690s , a system of Protestant English rule was designed to materially disadvantage the Catholic majority and Protestant dissenters , and was extended during the 18th century .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   27,\n",
       "   'With the Acts of Union in 1801 , Ireland became a part of the United Kingdom .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   28,\n",
       "   'A war of independence in the early 20th century was followed by the partition of the island , creating the Irish Free State , which became increasingly sovereign over the following decades , and Northern Ireland , which remained a part of the United Kingdom .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   29,\n",
       "   'Northern Ireland saw much civil unrest from the late 1960s until the 1990s .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   30,\n",
       "   'This subsided following a political agreement in 1998 .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   31,\n",
       "   'In 1973 the Republic of Ireland joined the European Economic Community while the United Kingdom , and Northern Ireland , as part of it , did the same .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   34,\n",
       "   'Irish culture has had a significant influence on other cultures , especially in the fields of literature .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   35,\n",
       "   'Alongside mainstream Western culture , a strong indigenous culture exists , as expressed through Gaelic games , Irish music , and the Irish language .',\n",
       "   0],\n",
       "  ['Ireland',\n",
       "   36,\n",
       "   'The culture of the island also shares many features with that of Great Britain , including the English language , and sports such as association football , rugby , horse racing , and golf .',\n",
       "   0],\n",
       "  ['Chinese_people',\n",
       "   0,\n",
       "   'Chinese people are the various individuals or groups of people associated with China , either through ancestry , ethnicity , nationality , citizenship , or other affiliation .',\n",
       "   1],\n",
       "  ['People_Are_People',\n",
       "   0,\n",
       "   \"`` People Are People '' is Depeche Mode 's 10th UK single , and was recorded at Hansa Mischraum in West Berlin , and released on 12 March 1984 .\",\n",
       "   0],\n",
       "  ['People_Are_People',\n",
       "   1,\n",
       "   'It was their first hit single in the US and the first single from the Some Great Reward album .',\n",
       "   0]],\n",
       " 'label': 'REFUTES'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_train_list[8880]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6a7124cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_dev_list = []\n",
    "with open('data/all_dev.json', 'r') as all_dev:\n",
    "    for example in all_dev:\n",
    "        instance = json.loads(example.strip())\n",
    "        all_dev_list.append(instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "acc10d5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19998"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_dev_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7c114164",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_mini = all_train_list[:5]\n",
    "all_dev_mini = all_dev_list[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "9dcc23ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_dev_mini[1]['evidence'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1339def",
   "metadata": {},
   "source": [
    "## sentence retrieval"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc45b7d3",
   "metadata": {},
   "source": [
    "### prep the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "169f9673",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_list = all_train_list\n",
    "#dev_list = all_dev_list\n",
    "\n",
    "train_list = all_train_mini\n",
    "dev_list = all_dev_mini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "74795304",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 75397,\n",
       " 'claim': 'Nikolaj Coster-Waldau worked with the Fox Broadcasting Company.',\n",
       " 'evidence': [['Waldau', 0, 'Waldau may refer to :', 0],\n",
       "  ['Waldau', 3, 'Waldau LRB surname RRB', 0],\n",
       "  ['Waldau',\n",
       "   5,\n",
       "   'Waldau LRB Bern RRB , a psychiatric clinic in Berne , Switzerland',\n",
       "   0],\n",
       "  ['Waldau',\n",
       "   7,\n",
       "   'Waldau LRB Burgenlandkreis RRB , a village and a former municipality in Saxony Anhalt , Germany',\n",
       "   0],\n",
       "  ['Waldau',\n",
       "   9,\n",
       "   'Waldau LRB Kassel RRB , in Northern Hesse , near the border with Lower Saxony and Thuringia , Germany',\n",
       "   0],\n",
       "  ['Waldau',\n",
       "   11,\n",
       "   'Waldau LRB Titisee Neustadt RRB , a village in Baden Württemberg , Germany',\n",
       "   0],\n",
       "  ['Waldau',\n",
       "   13,\n",
       "   'Waldau LRB Victoria RRB , a suburb of Melbourne , Australia',\n",
       "   0],\n",
       "  ['Waldau',\n",
       "   15,\n",
       "   'Waldau LRB Vojvodina RRB , the German name for the village of Sonta , Serbia',\n",
       "   0],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   0,\n",
       "   'Nikolaj Coster Waldau LRB LSB neɡ̊olaɪ̯ kʰʌsd̥ɐ ˈʋald̥ɑʊ̯ RSB ; born 27 July 1970 RRB is a Danish actor , producer and screenwriter .',\n",
       "   0],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   1,\n",
       "   'He graduated from Danish National School of Theatre in Copenhagen in 1993 .',\n",
       "   0],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   2,\n",
       "   \"Coster Waldau 's breakthrough performance in Denmark was his role in the film Nightwatch LRB 1994 RRB .\",\n",
       "   0],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   3,\n",
       "   'Since then he has appeared in numerous films in his native Scandinavia and Europe in general , including Headhunters LRB 2011 RRB and A Thousand Times Good Night LRB 2013 RRB .',\n",
       "   0],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   6,\n",
       "   'In the United States , his debut film role was in the war film Black Hawk Down LRB 2001 RRB , playing Medal of Honor recipient Gary Gordon .',\n",
       "   0],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   7,\n",
       "   'He then played Detective John Amsterdam in the short lived Fox television series New Amsterdam LRB 2008 RRB , as well as appearing as Frank Pike in the 2009 Fox television film Virtuality , originally intended as a pilot .',\n",
       "   1],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   8,\n",
       "   'He became widely known to a broad audience for his current role as Ser Jaime Lannister , in the HBO series Game of Thrones .',\n",
       "   0],\n",
       "  ['Nikolaj_Coster-Waldau',\n",
       "   9,\n",
       "   'In 2017 , he became one of the highest paid actors on television and earned 2 million per episode of Game of Thrones .',\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   0,\n",
       "   'The Fox Broadcasting Company LRB often shortened to Fox and stylized as FOX RRB is an American English language commercial broadcast television network that is owned by the Fox Entertainment Group subsidiary of 21st Century Fox .',\n",
       "   2],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   1,\n",
       "   'The network is headquartered at the 20th Century Fox studio lot on Pico Boulevard in the Century City section of Los Angeles , with additional major offices and production facilities at the Fox Television Center in nearby West Los Angeles and the Fox Broadcasting Center in the Yorkville neighborhood of Manhattan , New York City .',\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   2,\n",
       "   'It is the third largest major television network in the world based on total revenues , assets , and international coverage .',\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   5,\n",
       "   'Launched on October 9 , 1986 as a competitor to the Big Three television networks LRB ABC , NBC and CBS RRB , Fox went on to become the most successful attempt at a fourth television network .',\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   6,\n",
       "   'It was the highest rated broadcast network in the 18 49 demographic from 2004 to 2012 , and earned the position as the most watched American television network in total viewership during the 2007 08 season .',\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   9,\n",
       "   'Fox and its affiliated companies operate many entertainment channels in international markets , although these do not necessarily air the same programming as the U.S. network .',\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   10,\n",
       "   \"Most viewers in Canada have access to at least one U.S. based Fox affiliate , either over the air or through a pay television provider , although Fox 's National Football League telecasts and most of its prime time programming are subject to simultaneous substitution regulations for cable and satellite providers imposed by the Canadian Radio television and Telecommunications Commission LRB CRTC RRB to protect rights held by domestically based networks .\",\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   13,\n",
       "   \"The network is named after sister company 20th Century Fox , and indirectly for producer William Fox , who founded one of the movie studio 's predecessors , Fox Film .\",\n",
       "   0],\n",
       "  ['Fox_Broadcasting_Company',\n",
       "   14,\n",
       "   'Fox is a member of the North American Broadcasters Association and the National Association of Broadcasters .',\n",
       "   0],\n",
       "  ['Waldau_-LRB-surname-RRB-', 0, 'Waldau is a surname .', 0],\n",
       "  ['Waldau_-LRB-surname-RRB-',\n",
       "   1,\n",
       "   'Notable people with the surname include :',\n",
       "   0],\n",
       "  ['Waldau_-LRB-surname-RRB-',\n",
       "   4,\n",
       "   'Åsa Waldau LRB born 1965 RRB , Swedish religious leader',\n",
       "   0],\n",
       "  ['Waldau_-LRB-surname-RRB-',\n",
       "   6,\n",
       "   'Harry Waldau LRB 1876 1943 RRB , German musician',\n",
       "   0],\n",
       "  ['Waldau_-LRB-surname-RRB-',\n",
       "   8,\n",
       "   'Max Waldau LRB 1825 1855 RRB , German poet and writer',\n",
       "   0],\n",
       "  ['Waldau_-LRB-surname-RRB-', 10, 'Paul Waldau , American academic', 0]],\n",
       " 'label': 'SUPPORTS'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_list[0]\n",
    "#train_list[0]['evidence'][9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1543dd0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent1_list_train = [] #sent1: evidence candidate\n",
    "sent2_list_train = [] #sent2: claim\n",
    "label_list_train = [] #label: 1 if is evidence, else 0\n",
    "\n",
    "for data in train_list:\n",
    "    sent2 = data['claim']\n",
    "    for evidence in data['evidence']:\n",
    "        sent1 = evidence[2]\n",
    "        sent1_list_train.append(sent1)\n",
    "        sent2_list_train.append(sent2)\n",
    "        label = evidence[3]\n",
    "        if label == 0:\n",
    "            label_list_train.append(0)\n",
    "        else:\n",
    "            label_list_train.append(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "087baa56",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sent1_list_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c25314bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sent2_list_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89085f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "#label_list_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ed4a7e32",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent1_list_dev = [] #sent1: evidence candidate\n",
    "sent2_list_dev = [] #sent2: claim\n",
    "label_list_dev = [] #label: 1 if is evidence, else 0\n",
    "\n",
    "for data in dev_list:\n",
    "    sent2 = data['claim']\n",
    "    for evidence in data['evidence']:\n",
    "        sent1 = evidence[2]\n",
    "        sent1_list_dev.append(sent1)\n",
    "        sent2_list_dev.append(sent2)\n",
    "        label = evidence[3]\n",
    "        if label == 0:\n",
    "            label_list_dev.append(0)\n",
    "        else:\n",
    "            label_list_dev.append(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8957e195",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sent2_list_dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "82000216",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenizer \n",
    "model_name = 'bert-base-uncased'\n",
    "#model_name = 'albert-base-v2'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "b1aa7908",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_encodings = tokenizer(sent1_list_train, sent2_list_train, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "dev_encodings = tokenizer(sent1_list_dev, sent2_list_dev, padding=True, truncation=True, return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8d975f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeverDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7276b54b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = FeverDataset(train_encodings, label_list_train)\n",
    "dev_dataset = FeverDataset(dev_encodings, label_list_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4e2d263e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-15-a5eb445949a7>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([  101,  2002,  2059,  2209,  6317,  2198,  7598,  1999,  1996,  2460,\n",
       "          2973,  4419,  2547,  2186,  2047,  7598,  1048, 15185,  2263, 25269,\n",
       "          2497,  1010,  2004,  2092,  2004,  6037,  2004,  3581, 12694,  1999,\n",
       "          1996,  2268,  4419,  2547,  2143,  7484,  3012,  1010,  2761,  3832,\n",
       "          2004,  1037,  4405,  1012,   102, 24794,  3501,  3465,  2121,  1011,\n",
       "         24547,  2850,  2226,  2499,  2007,  1996,  4419,  5062,  2194,  1012,\n",
       "           102,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0]),\n",
       " 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " 'labels': tensor(1)}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset[13]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72e455b5",
   "metadata": {},
   "source": [
    "### train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cb4f42e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "model = BertForSequenceClassification.from_pretrained(model_name, num_labels=2)\n",
    "#model = AlbertForSequenceClassification.from_pretrained(model_name, num_labels=2)\n",
    "model.to(device)\n",
    "model.train()\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "optim = AdamW(model.parameters(), lr=2e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8a334889",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start training...................................................\n",
      "epoch:  0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-15-a5eb445949a7>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training time:  247.60690212249756 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "print ('start training...................................................')\n",
    "\n",
    "for epoch in range(1):\n",
    "    print(\"epoch: \", epoch)\n",
    "    for batch in train_loader:\n",
    "        #print(\"batch\")\n",
    "        optim.zero_grad()\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        token_type_ids = batch['token_type_ids'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        outputs = model(input_ids, \n",
    "                        attention_mask=attention_mask,\n",
    "                        token_type_ids=token_type_ids,\n",
    "                        labels=labels)\n",
    "        loss = outputs[0]\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        \n",
    "end = time.time()\n",
    "time_to_train = end-start\n",
    "print('training time: ', time_to_train, 'seconds')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5df9c461",
   "metadata": {},
   "source": [
    "### pred the top 5 sentences for each claim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "73f97f46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "55ed1e80",
   "metadata": {},
   "outputs": [],
   "source": [
    "#inputs = train_encodings\n",
    "inputs = dev_encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "bbf96089",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = model(**inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "71f32d8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "classification_logits = outputs.logits.detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "c09dc186",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28, 2)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classification_logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95c8c697",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_sent_train = []\n",
    "\n",
    "n = 0\n",
    "for claim in train_list:\n",
    "    sent_top5 = {'id':claim['id'], 'label':claim['label'], \"claim\":claim['claim'], 'pred.sent':[]}\n",
    "    sent_with_Score = []\n",
    "    for i in range(len(claim['evidence'])):\n",
    "        score = classification_logits[n][1]\n",
    "        sent_with_Score.append([score,claim['evidence'][i]])\n",
    "        n = n+1\n",
    "        sent_top5['pred.sent'] = sorted(sent_with_Score, key=lambda t: t[0], reverse=True)[:5]\n",
    "    pred_sent_dev.append(sent_top5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dd7b86c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_sent_dev = []\n",
    "\n",
    "n = 0\n",
    "for claim in dev_list:\n",
    "    sent_top5 = {'id':claim['id'], 'label':claim['label'], \"claim\":claim['claim'], 'pred.sent':[]}\n",
    "    sent_with_Score = []\n",
    "    for i in range(len(claim['evidence'])):\n",
    "        score = classification_logits[n][1]\n",
    "        sent_with_Score.append([score,claim['evidence'][i]])\n",
    "        n = n+1\n",
    "        sent_top5['pred.sent'] = sorted(sent_with_Score, key=lambda t: t[0], reverse=True)[:5]\n",
    "    pred_sent_dev.append(sent_top5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "dbb5c30f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 91198,\n",
       "  'label': 91198,\n",
       "  'claim': 'Colin Kaepernick became a starting quarterback during the 49ers 63rd season in the National Football League.',\n",
       "  'pred.sent': [[0.009892965,\n",
       "    ['National_League_-LRB-English_football-RRB-',\n",
       "     2,\n",
       "     'Between 1986 and 2015 , the league was known as the Football Conference .',\n",
       "     0]],\n",
       "   [-0.0075979084,\n",
       "    ['National_League_-LRB-English_football-RRB-',\n",
       "     9,\n",
       "     'The National League North and National League South form the sixth tier of English football .',\n",
       "     0]],\n",
       "   [-0.019916832,\n",
       "    ['National_League_-LRB-English_football-RRB-',\n",
       "     7,\n",
       "     'The professional clubs are usually clubs which have been in the English Football League LRB EFL RRB in the past , as opposed to those who have always been non League .',\n",
       "     0]],\n",
       "   [-0.022101209,\n",
       "    ['National_League_-LRB-English_football-RRB-',\n",
       "     3,\n",
       "     'As part of a sponsorship deal with car leasing company Vanarama , the league is known as the Vanarama National League .',\n",
       "     0]],\n",
       "   [-0.02651316,\n",
       "    ['National_League_-LRB-English_football-RRB-',\n",
       "     1,\n",
       "     'It was called the Alliance Premier League from 1979 until 1986 .',\n",
       "     0]]]}]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_sent_dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "ad598714",
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open('output_sent-retrieval/bert/sentences.predicted.train.jsonl', 'w') as fp:\n",
    "#with open('output_sent-retrieval/albert/sentences.predicted.train.jsonl', 'w') as fp:\n",
    "with open('output_sent-retrieval/bert/sentences.predicted.dev.jsonl', 'w') as fp:\n",
    "#with open('output_sent-retrieval/albert/sentences.predicted.dev.jsonl', 'w') as fp:\n",
    "    \n",
    "    #json.dump(str(pred_sent_train), fp)\n",
    "    json.dump(str(pred_sent_dev), fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ef328bf",
   "metadata": {},
   "source": [
    "# claim verification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "54394e6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the top 5 sent for each claim\n",
    "\n",
    "sent_train_list = list()\n",
    "with open('output_sent-retrieval/bert/sentences.predicted.train.jsonl', 'r') as sent_train:\n",
    "#with open('output_sent-retrieval/albert/sentences.predicted.train.jsonl', 'r') as sent_train:\n",
    "\n",
    "    for example in sent_train:\n",
    "        instance = json.loads(example.strip())\n",
    "        sent_train_list.append(instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5be14a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sent_train_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "792d6dd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_dev_list = list()\n",
    "with open('output_sent-retrieval/bert/sentences.predicted.dev.jsonl', 'r') as sent:\n",
    "#with open('output_sent-retrieval/albert/sentences.predicted.dev.jsonl', 'r') as sent:\n",
    "\n",
    "    for example in sent:\n",
    "        instance = json.loads(example.strip())\n",
    "        sent_dev_list.append(instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "d3458ff4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 91198,\n",
       " 'verifiable': 'NOT VERIFIABLE',\n",
       " 'label': 'NOT ENOUGH INFO',\n",
       " 'claim': 'Colin Kaepernick became a starting quarterback during the 49ers 63rd season in the National Football League.',\n",
       " 'evidence': [[[108548, None, None, None]]],\n",
       " 'predicted_sentences': [[0.7208946,\n",
       "   ['Colin_Kaepernick',\n",
       "    6,\n",
       "    \"He remained the team 's starting quarterback for the rest of the season and went on to lead the 49ers to their first Super Bowl appearance since 1994 , losing to the Baltimore Ravens .\"]],\n",
       "  [0.06825966,\n",
       "   ['Colin_Kaepernick',\n",
       "    0,\n",
       "    'Colin Rand Kaepernick -LRB- -LSB- ` kæpərnɪk -RSB- ; born November 3 , 1987 -RRB- is an American football quarterback who is currently a free agent .']],\n",
       "  [0.06412292,\n",
       "   ['Colin_Kaepernick',\n",
       "    8,\n",
       "    'In the following seasons , Kaepernick lost and won back his starting job , with the 49ers missing the playoffs for three years consecutively .']],\n",
       "  [0.06392619,\n",
       "   ['Colin_Kaepernick',\n",
       "    5,\n",
       "    \"Kaepernick began his professional career as a backup to Alex Smith , but became the 49ers ' starter in the middle of the 2012 season after Smith suffered a concussion .\"]],\n",
       "  [0.04323595,\n",
       "   ['National_Football_League',\n",
       "    0,\n",
       "    'The National Football League -LRB- NFL -RRB- is a professional American football league consisting of 32 teams , divided equally between the National Football Conference -LRB- NFC -RRB- and the American Football Conference -LRB- AFC -RRB- .']]]}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_dev_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "074192a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_train_mini = sent_train_list[:5]\n",
    "sent_dev_mini = sent_dev_list[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "67c8e3e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_list = all_train_list\n",
    "#dev_list = all_dev_list\n",
    "\n",
    "train_list = sent_train_mini\n",
    "dev_list = sent_dev_mini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "d714f479",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent1_list_train = []\n",
    "sent2_list_train = []\n",
    "label_list_train = []\n",
    "\n",
    "for data in train_list:\n",
    "    label = data['label']\n",
    "    sent2 = data['claim']\n",
    "    for evidence in data['predicted_sentences']:\n",
    "        sent1 = evidence[1][2]\n",
    "        sent1_list_train.append(sent1)\n",
    "        sent2_list_train.append(sent2)\n",
    "        if label == \"NOT ENOUGH INFO\" :\n",
    "            label_list_train.append(0)\n",
    "        if label == \"REFUTES\":\n",
    "            label_list_train.append(1)\n",
    "        if label == \"SUPPORTS\":\n",
    "            label_list_train.append(2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "cbf0ab0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sent1_list_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "225af20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent1_list_dev = []\n",
    "sent2_list_dev = []\n",
    "label_list_dev = []\n",
    "\n",
    "for data in dev_list:\n",
    "    label = data['label']\n",
    "    sent2 = data['claim']\n",
    "    for evidence in data['predicted_sentences']:\n",
    "        sent1 = evidence[1][2]\n",
    "        sent1_list_dev.append(sent1)\n",
    "        sent2_list_dev.append(sent2)\n",
    "        if label == \"NOT ENOUGH INFO\" :\n",
    "            label_list_dev.append(0)\n",
    "        if label == \"REFUTES\":\n",
    "            label_list_dev.append(1)\n",
    "        if label == \"SUPPORTS\":\n",
    "            label_list_dev.append(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "49c2c1e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"He remained the team 's starting quarterback for the rest of the season and went on to lead the 49ers to their first Super Bowl appearance since 1994 , losing to the Baltimore Ravens .\",\n",
       " 'Colin Rand Kaepernick -LRB- -LSB- ` kæpərnɪk -RSB- ; born November 3 , 1987 -RRB- is an American football quarterback who is currently a free agent .',\n",
       " 'In the following seasons , Kaepernick lost and won back his starting job , with the 49ers missing the playoffs for three years consecutively .',\n",
       " \"Kaepernick began his professional career as a backup to Alex Smith , but became the 49ers ' starter in the middle of the 2012 season after Smith suffered a concussion .\",\n",
       " 'The National Football League -LRB- NFL -RRB- is a professional American football league consisting of 32 teams , divided equally between the National Football Conference -LRB- NFC -RRB- and the American Football Conference -LRB- AFC -RRB- .']"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent1_list_dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "26f16f54",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenizer \n",
    "model_name = 'bert-base-uncased'\n",
    "#model_name = 'albert-base-v2'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "2cf08468",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_encodings = tokenizer(sent1_list_train, sent2_list_train, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "dev_encodings = tokenizer(sent1_list_dev, sent2_list_dev, padding=True, truncation=True, return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e608cb50",
   "metadata": {},
   "source": [
    "### train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "46639241",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = FeverDataset(train_encodings, label_list_train)\n",
    "dev_dataset = FeverDataset(dev_encodings, label_list_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "47c8ccab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "model = BertForSequenceClassification.from_pretrained(model_name, num_labels=3)\n",
    "#model = AlbertForSequenceClassification.from_pretrained(model_name, num_labels=3)\n",
    "model.to(device)\n",
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "a5c84c7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "\n",
    "optim = AdamW(model.parameters(), lr=2e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "10013ae7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start training...................................................\n",
      "epoch\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-15-a5eb445949a7>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training time:  19.757014751434326 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "print ('start training...................................................')\n",
    "\n",
    "for epoch in range(1):\n",
    "    print(\"epoch\")\n",
    "    for batch in train_loader:\n",
    "        optim.zero_grad()\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        token_type_ids = batch['token_type_ids'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        outputs = model(input_ids, \n",
    "                        attention_mask=attention_mask,\n",
    "                        token_type_ids=token_type_ids,\n",
    "                        labels=labels)\n",
    "        loss = outputs[0]\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        \n",
    "end = time.time()\n",
    "time_to_train = end-start\n",
    "print('training time: ', time_to_train, 'seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "10b75a77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "62f5cc95",
   "metadata": {},
   "outputs": [],
   "source": [
    "#inputs = train_encodings\n",
    "inputs = dev_encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "426f75d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = model(**inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "911116a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.50237155, -0.06594548,  0.03114632],\n",
       "       [ 0.5220732 , -0.03692114,  0.04443705],\n",
       "       [ 0.5190381 , -0.04177365,  0.03736544],\n",
       "       [ 0.52316904, -0.04150811,  0.04119575],\n",
       "       [ 0.1020293 , -0.24614841, -0.22621267]], dtype=float32)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#train_classification_logits = outputs.logits.detach().numpy()\n",
    "dev_classification_logits = outputs.logits.detach().numpy()\n",
    "\n",
    "#train_classification_logits\n",
    "dev_classification_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "1fc9dd7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#predictions = np.argmax(train_classification_logits, axis=1)\n",
    "predictions = np.argmax(dev_classification_logits, axis=1)\n",
    "\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "a8674c67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 91198,\n",
       "  'verifiable': 'NOT VERIFIABLE',\n",
       "  'label': 'NOT ENOUGH INFO',\n",
       "  'claim': 'Colin Kaepernick became a starting quarterback during the 49ers 63rd season in the National Football League.',\n",
       "  'evidence': [[[108548, None, None, None]]],\n",
       "  'predicted_sentences': [[0.7208946,\n",
       "    ['Colin_Kaepernick',\n",
       "     6,\n",
       "     \"He remained the team 's starting quarterback for the rest of the season and went on to lead the 49ers to their first Super Bowl appearance since 1994 , losing to the Baltimore Ravens .\"]],\n",
       "   [0.06825966,\n",
       "    ['Colin_Kaepernick',\n",
       "     0,\n",
       "     'Colin Rand Kaepernick -LRB- -LSB- ` kæpərnɪk -RSB- ; born November 3 , 1987 -RRB- is an American football quarterback who is currently a free agent .']],\n",
       "   [0.06412292,\n",
       "    ['Colin_Kaepernick',\n",
       "     8,\n",
       "     'In the following seasons , Kaepernick lost and won back his starting job , with the 49ers missing the playoffs for three years consecutively .']],\n",
       "   [0.06392619,\n",
       "    ['Colin_Kaepernick',\n",
       "     5,\n",
       "     \"Kaepernick began his professional career as a backup to Alex Smith , but became the 49ers ' starter in the middle of the 2012 season after Smith suffered a concussion .\"]],\n",
       "   [0.04323595,\n",
       "    ['National_Football_League',\n",
       "     0,\n",
       "     'The National Football League -LRB- NFL -RRB- is a professional American football league consisting of 32 teams , divided equally between the National Football Conference -LRB- NFC -RRB- and the American Football Conference -LRB- AFC -RRB- .']]]}]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#train_list\n",
    "dev_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d969b2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "claim_veri_train = []\n",
    "\n",
    "n = 0\n",
    "for claim in train_list:\n",
    "    claim_veri_pred = {'id':claim['id'], 'pred_label':'REFUTES', \"claim\":claim['claim']}\n",
    "    \n",
    "    for i in range(len(claim['predicted_sentences'])):\n",
    "        top5_predictions = []\n",
    "        if predictions[n] == 0:\n",
    "            pred_label = \"NOT ENOUGH INFO\"\n",
    "        if predictions[n] == 1:\n",
    "            pred_label = \"REFUTES\"\n",
    "        if predictions[n] == 2:\n",
    "            pred_label = \"SUPPORTS\"\n",
    "        \n",
    "        top5_predictions.append(pred_label)\n",
    "        counter =  Counter(top5_predictions)\n",
    "        pred_label = counter.most_common(1)[0][0] \n",
    "        \n",
    "        claim_veri_pred['pred_label'] = pred_label\n",
    "        \n",
    "        n = n+1\n",
    "        \n",
    "    claim_veri_train.append(claim_veri_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "63383d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "claim_veri_dev = []\n",
    "\n",
    "n = 0\n",
    "for claim in dev_list:\n",
    "    claim_veri_pred = {'id':claim['id'], 'pred_label':'REFUTES', \"claim\":claim['claim']}\n",
    "    \n",
    "    for i in range(len(claim['predicted_sentences'])):\n",
    "        top5_predictions = []\n",
    "        if predictions[n] == 0:\n",
    "            pred_label = \"NOT ENOUGH INFO\"\n",
    "        if predictions[n] == 1:\n",
    "            pred_label = \"REFUTES\"\n",
    "        if predictions[n] == 2:\n",
    "            pred_label = \"SUPPORTS\"\n",
    "        \n",
    "        top5_predictions.append(pred_label)\n",
    "        counter =  Counter(top5_predictions)\n",
    "        pred_label = counter.most_common(1)[0][0] \n",
    "        \n",
    "        claim_veri_pred['pred_label'] = pred_label\n",
    "        \n",
    "        n = n+1\n",
    "        \n",
    "    claim_veri_dev.append(claim_veri_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "48743685",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 91198,\n",
       "  'pred_label': 'NOT ENOUGH INFO',\n",
       "  'claim': 'Colin Kaepernick became a starting quarterback during the 49ers 63rd season in the National Football League.'}]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#claim_veri_train\n",
    "claim_veri_dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "94c4d4c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open('output_claim-verification/bert/claims.predicted.train.jsonl.jsonl', 'w') as fp:\n",
    "#with open('output_claim-verification/albert/claims.predicted.train.jsonl.jsonl', 'w') as fp:\n",
    "with open('output_claim-verification/bert/claims.predicted.dev.jsonl.jsonl', 'w') as fp:\n",
    "#with open('output_claim-verification/albert/claims.predicted.dev.jsonl.jsonl', 'w') as fp:\n",
    "\n",
    "    #json.dump(str(claim_veri_train), fp)\n",
    "    json.dump(str(claim_veri_dev), fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e159b1b",
   "metadata": {},
   "source": [
    "# compare the dev results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "8b4f4f8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_output = []\n",
    "\n",
    "with open('actual_output/shared_task_dev.jsonl') as output:\n",
    "    \n",
    "    for step, line in enumerate(output):\n",
    "        instance = json.loads(line.strip())\n",
    "        actual_output.append(instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "5878e2c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_bert = []\n",
    "\n",
    "with open('output_claim-verification/bert/claims.predicted.dev.jsonl') as output:\n",
    "    \n",
    "    for step, line in enumerate(output):\n",
    "        instance = json.loads(line.strip())\n",
    "        dev_bert.append(instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "3abce3dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_albert = []\n",
    "\n",
    "with open('output_claim-verification/albert/claims.predicted.dev.jsonl') as output:\n",
    "    \n",
    "    for step, line in enumerate(output):\n",
    "        instance = json.loads(line.strip())\n",
    "        dev_albert.append(instance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "8434342d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = PrettyTable()\n",
    "tab.field_names = [\"Model Name\", \"FEVER Score\", \"Label Accuracy\", \"Evidence Precision\", \"Evidence Recall\", \"Evidence F1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "7421d735",
   "metadata": {},
   "outputs": [],
   "source": [
    "score,acc,precision,recall,f1 = fever_score(dev_bert, actual_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "c03d76d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "tab.add_row(('BERT', round(score,4),round(acc,4),round(precision,4),round(recall,4),round(f1,4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "861384bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "score,acc,precision,recall,f1 = fever_score(dev_albert, actual_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "09a4b9fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "tab.add_row(('ALBERT', round(score,4),round(acc,4),round(precision,4),round(recall,4),round(f1,4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "b9015c85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+-------------+----------------+--------------------+-----------------+-------------+\n",
      "| Model Name | FEVER Score | Label Accuracy | Evidence Precision | Evidence Recall | Evidence F1 |\n",
      "+------------+-------------+----------------+--------------------+-----------------+-------------+\n",
      "|    BERT    |    0.6898   |     0.7407     |       0.8931       |      0.7045     |    0.7877   |\n",
      "|   ALBERT   |    0.6054   |     0.6564     |       0.859        |      0.6072     |    0.7115   |\n",
      "+------------+-------------+----------------+--------------------+-----------------+-------------+\n"
     ]
    }
   ],
   "source": [
    "print(tab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d33c5f7e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}