QAMigration(CHIP-STS).ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.5 64-bit"
  },
  "interpreter": {
   "hash": "f82ab70f6e4e0673e87f3f6960512c9807b8b91e8473aea7edbc34f463490a2f"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "## 1.任务描述\n",
    "迁移学习是自然语言处理中的重要一环，其主要目的是通过从已学习的相关任务中转移知识来改进新任务的学习效果，从而提高模型的泛化能力。\n",
    "本次评测任务的主要目标是针对中文的疾病问答数据，进行病种间的迁移学习。具体而言，给定来自5个不同病种的问句对，要求判定两个句子语义是否相同或者相近，并在CHIP2019会议发布了评测任务(http://cips-chip.org.cn/)。\n",
    "\n",
    "## 2.任务说明\n",
    "category表示问句对的病种名称，分别对应：diabetes-糖尿病，hypertension-高血压，hepatitis-乙肝，aids-艾滋病，breast_cancer-乳腺癌。label表示问句之间的语义是否相同。若相同，标为1，若不相同，标为0。\n",
    "### 标注示例如下：\n",
    "category: diabetes\n",
    "问句1：糖尿病吃什么？\n",
    "问句2：糖尿病的食谱？\n",
    "label:1\n",
    "\n",
    "category: hepatitis\n",
    "问句1：乙肝小三阳的危害？\n",
    "问句2：乙肝大三阳的危害？\n",
    "label:0\n",
    "## 3.评测指标\n",
    "同CHIP-CTC任务，本任务的评价指标使用宏观F1值(Macro-F1，或称Average-F1)。\n",
    "\n",
    "## 4.评测数据\n",
    "本评测开放训练集数据16000条，验证集数据4000条，测试集数据10000条（注：榜单的训练数据和验证集来自原CHIP评测任务的训练集，榜单的测试数据10000条来自CHIP评测任务的B榜）。\n",
    "\n",
    "## 说明\n",
    "处理步骤基本上和KUAKE-QTR完全相同，不加赘述"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cut by word\n",
    "def tokenize(text):\n",
    "    return list(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word to sequence\n",
    "UNK_TAG = \"UNK\"\n",
    "PAD_TAG = \"PAD\"\n",
    "class Word2Sequence():\n",
    "    UNK = 0\n",
    "    PAD = 1\n",
    "\n",
    "    def __init__(self):\n",
    "        self.word2index_dict = {\n",
    "            UNK_TAG : self.UNK,\n",
    "            PAD_TAG : self.PAD,\n",
    "        }\n",
    "        self.count = {}\n",
    "\n",
    "\n",
    "    def fit(self, sentence):\n",
    "        # 保存句子到dict, 统计词频\n",
    "        for word in sentence:\n",
    "            self.count[word] = self.count.get(word, 0) + 1\n",
    "        \n",
    "\n",
    "    def build_vocab(self,min=0,max=None,max_features=None):\n",
    "        self.count = {word:value for word,value in self.count.items() if value > min}\n",
    "        if(max is not None):\n",
    "            self.count = {word:value for word,value in self.count.items() if value < max}\n",
    "        if max_features is not None:\n",
    "            self.count = dict(sorted(self.count.items(), key = lambda x:x[-1], reverse=True)[:max_features])\n",
    "\n",
    "        for word in self.count:\n",
    "            self.word2index_dict[word] = len(self.word2index_dict)\n",
    "        self.index2word_dict = dict(zip(self.word2index_dict.values(), self.word2index_dict.keys()))\n",
    "\n",
    "\n",
    "    def words2index_transform(self, sentence, max_len=None):\n",
    "        if max_len is not None:\n",
    "            if max_len > len(sentence):\n",
    "                sentence = sentence + [PAD_TAG] * (max_len - len(sentence))\n",
    "            else:\n",
    "                sentence = sentence[:max_len]\n",
    "        return [self.word2index_dict.get(word, self.UNK) for word in sentence]\n",
    "\n",
    "\n",
    "    def index2words_transform(self, sentence):\n",
    "        return [self.index2word_dict.get(index) for index in sentence]\n",
    "\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.word2index_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n2148\n"
     ]
    }
   ],
   "source": [
    "# dictionary build\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "import os\n",
    "\n",
    "train_data_path = r\"data\\CHIP-STS\\CHIP-STS_train.json\"\n",
    "test_data_path = r\"data\\CHIP-STS\\CHIP-STS_test.json\"\n",
    "dev_data_path = r\"data\\CHIP-STS\\CHIP-STS_dev.json\"\n",
    "if(not os.path.exists(\"models/CHIP-STS_Word2Sequence.pkl\")):\n",
    "    word_index_tranformer = Word2Sequence()\n",
    "    with open(train_data_path, encoding=\"utf-8\") as f:\n",
    "        for data in tqdm(json.load(f)):\n",
    "            word_index_tranformer.fit(tokenize(data['text1']))\n",
    "            word_index_tranformer.fit(tokenize(data['text2']))\n",
    "    word_index_tranformer.build_vocab()\n",
    "    pickle.dump(word_index_tranformer, open(r\"models/CHIP-STS_Word2Sequence.pkl\", 'wb'))\n",
    "else:\n",
    "    word_index_tranformer = pickle.load(open(r\"models/CHIP-STS_Word2Sequence.pkl\", 'rb'))\n",
    "print('\\n' + str(len(word_index_tranformer)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(0, tensor([ 2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,  1,  1,  1,  1,\n         1,  1,  1,  1,  1]), tensor([16, 17, 11, 12, 18, 19, 20, 21, 22, 23, 22,  2,  3,  4,  1,  1,  1,  1,\n         1,  1,  1,  1,  1])) 16000\n"
     ]
    }
   ],
   "source": [
    "# dataset\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "import json\n",
    "\n",
    "max_sentece_length = 23\n",
    "class RosDataset(Dataset):\n",
    "    def __init__(self, data_path, train=True):\n",
    "        self.train = train\n",
    "        with open(data_path, encoding=\"utf-8\") as f:\n",
    "            self.data_list = json.load(f)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        # 获取索引对应位置的一条数据\n",
    "        cuted_text1 = tokenize(self.data_list[index][\"text1\"])\n",
    "        cuted_text2 = tokenize(self.data_list[index][\"text2\"])\n",
    "        indexed_text1 = torch.LongTensor(word_index_tranformer.words2index_transform(cuted_text1, max_len=max_sentece_length))\n",
    "        indexed_text2 = torch.LongTensor(word_index_tranformer.words2index_transform(cuted_text2, max_len=max_sentece_length))\n",
    "        if(self.train):\n",
    "            label = int(self.data_list[index][\"label\"])\n",
    "            return label, indexed_text1, indexed_text2\n",
    "        else:\n",
    "            return indexed_text1, indexed_text2\n",
    "\n",
    "    def __len__(self):\n",
    "        # 返回数据的总数量\n",
    "        return len(self.data_list)\n",
    "\n",
    "train_dataset = RosDataset(train_data_path)\n",
    "dev_dataset = RosDataset(dev_data_path)\n",
    "test_dataset = RosDataset(test_data_path, train=False)\n",
    "print(train_dataset[0], len(train_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0:tensor([1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1,\n        1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0,\n        1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0,\n        1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1,\n        1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1,\n        0, 0, 0, 0, 1, 1, 1, 1]),tensor([[175, 176, 442,  ...,   1,   1,   1],\n        [ 26,  27,   4,  ...,   1,   1,   1],\n        [ 47,  48,  49,  ...,   1,   1,   1],\n        ...,\n        [210, 211, 542,  ...,   1,   1,   1],\n        [ 26,  27,   4,  ...,   1,   1,   1],\n        [210, 211, 212,  ...,   1,   1,   1]]),tensor([[ 26,  27,   4,  ...,   1,   1,   1],\n        [ 26,  27,   4,  ...,   1,   1,   1],\n        [344, 142,  47,  ...,   1,   1,   1],\n        ...,\n        [210, 211, 542,  ...,   1,   1,   1],\n        [ 26,  27,   4,  ...,   1,   1,   1],\n        [452,   7,  41,  ...,   1,   1,   1]])\n"
     ]
    }
   ],
   "source": [
    "# dataloader\n",
    "from torch.utils.data import DataLoader\n",
    "import torch\n",
    "\n",
    "train_data_loader = DataLoader(dataset=train_dataset,batch_size=128,shuffle=True)\n",
    "dev_data_loader = DataLoader(dataset=dev_dataset,batch_size=1,shuffle=True)\n",
    "test_data_loader = DataLoader(dataset=test_dataset,batch_size=1,shuffle=False, drop_last=False)\n",
    "for index, (label, indexed_text1, indexed_text2) in enumerate(train_data_loader):\n",
    "    if(index > 0):\n",
    "        break\n",
    "    print(f\"{index}:{label},{indexed_text1},{indexed_text2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Siamese network\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch\n",
    "\n",
    "class SiameseNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SiameseNetwork, self).__init__()\n",
    "        self.embedding = nn.Embedding(num_embeddings=len(word_index_tranformer),embedding_dim=300,padding_idx=word_index_tranformer.PAD)\n",
    "        self.gru1 = nn.GRU(input_size=300,hidden_size=256,num_layers=2,batch_first=True,bidirectional=True)\n",
    "        self.gru2 = nn.GRU(input_size=256*4,hidden_size=256,num_layers=1,batch_first=True,bidirectional=False)\n",
    "        self.dnn = nn.Sequential(\n",
    "            nn.Linear(256*4,256),\n",
    "            nn.ELU(inplace=True),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.Dropout(0.3),\n",
    "\n",
    "            nn.Linear(256,256),\n",
    "            nn.ELU(inplace=True),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.Dropout(0.3),\n",
    "\n",
    "            nn.Linear(256, 2)\n",
    "        )\n",
    "\n",
    "\n",
    "    def forward(self, input1, input2):\n",
    "        mask1 = input1.eq(word_index_tranformer.PAD)\n",
    "        mask2 = input2.eq(word_index_tranformer.PAD)\n",
    "        input1 = self.embedding(input1)\n",
    "        input2 = self.embedding(input2)\n",
    "        output1,_ = self.gru1(input1)\n",
    "        output2,_ = self.gru1(input2)\n",
    "        \n",
    "        output1_align, output2_align = self.soft_attention_align(output1, output2, mask1, mask2)\n",
    "        output1 = torch.cat([output1, output1_align], 2)\n",
    "        output2 = torch.cat([output2, output2_align], 2)\n",
    "        \n",
    "        gru2_output1,_ = self.gru2(output1)\n",
    "        gru2_output2,_ = self.gru2(output2)\n",
    "        \n",
    "        output1_pooled = self.apply_pooling(gru2_output1)\n",
    "        output2_pooled = self.apply_pooling(gru2_output2)\n",
    "        out = torch.cat([output1_pooled, output2_pooled], dim=-1)\n",
    "        out = self.dnn(out)\n",
    "        \n",
    "        return F.log_softmax(out, dim=-1)\n",
    "\n",
    "\n",
    "    def apply_pooling(self, output):\n",
    "        avg_pooled = F.avg_pool1d(output.transpose(1,2), kernel_size=output.size(1)).squeeze(-1)\n",
    "        max_pooled = F.max_pool1d(output.transpose(1,2), kernel_size=output.size(1)).squeeze(-1)\n",
    "        return torch.cat([avg_pooled, max_pooled], dim=-1)\n",
    "\n",
    "\n",
    "    def soft_attention_align(self, x1, x2, mask1, mask2):\n",
    "        mask1 = mask1.float().masked_fill_(mask1, float(\"-inf\"))\n",
    "        mask2 = mask2.float().masked_fill_(mask2, float(\"-inf\"))\n",
    "\n",
    "        attention_weight = x1.bmm(x2.transpose(1, 2))\n",
    "        x1_weight = F.softmax(attention_weight + mask2.unsqueeze(1), dim=-1) \n",
    "        x2_output = x1_weight.bmm(x2)\n",
    "\n",
    "        x2_weight = F.softmax(attention_weight.transpose(1, 2) + mask1.unsqueeze(1), dim=-1) \n",
    "        x1_output = x2_weight.bmm(x1)\n",
    "        \n",
    "        return x1_output, x2_output"
   ]
  },
  {
   "source": [
    "## Siamese Network\n",
    "- embedding\n",
    "- BiGRU\n",
    "- attention\n",
    "- GRU\n",
    "- Pooling\n",
    "- FC+softmax\n",
    "## 说明\n",
    "- 该任务和KUAKE-QQR使用完全相同的模型\n",
    "- 该任务和KUAKE-QTR使用的模型大致相同\n",
    "    - 该任务使用的模型规模小的多\n",
    "        - 第二层GRU不再是双向的\n",
    "        - 第一层GRU为2层，而第二层GRU只有一层；相较之下KUAKE-QTR为6+3\n",
    "        - 增加了pooling理论上应该是缩小计算量，但实际上由于kernel size=1且做了cat，是增加了计算量\n",
    "    - 完全有理由相信在使用和KUAKE-QTR模型中采用的增加网络深度的方式可以得到十分明显的性能提升\n",
    "    - 但相应的代价是训练时间由几分钟涨到几十分钟\n",
    "- 需要特别说明的是，原任务要求利用领域迁移知识而给出了问题类别的标签，但这里完全没有用到\n",
    "    - 其实没有理解有了这个标签要怎么利用迁移知识\n",
    "    - 但或许可以在使用专门设计的loss function时排上一定用处"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from torch.optim import Adam\n",
    "\n",
    "\n",
    "def train(epochs, model, model_path=None, optimizer_path=None, device=None):\n",
    "    model = model.to(device)\n",
    "    model.train()\n",
    "    optimizer = Adam(model.parameters(), lr=0.001)\n",
    "    for epoch in tqdm(range(epochs), desc=\"Train\"):\n",
    "        for index, (label, text1, text2) in enumerate(train_data_loader):\n",
    "            if not device is None:\n",
    "                label = label.to(device)\n",
    "                text1 = text1.to(device)\n",
    "                text2 = text2.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            output = model(text1, text2)\n",
    "            loss = F.nll_loss(output, label)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "    \n",
    "    if not model_path is None:\n",
    "        torch.save(model.state_dict(), model_path)\n",
    "    if not optimizer_path is None:\n",
    "        torch.save(optimizer.state_dict(), optimizer_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluation_accuracy(model, test_data_loader, device=None):\n",
    "    count_correct = 0\n",
    "    model = model.to(device)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for label, text1, text2 in tqdm(dev_data_loader, desc=\"Evaluation\"):\n",
    "            if not device is None:\n",
    "                label = label.to(device)\n",
    "                text1 = text1.to(device)\n",
    "                text2 = text2.to(device)\n",
    "            if(model(text1, text2).argmax() == label):\n",
    "                count_correct = count_correct + 1\n",
    "    print(f\"\\n{count_correct}/{len(test_data_loader)}\")\n",
    "    return count_correct / len(test_data_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Train: 100%|██████████| 35/35 [04:26<00:00,  7.62s/it]\n",
      "Evaluation: 100%|██████████| 4000/4000 [00:45<00:00, 88.51it/s]\n",
      "3244/4000\n",
      "\n",
      "0.811\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_mode = True\n",
    "model = SiameseNetwork()\n",
    "model_path = \"models\\STS_SiameseNetwork.pth\"\n",
    "optimizer_path = \"models\\STS_SiameseNetwork_optim.pth\"\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "if(train_mode):\n",
    "    train(epochs=35, model=model, model_path=model_path, optimizer_path=optimizer_path, device=device)\n",
    "else:\n",
    "    model.load_state_dict(torch.load(model_path))\n",
    "\n",
    "print('\\n' + str(evaluation_accuracy(model, dev_data_loader, device=device)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Evaluation: 100%|██████████| 10000/10000 [04:53<00:00, 34.06it/s]\n"
     ]
    }
   ],
   "source": [
    "dump_file_path = \"result\\CHIP-STS_test.json\"\n",
    "with open(test_data_path,'r',encoding=\"utf-8\") as source:\n",
    "    data = json.load(source)\n",
    "    model = model.to(device)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for index, (text1, text2) in tqdm(enumerate(test_data_loader), desc=\"Evaluation\", total=len(test_data_loader)):\n",
    "            if not device is None:\n",
    "                text1 = text1.to(device)\n",
    "                text2 = text2.to(device)\n",
    "            data[index][\"label\"] = model(text1, text2).argmax().item()\n",
    "            json_result = json.dumps(data, ensure_ascii=False)\n",
    "\n",
    "with open(dump_file_path,'w',encoding=\"utf-8\") as destination:\n",
    "    destination.write(json_result)"
   ]
  }
 ]
}