train_mle_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "/home/galprz/miniconda3/envs/simv2/lib/python3.7/site-packages/librosa/util/decorators.py:9: NumbaDeprecationWarning: An import was requested from a module that has moved location.\n",
      "Import requested from: 'numba.decorators', please update to use 'numba.core.decorators' or pin to Numba version 0.48.0. This alias will not be present in Numba version 0.50.0.\n",
      "  from numba.decorators import jit as optional_jit\n",
      "/home/galprz/miniconda3/envs/simv2/lib/python3.7/site-packages/librosa/util/decorators.py:9: NumbaDeprecationWarning: An import was requested from a module that has moved location.\n",
      "Import of 'jit' requested from: 'numba.decorators', please update to use 'numba.core.decorators' or pin to Numba version 0.48.0. This alias will not be present in Numba version 0.50.0.\n",
      "  from numba.decorators import jit as optional_jit\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import logging\n",
    "from tensorboardX import SummaryWriter\n",
    "import onmt.translate\n",
    "from metrics import BleuScore, CorrectAnswersScore\n",
    "from trainer import SimMLETrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "2020-11-23 21:20:55,345 INFO cuda\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "mode=\"v2\"\n",
    "writer = SummaryWriter(comment=\"-argmax\")\n",
    "fmt = \"%(asctime)-15s %(levelname)s %(message)s\"\n",
    "logging.basicConfig(format=fmt, level=logging.INFO)\n",
    "scenario_number = 8\n",
    "vocab_fields = torch.load(f\"processed/SimQA.vocab.pt\")\n",
    "\n",
    "src_text_field = vocab_fields[\"src\"].base_field\n",
    "src_vocab = src_text_field.vocab\n",
    "src_padding = src_vocab.stoi[src_text_field.pad_token]\n",
    "\n",
    "tgt_text_field = vocab_fields['tgt'].base_field\n",
    "tgt_vocab = tgt_text_field.vocab\n",
    "tgt_padding = tgt_vocab.stoi[tgt_text_field.pad_token]\n",
    "\n",
    "emb_size = 200\n",
    "rnn_size = 200\n",
    "encoder_embeddings = onmt.modules.Embeddings(\n",
    "    emb_size,\n",
    "    len(src_vocab),\n",
    "    word_padding_idx=src_padding,\n",
    "    position_encoding=True,\n",
    "    fix_word_vecs=True)\n",
    "encoder_embeddings.load_pretrained_vectors(f\"src.embeddings.pt\")\n",
    "\n",
    "\n",
    "encoder = onmt.encoders.TransformerEncoder(\n",
    "    num_layers=6,\n",
    "    d_model=emb_size,\n",
    "    max_relative_positions=100,\n",
    "    embeddings=encoder_embeddings,\n",
    "    d_ff=1,\n",
    "    dropout=0.1,\n",
    "    attention_dropout=0.1,\n",
    "    heads=8\n",
    ")\n",
    "\n",
    "decoder_embeddings = onmt.modules.Embeddings(emb_size,\n",
    "                                             len(tgt_vocab),\n",
    "                                             word_padding_idx=tgt_padding,\n",
    "                                             position_encoding=True)\n",
    "decoder = onmt.decoders.TransformerDecoder(\n",
    "    num_layers=6,\n",
    "    d_model=emb_size,\n",
    "    embeddings=decoder_embeddings,\n",
    "    max_relative_positions=100,\n",
    "    aan_useffn=False,\n",
    "    full_context_alignment=True,\n",
    "    alignment_layer=1,\n",
    "    alignment_heads=1,\n",
    "    heads=8,\n",
    "    d_ff=1,\n",
    "    copy_attn=False,\n",
    "    self_attn_type=\"scaled-dot\",\n",
    "    dropout=0.1,\n",
    "    attention_dropout=0.1\n",
    ")\n",
    "\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "model = onmt.models.model.NMTModel(encoder, decoder)\n",
    "model.to(device)\n",
    "logging.getLogger().info(device)\n",
    "# Specify the tgt word generator and loss computation module\n",
    "model.generator = nn.Sequential(\n",
    "    nn.Linear(emb_size, len(tgt_vocab)),\n",
    "    nn.LogSoftmax(dim=-1)).to(device)\n",
    "\n",
    "#checkpoint = torch.load(\"./models/model_step_5000.pt\")\n",
    "#model.load_state_dict(checkpoint[\"model\"], strict=False)\n",
    "#model.generator.load_state_dict(checkpoint[\"generator\"], strict=False)\n",
    "\n",
    "import onmt.utils.loss\n",
    "loss = onmt.utils.loss.NMTLossCompute(\n",
    "    criterion=nn.NLLLoss(ignore_index=tgt_padding, reduction=\"sum\"),\n",
    "    generator=model.generator)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "import onmt.opts\n",
    "lr = 0.001\n",
    "torch_optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "optim = onmt.utils.optimizers.Optimizer(\n",
    "torch_optimizer, learning_rate=lr, max_grad_norm=2)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "2020-11-23 21:20:55,414 INFO Start training loop and validate every 100 steps...\n",
      "2020-11-23 21:20:55,414 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:20:55,468 INFO number of examples: 3029\n",
      "2020-11-23 21:21:31,893 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:21:31,952 INFO number of examples: 3029\n",
      "2020-11-23 21:21:33,726 INFO Step 100/10000; acc:  63.02; ppl:  3.69; xent: 1.31; lr: 0.00100; 17316/3725 tok/s;     38 sec\n",
      "2020-11-23 21:21:33,728 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:21:33,741 INFO number of examples: 534\n",
      "/opt/conda/conda-bld/pytorch_1591914855613/work/aten/src/ATen/native/BinaryOps.cpp:66: UserWarning: Integer division of tensors using div or / is deprecated, and in a future release div will perform true division as in Python 3. Use true_divide or floor_divide (// in Python) instead.\n",
      "2020-11-23 21:22:29,467 INFO Validation metrics: Bleu: 0.5975,Correct answers 0.1097, State transitions score: 0.0793\n",
      "2020-11-23 21:22:29,468 INFO Validation perplexity: 1.89107\n",
      "2020-11-23 21:22:29,468 INFO Validation accuracy: 79.4884\n",
      "2020-11-23 21:22:55,059 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:22:55,108 INFO number of examples: 3029\n",
      "2020-11-23 21:22:57,420 INFO Step 200/10000; acc:  80.12; ppl:  1.84; xent: 0.61; lr: 0.00100; 7885/1705 tok/s;    122 sec\n",
      "2020-11-23 21:22:57,422 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:22:57,434 INFO number of examples: 534\n",
      "2020-11-23 21:23:58,026 INFO Validation metrics: Bleu: 0.7740,Correct answers 0.1115, State transitions score: 0.1021\n",
      "2020-11-23 21:23:58,027 INFO Validation perplexity: 1.75046\n",
      "2020-11-23 21:23:58,027 INFO Validation accuracy: 81.2495\n",
      "2020-11-23 21:24:11,122 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:24:11,166 INFO number of examples: 3029\n",
      "2020-11-23 21:24:13,570 INFO Step 300/10000; acc:  81.33; ppl:  1.75; xent: 0.56; lr: 0.00100; 8745/1881 tok/s;    198 sec\n",
      "2020-11-23 21:24:13,571 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:24:13,580 INFO number of examples: 534\n",
      "2020-11-23 21:25:03,825 INFO Validation metrics: Bleu: 0.7894,Correct answers 0.1115, State transitions score: 0.1058\n",
      "2020-11-23 21:25:03,826 INFO Validation perplexity: 1.69947\n",
      "2020-11-23 21:25:03,826 INFO Validation accuracy: 82.08\n",
      "2020-11-23 21:25:16,150 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:25:16,273 INFO number of examples: 3029\n",
      "2020-11-23 21:25:19,221 INFO Step 400/10000; acc:  82.01; ppl:  1.70; xent: 0.53; lr: 0.00100; 10110/2184 tok/s;    264 sec\n",
      "2020-11-23 21:25:19,223 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:25:19,232 INFO number of examples: 534\n",
      "2020-11-23 21:26:09,619 INFO Validation metrics: Bleu: 0.7942,Correct answers 0.1134, State transitions score: 0.1039\n",
      "2020-11-23 21:26:09,621 INFO Validation perplexity: 1.66756\n",
      "2020-11-23 21:26:09,621 INFO Validation accuracy: 82.5849\n",
      "2020-11-23 21:26:22,047 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:26:22,093 INFO number of examples: 3029\n",
      "2020-11-23 21:26:26,279 INFO Step 500/10000; acc:  82.61; ppl:  1.67; xent: 0.51; lr: 0.00100; 10020/2153 tok/s;    331 sec\n",
      "2020-11-23 21:26:26,280 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:26:26,293 INFO number of examples: 534\n",
      "2020-11-23 21:27:17,170 INFO Validation metrics: Bleu: 0.7896,Correct answers 0.1005, State transitions score: 0.1047\n",
      "2020-11-23 21:27:17,171 INFO Validation perplexity: 1.6569\n",
      "2020-11-23 21:27:17,171 INFO Validation accuracy: 82.7894\n",
      "2020-11-23 21:27:27,892 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:27:27,939 INFO number of examples: 3029\n",
      "2020-11-23 21:27:32,517 INFO Step 600/10000; acc:  83.13; ppl:  1.64; xent: 0.49; lr: 0.00100; 10083/2170 tok/s;    397 sec\n",
      "2020-11-23 21:27:32,519 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:27:32,530 INFO number of examples: 534\n",
      "2020-11-23 21:28:23,994 INFO Validation metrics: Bleu: 0.8082,Correct answers 0.0729, State transitions score: 0.1110\n",
      "2020-11-23 21:28:23,995 INFO Validation perplexity: 1.62247\n",
      "2020-11-23 21:28:23,995 INFO Validation accuracy: 83.5573\n",
      "2020-11-23 21:28:33,819 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:28:33,942 INFO number of examples: 3029\n",
      "2020-11-23 21:28:39,218 INFO Step 700/10000; acc:  83.99; ppl:  1.60; xent: 0.47; lr: 0.00100; 9946/2148 tok/s;    464 sec\n",
      "2020-11-23 21:28:39,219 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:28:39,229 INFO number of examples: 534\n",
      "2020-11-23 21:29:31,101 INFO Validation metrics: Bleu: 0.8154,Correct answers 0.0950, State transitions score: 0.1162\n",
      "2020-11-23 21:29:31,102 INFO Validation perplexity: 1.58386\n",
      "2020-11-23 21:29:31,102 INFO Validation accuracy: 84.6131\n",
      "2020-11-23 21:29:40,331 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:29:40,378 INFO number of examples: 3029\n",
      "2020-11-23 21:29:47,005 INFO Step 800/10000; acc:  85.16; ppl:  1.55; xent: 0.44; lr: 0.00100; 9825/2122 tok/s;    532 sec\n",
      "2020-11-23 21:29:47,007 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:29:47,015 INFO number of examples: 534\n",
      "2020-11-23 21:30:38,070 INFO Validation metrics: Bleu: 0.8307,Correct answers 0.0790, State transitions score: 0.1157\n",
      "2020-11-23 21:30:38,071 INFO Validation perplexity: 1.52033\n",
      "2020-11-23 21:30:38,071 INFO Validation accuracy: 85.9402\n",
      "2020-11-23 21:30:46,844 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:30:46,892 INFO number of examples: 3029\n",
      "2020-11-23 21:30:53,872 INFO Step 900/10000; acc:  87.32; ppl:  1.47; xent: 0.39; lr: 0.00100; 9978/2148 tok/s;    598 sec\n",
      "2020-11-23 21:30:53,874 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:30:53,885 INFO number of examples: 534\n",
      "2020-11-23 21:31:45,505 INFO Validation metrics: Bleu: 0.8623,Correct answers 0.0968, State transitions score: 0.1332\n",
      "2020-11-23 21:31:45,505 INFO Validation perplexity: 1.4339\n",
      "2020-11-23 21:31:45,506 INFO Validation accuracy: 88.7655\n",
      "2020-11-23 21:31:53,218 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:31:53,265 INFO number of examples: 3029\n",
      "2020-11-23 21:32:00,876 INFO Step 1000/10000; acc:  89.77; ppl:  1.38; xent: 0.32; lr: 0.00100; 9897/2128 tok/s;    665 sec\n",
      "2020-11-23 21:32:00,877 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:32:00,890 INFO number of examples: 534\n",
      "2020-11-23 21:32:52,036 INFO Validation metrics: Bleu: 0.8848,Correct answers 0.1207, State transitions score: 0.1632\n",
      "2020-11-23 21:32:52,037 INFO Validation perplexity: 1.33357\n",
      "2020-11-23 21:32:52,037 INFO Validation accuracy: 90.7604\n",
      "2020-11-23 21:32:52,051 INFO Saving checkpoint model_step_1000.pt\n",
      "2020-11-23 21:32:59,235 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:32:59,353 INFO number of examples: 3029\n",
      "2020-11-23 21:33:07,739 INFO Step 1100/10000; acc:  91.27; ppl:  1.32; xent: 0.28; lr: 0.00100; 9952/2145 tok/s;    732 sec\n",
      "2020-11-23 21:33:07,741 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:33:07,750 INFO number of examples: 534\n",
      "2020-11-23 21:34:00,172 INFO Validation metrics: Bleu: 0.8983,Correct answers 0.1428, State transitions score: 0.2019\n",
      "2020-11-23 21:34:00,172 INFO Validation perplexity: 1.29318\n",
      "2020-11-23 21:34:00,173 INFO Validation accuracy: 91.7995\n",
      "2020-11-23 21:34:06,528 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:34:06,576 INFO number of examples: 3029\n",
      "2020-11-23 21:34:15,841 INFO Step 1200/10000; acc:  92.71; ppl:  1.26; xent: 0.23; lr: 0.00100; 9773/2112 tok/s;    800 sec\n",
      "2020-11-23 21:34:15,842 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:34:15,851 INFO number of examples: 534\n",
      "2020-11-23 21:35:07,158 INFO Validation metrics: Bleu: 0.9168,Correct answers 0.2071, State transitions score: 0.2673\n",
      "2020-11-23 21:35:07,159 INFO Validation perplexity: 1.22925\n",
      "2020-11-23 21:35:07,159 INFO Validation accuracy: 93.6691\n",
      "2020-11-23 21:35:12,746 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:35:12,791 INFO number of examples: 3029\n",
      "2020-11-23 21:35:22,503 INFO Step 1300/10000; acc:  93.87; ppl:  1.22; xent: 0.20; lr: 0.00100; 9934/2148 tok/s;    867 sec\n",
      "2020-11-23 21:35:22,504 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:35:22,514 INFO number of examples: 534\n",
      "2020-11-23 21:36:14,523 INFO Validation metrics: Bleu: 0.9293,Correct answers 0.2420, State transitions score: 0.3253\n",
      "2020-11-23 21:36:14,524 INFO Validation perplexity: 1.19283\n",
      "2020-11-23 21:36:14,525 INFO Validation accuracy: 94.6749\n",
      "2020-11-23 21:36:19,537 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:36:19,583 INFO number of examples: 3029\n",
      "2020-11-23 21:36:31,164 INFO Step 1400/10000; acc:  94.97; ppl:  1.17; xent: 0.16; lr: 0.00100; 9750/2093 tok/s;    936 sec\n",
      "2020-11-23 21:36:31,165 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:36:31,175 INFO number of examples: 534\n",
      "2020-11-23 21:37:24,323 INFO Validation metrics: Bleu: 0.9443,Correct answers 0.3119, State transitions score: 0.3875\n",
      "2020-11-23 21:37:24,324 INFO Validation perplexity: 1.1531\n",
      "2020-11-23 21:37:24,324 INFO Validation accuracy: 95.7224\n",
      "2020-11-23 21:37:28,765 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:37:28,815 INFO number of examples: 3029\n",
      "2020-11-23 21:37:40,346 INFO Step 1500/10000; acc:  95.81; ppl:  1.14; xent: 0.13; lr: 0.00100; 9762/2081 tok/s;   1005 sec\n",
      "2020-11-23 21:37:40,348 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:37:40,360 INFO number of examples: 534\n",
      "2020-11-23 21:38:32,693 INFO Validation metrics: Bleu: 0.9476,Correct answers 0.3738, State transitions score: 0.4388\n",
      "2020-11-23 21:38:32,694 INFO Validation perplexity: 1.13976\n",
      "2020-11-23 21:38:32,694 INFO Validation accuracy: 96.123\n",
      "2020-11-23 21:38:35,797 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:38:35,923 INFO number of examples: 3029\n",
      "2020-11-23 21:38:48,678 INFO Step 1600/10000; acc:  96.30; ppl:  1.13; xent: 0.12; lr: 0.00100; 9702/2093 tok/s;   1073 sec\n",
      "2020-11-23 21:38:48,680 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:38:48,690 INFO number of examples: 534\n",
      "2020-11-23 21:39:40,797 INFO Validation metrics: Bleu: 0.9569,Correct answers 0.4105, State transitions score: 0.4845\n",
      "2020-11-23 21:39:40,798 INFO Validation perplexity: 1.12087\n",
      "2020-11-23 21:39:40,798 INFO Validation accuracy: 96.653\n",
      "2020-11-23 21:39:43,240 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:39:43,287 INFO number of examples: 3029\n",
      "2020-11-23 21:39:56,654 INFO Step 1700/10000; acc:  97.02; ppl:  1.10; xent: 0.09; lr: 0.00100; 9789/2102 tok/s;   1141 sec\n",
      "2020-11-23 21:39:56,655 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:39:56,664 INFO number of examples: 534\n",
      "2020-11-23 21:40:48,662 INFO Validation metrics: Bleu: 0.9537,Correct answers 0.4234, State transitions score: 0.4863\n",
      "2020-11-23 21:40:48,662 INFO Validation perplexity: 1.11787\n",
      "2020-11-23 21:40:48,663 INFO Validation accuracy: 96.653\n",
      "2020-11-23 21:40:50,221 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:40:50,266 INFO number of examples: 3029\n",
      "2020-11-23 21:41:04,516 INFO Step 1800/10000; acc:  97.43; ppl:  1.08; xent: 0.08; lr: 0.00100; 9798/2111 tok/s;   1209 sec\n",
      "2020-11-23 21:41:04,518 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:41:04,530 INFO number of examples: 534\n",
      "2020-11-23 21:41:56,630 INFO Validation metrics: Bleu: 0.9643,Correct answers 0.4988, State transitions score: 0.5605\n",
      "2020-11-23 21:41:56,631 INFO Validation perplexity: 1.09498\n",
      "2020-11-23 21:41:56,631 INFO Validation accuracy: 97.4418\n",
      "2020-11-23 21:41:57,494 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:41:57,540 INFO number of examples: 3029\n",
      "2020-11-23 21:42:11,897 INFO Step 1900/10000; acc:  97.74; ppl:  1.07; xent: 0.07; lr: 0.00100; 9852/2128 tok/s;   1276 sec\n",
      "2020-11-23 21:42:11,899 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:42:11,911 INFO number of examples: 534\n",
      "2020-11-23 21:43:03,811 INFO Validation metrics: Bleu: 0.9665,Correct answers 0.5061, State transitions score: 0.5617\n",
      "2020-11-23 21:43:03,812 INFO Validation perplexity: 1.08451\n",
      "2020-11-23 21:43:03,812 INFO Validation accuracy: 97.5795\n",
      "2020-11-23 21:43:03,813 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:43:03,928 INFO number of examples: 3029\n",
      "2020-11-23 21:43:18,269 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:43:18,317 INFO number of examples: 3029\n",
      "2020-11-23 21:43:19,180 INFO Step 2000/10000; acc:  98.06; ppl:  1.06; xent: 0.06; lr: 0.00100; 9860/2121 tok/s;   1344 sec\n",
      "2020-11-23 21:43:19,181 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:43:19,191 INFO number of examples: 534\n",
      "2020-11-23 21:44:11,646 INFO Validation metrics: Bleu: 0.9685,Correct answers 0.5374, State transitions score: 0.5977\n",
      "2020-11-23 21:44:11,647 INFO Validation perplexity: 1.08092\n",
      "2020-11-23 21:44:11,648 INFO Validation accuracy: 97.8132\n",
      "2020-11-23 21:44:11,661 INFO Saving checkpoint model_step_2000.pt\n",
      "2020-11-23 21:44:25,611 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:44:25,721 INFO number of examples: 3029\n",
      "2020-11-23 21:44:27,144 INFO Step 2100/10000; acc:  98.29; ppl:  1.06; xent: 0.05; lr: 0.00100; 9710/2100 tok/s;   1412 sec\n",
      "2020-11-23 21:44:27,146 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:44:27,155 INFO number of examples: 534\n",
      "2020-11-23 21:45:18,868 INFO Validation metrics: Bleu: 0.9704,Correct answers 0.5613, State transitions score: 0.6160\n",
      "2020-11-23 21:45:18,869 INFO Validation perplexity: 1.07589\n",
      "2020-11-23 21:45:18,869 INFO Validation accuracy: 98.001\n",
      "2020-11-23 21:45:31,893 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:45:31,938 INFO number of examples: 3029\n",
      "2020-11-23 21:45:34,206 INFO Step 2200/10000; acc:  98.49; ppl:  1.05; xent: 0.05; lr: 0.00100; 9930/2136 tok/s;   1479 sec\n",
      "2020-11-23 21:45:34,207 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:45:34,220 INFO number of examples: 534\n",
      "2020-11-23 21:46:26,311 INFO Validation metrics: Bleu: 0.9720,Correct answers 0.5613, State transitions score: 0.6086\n",
      "2020-11-23 21:46:26,312 INFO Validation perplexity: 1.07382\n",
      "2020-11-23 21:46:26,312 INFO Validation accuracy: 98.1805\n",
      "2020-11-23 21:46:38,505 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:46:38,550 INFO number of examples: 3029\n",
      "2020-11-23 21:46:41,560 INFO Step 2300/10000; acc:  98.65; ppl:  1.04; xent: 0.04; lr: 0.00100; 9854/2129 tok/s;   1546 sec\n",
      "2020-11-23 21:46:41,561 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:46:41,573 INFO number of examples: 534\n",
      "2020-11-23 21:47:34,099 INFO Validation metrics: Bleu: 0.9741,Correct answers 0.5962, State transitions score: 0.6517\n",
      "2020-11-23 21:47:34,100 INFO Validation perplexity: 1.06576\n",
      "2020-11-23 21:47:34,101 INFO Validation accuracy: 98.3474\n",
      "2020-11-23 21:47:45,731 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:47:45,776 INFO number of examples: 3029\n",
      "2020-11-23 21:47:49,632 INFO Step 2400/10000; acc:  98.79; ppl:  1.04; xent: 0.04; lr: 0.00100; 9870/2121 tok/s;   1614 sec\n",
      "2020-11-23 21:47:49,634 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:47:49,645 INFO number of examples: 534\n",
      "2020-11-23 21:48:41,554 INFO Validation metrics: Bleu: 0.9746,Correct answers 0.6232, State transitions score: 0.6643\n",
      "2020-11-23 21:48:41,555 INFO Validation perplexity: 1.06273\n",
      "2020-11-23 21:48:41,555 INFO Validation accuracy: 98.3975\n",
      "2020-11-23 21:48:52,319 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:48:52,431 INFO number of examples: 3029\n",
      "2020-11-23 21:48:56,948 INFO Step 2500/10000; acc:  98.91; ppl:  1.04; xent: 0.04; lr: 0.00100; 9921/2135 tok/s;   1682 sec\n",
      "2020-11-23 21:48:56,949 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:48:56,962 INFO number of examples: 534\n",
      "2020-11-23 21:49:49,481 INFO Validation metrics: Bleu: 0.9764,Correct answers 0.6422, State transitions score: 0.6793\n",
      "2020-11-23 21:49:49,482 INFO Validation perplexity: 1.0633\n",
      "2020-11-23 21:49:49,482 INFO Validation accuracy: 98.4726\n",
      "2020-11-23 21:49:59,308 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:49:59,351 INFO number of examples: 3029\n",
      "2020-11-23 21:50:04,651 INFO Step 2600/10000; acc:  98.99; ppl:  1.03; xent: 0.03; lr: 0.00100; 9799/2117 tok/s;   1749 sec\n",
      "2020-11-23 21:50:04,652 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:50:04,665 INFO number of examples: 534\n",
      "2020-11-23 21:50:56,613 INFO Validation metrics: Bleu: 0.9746,Correct answers 0.6330, State transitions score: 0.6772\n",
      "2020-11-23 21:50:56,614 INFO Validation perplexity: 1.06367\n",
      "2020-11-23 21:50:56,614 INFO Validation accuracy: 98.4475\n",
      "2020-11-23 21:51:05,818 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:51:05,864 INFO number of examples: 3029\n",
      "2020-11-23 21:51:11,951 INFO Step 2700/10000; acc:  99.06; ppl:  1.03; xent: 0.03; lr: 0.00100; 9896/2137 tok/s;   1817 sec\n",
      "2020-11-23 21:51:11,952 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:51:11,964 INFO number of examples: 534\n",
      "2020-11-23 21:52:04,312 INFO Validation metrics: Bleu: 0.9758,Correct answers 0.6305, State transitions score: 0.6805\n",
      "2020-11-23 21:52:04,313 INFO Validation perplexity: 1.05944\n",
      "2020-11-23 21:52:04,314 INFO Validation accuracy: 98.4642\n",
      "2020-11-23 21:52:12,700 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:52:12,750 INFO number of examples: 3029\n",
      "2020-11-23 21:52:19,580 INFO Step 2800/10000; acc:  99.14; ppl:  1.03; xent: 0.03; lr: 0.00100; 9866/2124 tok/s;   1884 sec\n",
      "2020-11-23 21:52:19,582 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:52:19,591 INFO number of examples: 534\n",
      "2020-11-23 21:53:12,236 INFO Validation metrics: Bleu: 0.9761,Correct answers 0.6489, State transitions score: 0.6927\n",
      "2020-11-23 21:53:12,237 INFO Validation perplexity: 1.06347\n",
      "2020-11-23 21:53:12,237 INFO Validation accuracy: 98.5394\n",
      "2020-11-23 21:53:19,895 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:53:20,006 INFO number of examples: 3029\n",
      "2020-11-23 21:53:27,449 INFO Step 2900/10000; acc:  99.16; ppl:  1.03; xent: 0.03; lr: 0.00100; 9771/2101 tok/s;   1952 sec\n",
      "2020-11-23 21:53:27,451 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:53:27,460 INFO number of examples: 534\n",
      "2020-11-23 21:54:19,925 INFO Validation metrics: Bleu: 0.9778,Correct answers 0.6661, State transitions score: 0.7107\n",
      "2020-11-23 21:54:19,926 INFO Validation perplexity: 1.05736\n",
      "2020-11-23 21:54:19,927 INFO Validation accuracy: 98.6896\n",
      "2020-11-23 21:54:26,821 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:54:26,867 INFO number of examples: 3029\n",
      "2020-11-23 21:54:35,110 INFO Step 3000/10000; acc:  99.23; ppl:  1.03; xent: 0.03; lr: 0.00100; 9835/2120 tok/s;   2020 sec\n",
      "2020-11-23 21:54:35,111 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:54:35,121 INFO number of examples: 534\n",
      "2020-11-23 21:55:27,190 INFO Validation metrics: Bleu: 0.9769,Correct answers 0.6605, State transitions score: 0.7038\n",
      "2020-11-23 21:55:27,191 INFO Validation perplexity: 1.05805\n",
      "2020-11-23 21:55:27,192 INFO Validation accuracy: 98.7146\n",
      "2020-11-23 21:55:27,205 INFO Saving checkpoint model_step_3000.pt\n",
      "2020-11-23 21:55:33,612 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:55:33,656 INFO number of examples: 3029\n",
      "2020-11-23 21:55:42,654 INFO Step 3100/10000; acc:  99.26; ppl:  1.02; xent: 0.02; lr: 0.00100; 9854/2130 tok/s;   2087 sec\n",
      "2020-11-23 21:55:42,656 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:55:42,665 INFO number of examples: 534\n",
      "2020-11-23 21:56:34,814 INFO Validation metrics: Bleu: 0.9805,Correct answers 0.6838, State transitions score: 0.7201\n",
      "2020-11-23 21:56:34,815 INFO Validation perplexity: 1.0537\n",
      "2020-11-23 21:56:34,816 INFO Validation accuracy: 98.7438\n",
      "2020-11-23 21:56:40,321 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:56:40,368 INFO number of examples: 3029\n",
      "2020-11-23 21:56:50,059 INFO Step 3200/10000; acc:  99.35; ppl:  1.02; xent: 0.02; lr: 0.00100; 9824/2125 tok/s;   2155 sec\n",
      "2020-11-23 21:56:50,061 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:56:50,069 INFO number of examples: 534\n",
      "2020-11-23 21:57:42,486 INFO Validation metrics: Bleu: 0.9772,Correct answers 0.6697, State transitions score: 0.7133\n",
      "2020-11-23 21:57:42,487 INFO Validation perplexity: 1.05811\n",
      "2020-11-23 21:57:42,488 INFO Validation accuracy: 98.7146\n",
      "2020-11-23 21:57:47,308 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:57:47,426 INFO number of examples: 3029\n",
      "2020-11-23 21:57:57,807 INFO Step 3300/10000; acc:  99.35; ppl:  1.02; xent: 0.02; lr: 0.00100; 9881/2121 tok/s;   2222 sec\n",
      "2020-11-23 21:57:57,809 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:57:57,818 INFO number of examples: 534\n",
      "2020-11-23 21:58:49,997 INFO Validation metrics: Bleu: 0.9766,Correct answers 0.6587, State transitions score: 0.7034\n",
      "2020-11-23 21:58:49,998 INFO Validation perplexity: 1.06251\n",
      "2020-11-23 21:58:49,998 INFO Validation accuracy: 98.6854\n",
      "2020-11-23 21:58:53,986 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 21:58:54,031 INFO number of examples: 3029\n",
      "2020-11-23 21:59:05,385 INFO Step 3400/10000; acc:  99.40; ppl:  1.02; xent: 0.02; lr: 0.00100; 9993/2130 tok/s;   2290 sec\n",
      "2020-11-23 21:59:05,386 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 21:59:05,396 INFO number of examples: 534\n",
      "2020-11-23 21:59:57,766 INFO Validation metrics: Bleu: 0.9763,Correct answers 0.6495, State transitions score: 0.6989\n",
      "2020-11-23 21:59:57,767 INFO Validation perplexity: 1.06462\n",
      "2020-11-23 21:59:57,768 INFO Validation accuracy: 98.602\n",
      "2020-11-23 22:00:00,824 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:00:00,872 INFO number of examples: 3029\n",
      "2020-11-23 22:00:13,048 INFO Step 3500/10000; acc:  99.36; ppl:  1.02; xent: 0.02; lr: 0.00100; 9798/2114 tok/s;   2358 sec\n",
      "2020-11-23 22:00:13,050 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:00:13,061 INFO number of examples: 534\n",
      "2020-11-23 22:01:05,551 INFO Validation metrics: Bleu: 0.9781,Correct answers 0.6642, State transitions score: 0.7081\n",
      "2020-11-23 22:01:05,552 INFO Validation perplexity: 1.05858\n",
      "2020-11-23 22:01:05,552 INFO Validation accuracy: 98.6896\n",
      "2020-11-23 22:01:07,944 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:01:07,993 INFO number of examples: 3029\n",
      "2020-11-23 22:01:20,897 INFO Step 3600/10000; acc:  99.44; ppl:  1.02; xent: 0.02; lr: 0.00100; 9808/2106 tok/s;   2425 sec\n",
      "2020-11-23 22:01:20,899 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:01:20,915 INFO number of examples: 534\n",
      "2020-11-23 22:02:13,434 INFO Validation metrics: Bleu: 0.9794,Correct answers 0.6881, State transitions score: 0.7347\n",
      "2020-11-23 22:02:13,435 INFO Validation perplexity: 1.06042\n",
      "2020-11-23 22:02:13,435 INFO Validation accuracy: 98.8273\n",
      "2020-11-23 22:02:14,949 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:02:14,995 INFO number of examples: 3029\n",
      "2020-11-23 22:02:28,657 INFO Step 3700/10000; acc:  99.47; ppl:  1.02; xent: 0.02; lr: 0.00100; 9813/2114 tok/s;   2493 sec\n",
      "2020-11-23 22:02:28,659 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:02:28,670 INFO number of examples: 534\n",
      "2020-11-23 22:03:20,715 INFO Validation metrics: Bleu: 0.9796,Correct answers 0.6893, State transitions score: 0.7265\n",
      "2020-11-23 22:03:20,716 INFO Validation perplexity: 1.05847\n",
      "2020-11-23 22:03:20,717 INFO Validation accuracy: 98.7981\n",
      "2020-11-23 22:03:21,553 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:03:21,662 INFO number of examples: 3029\n",
      "2020-11-23 22:03:35,917 INFO Step 3800/10000; acc:  99.46; ppl:  1.02; xent: 0.02; lr: 0.00100; 9870/2132 tok/s;   2561 sec\n",
      "2020-11-23 22:03:35,918 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:03:35,927 INFO number of examples: 534\n",
      "2020-11-23 22:04:28,520 INFO Validation metrics: Bleu: 0.9783,Correct answers 0.6765, State transitions score: 0.7203\n",
      "2020-11-23 22:04:28,521 INFO Validation perplexity: 1.05985\n",
      "2020-11-23 22:04:28,521 INFO Validation accuracy: 98.6896\n",
      "2020-11-23 22:04:28,522 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:04:28,573 INFO number of examples: 3029\n",
      "2020-11-23 22:04:42,866 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:04:42,980 INFO number of examples: 3029\n",
      "2020-11-23 22:04:43,746 INFO Step 3900/10000; acc:  99.50; ppl:  1.02; xent: 0.02; lr: 0.00100; 9781/2104 tok/s;   2628 sec\n",
      "2020-11-23 22:04:43,747 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:04:43,759 INFO number of examples: 534\n",
      "2020-11-23 22:05:35,775 INFO Validation metrics: Bleu: 0.9787,Correct answers 0.6771, State transitions score: 0.7219\n",
      "2020-11-23 22:05:35,776 INFO Validation perplexity: 1.06256\n",
      "2020-11-23 22:05:35,776 INFO Validation accuracy: 98.7605\n",
      "2020-11-23 22:05:49,353 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:05:49,399 INFO number of examples: 3029\n",
      "2020-11-23 22:05:50,904 INFO Step 4000/10000; acc:  99.53; ppl:  1.02; xent: 0.02; lr: 0.00100; 9827/2125 tok/s;   2695 sec\n",
      "2020-11-23 22:05:50,905 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:05:50,918 INFO number of examples: 534\n",
      "2020-11-23 22:06:43,529 INFO Validation metrics: Bleu: 0.9813,Correct answers 0.6863, State transitions score: 0.7284\n",
      "2020-11-23 22:06:43,530 INFO Validation perplexity: 1.05403\n",
      "2020-11-23 22:06:43,530 INFO Validation accuracy: 98.8273\n",
      "2020-11-23 22:06:43,543 INFO Saving checkpoint model_step_4000.pt\n",
      "2020-11-23 22:06:56,576 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:06:56,623 INFO number of examples: 3029\n",
      "2020-11-23 22:06:58,912 INFO Step 4100/10000; acc:  99.53; ppl:  1.02; xent: 0.02; lr: 0.00100; 9792/2106 tok/s;   2763 sec\n",
      "2020-11-23 22:06:58,914 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:06:58,923 INFO number of examples: 534\n",
      "2020-11-23 22:07:51,511 INFO Validation metrics: Bleu: 0.9776,Correct answers 0.6728, State transitions score: 0.7192\n",
      "2020-11-23 22:07:51,511 INFO Validation perplexity: 1.06107\n",
      "2020-11-23 22:07:51,512 INFO Validation accuracy: 98.7105\n",
      "2020-11-23 22:08:03,892 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:08:03,939 INFO number of examples: 3029\n",
      "2020-11-23 22:08:07,072 INFO Step 4200/10000; acc:  99.55; ppl:  1.01; xent: 0.01; lr: 0.00100; 9738/2104 tok/s;   2832 sec\n",
      "2020-11-23 22:08:07,073 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:08:07,082 INFO number of examples: 534\n",
      "2020-11-23 22:08:59,514 INFO Validation metrics: Bleu: 0.9806,Correct answers 0.6979, State transitions score: 0.7432\n",
      "2020-11-23 22:08:59,515 INFO Validation perplexity: 1.05272\n",
      "2020-11-23 22:08:59,515 INFO Validation accuracy: 98.9149\n",
      "2020-11-23 22:09:11,013 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:09:11,122 INFO number of examples: 3029\n",
      "2020-11-23 22:09:15,083 INFO Step 4300/10000; acc:  99.58; ppl:  1.01; xent: 0.01; lr: 0.00100; 9879/2123 tok/s;   2900 sec\n",
      "2020-11-23 22:09:15,085 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:09:15,097 INFO number of examples: 534\n",
      "2020-11-23 22:10:08,058 INFO Validation metrics: Bleu: 0.9809,Correct answers 0.7059, State transitions score: 0.7447\n",
      "2020-11-23 22:10:08,059 INFO Validation perplexity: 1.05192\n",
      "2020-11-23 22:10:08,060 INFO Validation accuracy: 98.8357\n",
      "2020-11-23 22:10:18,647 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:10:18,696 INFO number of examples: 3029\n",
      "2020-11-23 22:10:23,270 INFO Step 4400/10000; acc:  99.59; ppl:  1.01; xent: 0.01; lr: 0.00100; 9795/2108 tok/s;   2968 sec\n",
      "2020-11-23 22:10:23,272 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:10:23,281 INFO number of examples: 534\n",
      "2020-11-23 22:11:16,202 INFO Validation metrics: Bleu: 0.9814,Correct answers 0.7126, State transitions score: 0.7570\n",
      "2020-11-23 22:11:16,203 INFO Validation perplexity: 1.06098\n",
      "2020-11-23 22:11:16,204 INFO Validation accuracy: 98.8774\n",
      "2020-11-23 22:11:26,029 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:11:26,073 INFO number of examples: 3029\n",
      "2020-11-23 22:11:31,356 INFO Step 4500/10000; acc:  99.61; ppl:  1.01; xent: 0.01; lr: 0.00100; 9744/2105 tok/s;   3036 sec\n",
      "2020-11-23 22:11:31,358 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:11:31,367 INFO number of examples: 534\n",
      "2020-11-23 22:12:24,082 INFO Validation metrics: Bleu: 0.9805,Correct answers 0.6973, State transitions score: 0.7457\n",
      "2020-11-23 22:12:24,083 INFO Validation perplexity: 1.059\n",
      "2020-11-23 22:12:24,083 INFO Validation accuracy: 98.8732\n",
      "2020-11-23 22:12:33,262 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:12:33,306 INFO number of examples: 3029\n",
      "2020-11-23 22:12:39,379 INFO Step 4600/10000; acc:  99.58; ppl:  1.01; xent: 0.01; lr: 0.00100; 9791/2114 tok/s;   3104 sec\n",
      "2020-11-23 22:12:39,381 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:12:39,390 INFO number of examples: 534\n",
      "2020-11-23 22:13:33,863 INFO Validation metrics: Bleu: 0.9811,Correct answers 0.7083, State transitions score: 0.7468\n",
      "2020-11-23 22:13:33,864 INFO Validation perplexity: 1.05491\n",
      "2020-11-23 22:13:33,864 INFO Validation accuracy: 98.94\n",
      "2020-11-23 22:13:42,365 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:13:42,421 INFO number of examples: 3029\n",
      "2020-11-23 22:13:49,398 INFO Step 4700/10000; acc:  99.63; ppl:  1.01; xent: 0.01; lr: 0.00100; 9529/2051 tok/s;   3174 sec\n",
      "2020-11-23 22:13:49,400 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:13:49,410 INFO number of examples: 534\n",
      "2020-11-23 22:14:41,971 INFO Validation metrics: Bleu: 0.9827,Correct answers 0.7089, State transitions score: 0.7504\n",
      "2020-11-23 22:14:41,972 INFO Validation perplexity: 1.05429\n",
      "2020-11-23 22:14:41,973 INFO Validation accuracy: 98.9775\n",
      "2020-11-23 22:14:49,710 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:14:49,840 INFO number of examples: 3029\n",
      "2020-11-23 22:14:57,557 INFO Step 4800/10000; acc:  99.67; ppl:  1.01; xent: 0.01; lr: 0.00100; 9730/2092 tok/s;   3242 sec\n",
      "2020-11-23 22:14:57,559 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:14:57,573 INFO number of examples: 534\n",
      "2020-11-23 22:15:50,536 INFO Validation metrics: Bleu: 0.9823,Correct answers 0.7083, State transitions score: 0.7547\n",
      "2020-11-23 22:15:50,537 INFO Validation perplexity: 1.05729\n",
      "2020-11-23 22:15:50,537 INFO Validation accuracy: 98.965\n",
      "2020-11-23 22:15:57,463 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:15:57,509 INFO number of examples: 3029\n",
      "2020-11-23 22:16:05,937 INFO Step 4900/10000; acc:  99.65; ppl:  1.01; xent: 0.01; lr: 0.00100; 9732/2098 tok/s;   3311 sec\n",
      "2020-11-23 22:16:05,939 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:16:05,957 INFO number of examples: 534\n",
      "2020-11-23 22:16:59,187 INFO Validation metrics: Bleu: 0.9821,Correct answers 0.7040, State transitions score: 0.7445\n",
      "2020-11-23 22:16:59,188 INFO Validation perplexity: 1.05325\n",
      "2020-11-23 22:16:59,188 INFO Validation accuracy: 98.9483\n",
      "2020-11-23 22:17:05,648 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:17:05,698 INFO number of examples: 3029\n",
      "2020-11-23 22:17:14,710 INFO Step 5000/10000; acc:  99.66; ppl:  1.01; xent: 0.01; lr: 0.00100; 9678/2092 tok/s;   3379 sec\n",
      "2020-11-23 22:17:14,711 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:17:14,726 INFO number of examples: 534\n",
      "2020-11-23 22:18:08,028 INFO Validation metrics: Bleu: 0.9822,Correct answers 0.7096, State transitions score: 0.7520\n",
      "2020-11-23 22:18:08,029 INFO Validation perplexity: 1.05191\n",
      "2020-11-23 22:18:08,030 INFO Validation accuracy: 98.9442\n",
      "2020-11-23 22:18:08,044 INFO Saving checkpoint model_step_5000.pt\n",
      "2020-11-23 22:18:13,905 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:18:14,030 INFO number of examples: 3029\n",
      "2020-11-23 22:18:23,676 INFO Step 5100/10000; acc:  99.65; ppl:  1.01; xent: 0.01; lr: 0.00100; 9602/2077 tok/s;   3448 sec\n",
      "2020-11-23 22:18:23,677 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:18:23,691 INFO number of examples: 534\n",
      "2020-11-23 22:19:16,626 INFO Validation metrics: Bleu: 0.9833,Correct answers 0.7181, State transitions score: 0.7647\n",
      "2020-11-23 22:19:16,627 INFO Validation perplexity: 1.05602\n",
      "2020-11-23 22:19:16,628 INFO Validation accuracy: 98.9609\n",
      "2020-11-23 22:19:21,478 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:19:21,527 INFO number of examples: 3029\n",
      "2020-11-23 22:19:32,030 INFO Step 5200/10000; acc:  99.67; ppl:  1.01; xent: 0.01; lr: 0.00100; 9793/2102 tok/s;   3517 sec\n",
      "2020-11-23 22:19:32,031 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:19:32,045 INFO number of examples: 534\n",
      "2020-11-23 22:20:24,632 INFO Validation metrics: Bleu: 0.9830,Correct answers 0.7224, State transitions score: 0.7602\n",
      "2020-11-23 22:20:24,633 INFO Validation perplexity: 1.05336\n",
      "2020-11-23 22:20:24,634 INFO Validation accuracy: 99.0151\n",
      "2020-11-23 22:20:28,726 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:20:28,773 INFO number of examples: 3029\n",
      "2020-11-23 22:20:40,234 INFO Step 5300/10000; acc:  99.70; ppl:  1.01; xent: 0.01; lr: 0.00100; 9902/2111 tok/s;   3585 sec\n",
      "2020-11-23 22:20:40,235 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:20:40,246 INFO number of examples: 534\n",
      "2020-11-23 22:21:33,155 INFO Validation metrics: Bleu: 0.9822,Correct answers 0.7328, State transitions score: 0.7697\n",
      "2020-11-23 22:21:33,156 INFO Validation perplexity: 1.05844\n",
      "2020-11-23 22:21:33,157 INFO Validation accuracy: 98.9692\n",
      "2020-11-23 22:21:36,366 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:21:36,416 INFO number of examples: 3029\n",
      "2020-11-23 22:21:48,646 INFO Step 5400/10000; acc:  99.67; ppl:  1.01; xent: 0.01; lr: 0.00100; 9691/2091 tok/s;   3653 sec\n",
      "2020-11-23 22:21:48,648 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:21:48,659 INFO number of examples: 534\n",
      "2020-11-23 22:22:41,887 INFO Validation metrics: Bleu: 0.9826,Correct answers 0.7237, State transitions score: 0.7610\n",
      "2020-11-23 22:22:41,888 INFO Validation perplexity: 1.05462\n",
      "2020-11-23 22:22:41,888 INFO Validation accuracy: 99.0109\n",
      "2020-11-23 22:22:44,323 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:22:44,448 INFO number of examples: 3029\n",
      "2020-11-23 22:22:57,388 INFO Step 5500/10000; acc:  99.72; ppl:  1.01; xent: 0.01; lr: 0.00100; 9680/2079 tok/s;   3722 sec\n",
      "2020-11-23 22:22:57,389 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:22:57,403 INFO number of examples: 534\n",
      "2020-11-23 22:23:50,652 INFO Validation metrics: Bleu: 0.9838,Correct answers 0.7335, State transitions score: 0.7708\n",
      "2020-11-23 22:23:50,653 INFO Validation perplexity: 1.05233\n",
      "2020-11-23 22:23:50,654 INFO Validation accuracy: 99.036\n",
      "2020-11-23 22:23:52,251 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:23:52,298 INFO number of examples: 3029\n",
      "2020-11-23 22:24:06,102 INFO Step 5600/10000; acc:  99.69; ppl:  1.01; xent: 0.01; lr: 0.00100; 9676/2084 tok/s;   3791 sec\n",
      "2020-11-23 22:24:06,104 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:24:06,120 INFO number of examples: 534\n",
      "2020-11-23 22:24:59,507 INFO Validation metrics: Bleu: 0.9822,Correct answers 0.7365, State transitions score: 0.7734\n",
      "2020-11-23 22:24:59,508 INFO Validation perplexity: 1.05466\n",
      "2020-11-23 22:24:59,509 INFO Validation accuracy: 99.0068\n",
      "2020-11-23 22:25:00,343 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:25:00,391 INFO number of examples: 3029\n",
      "2020-11-23 22:25:14,860 INFO Step 5700/10000; acc:  99.69; ppl:  1.01; xent: 0.01; lr: 0.00100; 9655/2085 tok/s;   3859 sec\n",
      "2020-11-23 22:25:14,862 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:25:14,873 INFO number of examples: 534\n",
      "2020-11-23 22:26:07,329 INFO Validation metrics: Bleu: 0.9827,Correct answers 0.7243, State transitions score: 0.7671\n",
      "2020-11-23 22:26:07,330 INFO Validation perplexity: 1.0546\n",
      "2020-11-23 22:26:07,331 INFO Validation accuracy: 99.036\n",
      "2020-11-23 22:26:07,331 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:26:07,380 INFO number of examples: 3029\n",
      "2020-11-23 22:26:22,843 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:26:22,900 INFO number of examples: 3029\n",
      "2020-11-23 22:26:23,713 INFO Step 5800/10000; acc:  99.74; ppl:  1.01; xent: 0.01; lr: 0.00100; 9635/2073 tok/s;   3928 sec\n",
      "2020-11-23 22:26:23,714 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:26:23,729 INFO number of examples: 534\n",
      "2020-11-23 22:27:17,138 INFO Validation metrics: Bleu: 0.9829,Correct answers 0.7188, State transitions score: 0.7604\n",
      "2020-11-23 22:27:17,138 INFO Validation perplexity: 1.05792\n",
      "2020-11-23 22:27:17,139 INFO Validation accuracy: 99.0318\n",
      "2020-11-23 22:27:30,696 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:27:30,744 INFO number of examples: 3029\n",
      "2020-11-23 22:27:32,236 INFO Step 5900/10000; acc:  99.73; ppl:  1.01; xent: 0.01; lr: 0.00100; 9631/2083 tok/s;   3997 sec\n",
      "2020-11-23 22:27:32,238 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:27:32,251 INFO number of examples: 534\n",
      "2020-11-23 22:28:24,781 INFO Validation metrics: Bleu: 0.9831,Correct answers 0.7273, State transitions score: 0.7678\n",
      "2020-11-23 22:28:24,782 INFO Validation perplexity: 1.05445\n",
      "2020-11-23 22:28:24,783 INFO Validation accuracy: 99.0652\n",
      "2020-11-23 22:28:37,928 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:28:37,980 INFO number of examples: 3029\n",
      "2020-11-23 22:28:40,236 INFO Step 6000/10000; acc:  99.74; ppl:  1.01; xent: 0.01; lr: 0.00100; 9793/2107 tok/s;   4065 sec\n",
      "2020-11-23 22:28:40,238 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:28:40,254 INFO number of examples: 534\n",
      "2020-11-23 22:29:33,212 INFO Validation metrics: Bleu: 0.9829,Correct answers 0.7390, State transitions score: 0.7793\n",
      "2020-11-23 22:29:33,213 INFO Validation perplexity: 1.05071\n",
      "2020-11-23 22:29:33,213 INFO Validation accuracy: 99.1236\n",
      "2020-11-23 22:29:33,228 INFO Saving checkpoint model_step_6000.pt\n",
      "2020-11-23 22:29:45,720 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:29:45,849 INFO number of examples: 3029\n",
      "2020-11-23 22:29:48,781 INFO Step 6100/10000; acc:  99.73; ppl:  1.01; xent: 0.01; lr: 0.00100; 9683/2092 tok/s;   4133 sec\n",
      "2020-11-23 22:29:48,783 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:29:48,793 INFO number of examples: 534\n",
      "2020-11-23 22:30:41,146 INFO Validation metrics: Bleu: 0.9850,Correct answers 0.7230, State transitions score: 0.7650\n",
      "2020-11-23 22:30:41,147 INFO Validation perplexity: 1.05337\n",
      "2020-11-23 22:30:41,148 INFO Validation accuracy: 99.0735\n",
      "2020-11-23 22:30:52,838 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:30:52,887 INFO number of examples: 3029\n",
      "2020-11-23 22:30:56,693 INFO Step 6200/10000; acc:  99.75; ppl:  1.01; xent: 0.01; lr: 0.00100; 9894/2126 tok/s;   4201 sec\n",
      "2020-11-23 22:30:56,696 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:30:56,711 INFO number of examples: 534\n",
      "2020-11-23 22:31:49,808 INFO Validation metrics: Bleu: 0.9825,Correct answers 0.7267, State transitions score: 0.7675\n",
      "2020-11-23 22:31:49,809 INFO Validation perplexity: 1.04997\n",
      "2020-11-23 22:31:49,809 INFO Validation accuracy: 99.0443\n",
      "2020-11-23 22:32:00,529 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:32:00,580 INFO number of examples: 3029\n",
      "2020-11-23 22:32:05,420 INFO Step 6300/10000; acc:  99.76; ppl:  1.01; xent: 0.01; lr: 0.00100; 9718/2091 tok/s;   4270 sec\n",
      "2020-11-23 22:32:05,422 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:32:05,432 INFO number of examples: 534\n",
      "2020-11-23 22:32:58,479 INFO Validation metrics: Bleu: 0.9840,Correct answers 0.7451, State transitions score: 0.7787\n",
      "2020-11-23 22:32:58,479 INFO Validation perplexity: 1.05207\n",
      "2020-11-23 22:32:58,480 INFO Validation accuracy: 99.0443\n",
      "2020-11-23 22:33:08,367 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:33:08,488 INFO number of examples: 3029\n",
      "2020-11-23 22:33:13,719 INFO Step 6400/10000; acc:  99.75; ppl:  1.01; xent: 0.01; lr: 0.00100; 9713/2098 tok/s;   4338 sec\n",
      "2020-11-23 22:33:13,721 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:33:13,731 INFO number of examples: 534\n",
      "2020-11-23 22:34:06,745 INFO Validation metrics: Bleu: 0.9831,Correct answers 0.7445, State transitions score: 0.7814\n",
      "2020-11-23 22:34:06,746 INFO Validation perplexity: 1.05097\n",
      "2020-11-23 22:34:06,746 INFO Validation accuracy: 99.0861\n",
      "2020-11-23 22:34:16,109 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:34:16,255 INFO number of examples: 3029\n",
      "2020-11-23 22:34:22,713 INFO Step 6500/10000; acc:  99.74; ppl:  1.01; xent: 0.01; lr: 0.00100; 9653/2084 tok/s;   4407 sec\n",
      "2020-11-23 22:34:22,715 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:34:22,729 INFO number of examples: 534\n",
      "2020-11-23 22:35:16,052 INFO Validation metrics: Bleu: 0.9849,Correct answers 0.7580, State transitions score: 0.7884\n",
      "2020-11-23 22:35:16,053 INFO Validation perplexity: 1.04849\n",
      "2020-11-23 22:35:16,053 INFO Validation accuracy: 99.1111\n",
      "2020-11-23 22:35:24,551 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:35:24,600 INFO number of examples: 3029\n",
      "2020-11-23 22:35:31,644 INFO Step 6600/10000; acc:  99.72; ppl:  1.01; xent: 0.01; lr: 0.00100; 9680/2084 tok/s;   4476 sec\n",
      "2020-11-23 22:35:31,645 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:35:31,661 INFO number of examples: 534\n",
      "2020-11-23 22:36:25,012 INFO Validation metrics: Bleu: 0.9847,Correct answers 0.7396, State transitions score: 0.7792\n",
      "2020-11-23 22:36:25,013 INFO Validation perplexity: 1.05364\n",
      "2020-11-23 22:36:25,014 INFO Validation accuracy: 99.0861\n",
      "2020-11-23 22:36:32,942 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:36:32,993 INFO number of examples: 3029\n",
      "2020-11-23 22:36:40,503 INFO Step 6700/10000; acc:  99.74; ppl:  1.01; xent: 0.01; lr: 0.00100; 9631/2071 tok/s;   4545 sec\n",
      "2020-11-23 22:36:40,504 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:36:40,520 INFO number of examples: 534\n",
      "2020-11-23 22:37:33,486 INFO Validation metrics: Bleu: 0.9859,Correct answers 0.7402, State transitions score: 0.7793\n",
      "2020-11-23 22:37:33,487 INFO Validation perplexity: 1.05329\n",
      "2020-11-23 22:37:33,487 INFO Validation accuracy: 99.1695\n",
      "2020-11-23 22:37:40,470 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:37:40,589 INFO number of examples: 3029\n",
      "2020-11-23 22:37:48,865 INFO Step 6800/10000; acc:  99.78; ppl:  1.01; xent: 0.01; lr: 0.00100; 9734/2098 tok/s;   4613 sec\n",
      "2020-11-23 22:37:48,867 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:37:48,876 INFO number of examples: 534\n",
      "2020-11-23 22:38:42,309 INFO Validation metrics: Bleu: 0.9861,Correct answers 0.7555, State transitions score: 0.7911\n",
      "2020-11-23 22:38:42,310 INFO Validation perplexity: 1.04951\n",
      "2020-11-23 22:38:42,311 INFO Validation accuracy: 99.1862\n",
      "2020-11-23 22:38:49,046 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:38:49,097 INFO number of examples: 3029\n",
      "2020-11-23 22:38:58,076 INFO Step 6900/10000; acc:  99.75; ppl:  1.01; xent: 0.01; lr: 0.00100; 9617/2078 tok/s;   4683 sec\n",
      "2020-11-23 22:38:58,078 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:38:58,087 INFO number of examples: 534\n",
      "2020-11-23 22:39:51,681 INFO Validation metrics: Bleu: 0.9857,Correct answers 0.7555, State transitions score: 0.7876\n",
      "2020-11-23 22:39:51,682 INFO Validation perplexity: 1.0522\n",
      "2020-11-23 22:39:51,683 INFO Validation accuracy: 99.1194\n",
      "2020-11-23 22:39:57,158 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:39:57,207 INFO number of examples: 3029\n",
      "2020-11-23 22:40:06,848 INFO Step 7000/10000; acc:  99.77; ppl:  1.01; xent: 0.01; lr: 0.00100; 9629/2082 tok/s;   4751 sec\n",
      "2020-11-23 22:40:06,849 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:40:06,858 INFO number of examples: 534\n",
      "2020-11-23 22:40:59,246 INFO Validation metrics: Bleu: 0.9860,Correct answers 0.7488, State transitions score: 0.7859\n",
      "2020-11-23 22:40:59,247 INFO Validation perplexity: 1.05106\n",
      "2020-11-23 22:40:59,248 INFO Validation accuracy: 99.1695\n",
      "2020-11-23 22:40:59,261 INFO Saving checkpoint model_step_7000.pt\n",
      "2020-11-23 22:41:04,084 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:41:04,129 INFO number of examples: 3029\n",
      "2020-11-23 22:41:14,555 INFO Step 7100/10000; acc:  99.81; ppl:  1.01; xent: 0.01; lr: 0.00100; 9887/2122 tok/s;   4819 sec\n",
      "2020-11-23 22:41:14,557 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:41:14,569 INFO number of examples: 534\n",
      "2020-11-23 22:42:07,173 INFO Validation metrics: Bleu: 0.9832,Correct answers 0.7433, State transitions score: 0.7775\n",
      "2020-11-23 22:42:07,174 INFO Validation perplexity: 1.05306\n",
      "2020-11-23 22:42:07,174 INFO Validation accuracy: 99.1111\n",
      "2020-11-23 22:42:11,175 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:42:11,291 INFO number of examples: 3029\n",
      "2020-11-23 22:42:22,584 INFO Step 7200/10000; acc:  99.78; ppl:  1.01; xent: 0.01; lr: 0.00100; 9927/2116 tok/s;   4887 sec\n",
      "2020-11-23 22:42:22,586 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:42:22,597 INFO number of examples: 534\n",
      "2020-11-23 22:43:15,295 INFO Validation metrics: Bleu: 0.9856,Correct answers 0.7629, State transitions score: 0.7935\n",
      "2020-11-23 22:43:15,296 INFO Validation perplexity: 1.04995\n",
      "2020-11-23 22:43:15,297 INFO Validation accuracy: 99.1946\n",
      "2020-11-23 22:43:18,350 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:43:18,399 INFO number of examples: 3029\n",
      "2020-11-23 22:43:30,404 INFO Step 7300/10000; acc:  99.80; ppl:  1.01; xent: 0.01; lr: 0.00100; 9776/2109 tok/s;   4955 sec\n",
      "2020-11-23 22:43:30,406 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:43:30,483 INFO number of examples: 534\n",
      "2020-11-23 22:44:23,634 INFO Validation metrics: Bleu: 0.9865,Correct answers 0.7776, State transitions score: 0.8074\n",
      "2020-11-23 22:44:23,634 INFO Validation perplexity: 1.0435\n",
      "2020-11-23 22:44:23,635 INFO Validation accuracy: 99.2113\n",
      "2020-11-23 22:44:26,057 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:44:26,101 INFO number of examples: 3029\n",
      "2020-11-23 22:44:38,997 INFO Step 7400/10000; acc:  99.78; ppl:  1.01; xent: 0.01; lr: 0.00100; 9701/2084 tok/s;   5024 sec\n",
      "2020-11-23 22:44:38,999 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:44:39,008 INFO number of examples: 534\n",
      "2020-11-23 22:45:31,625 INFO Validation metrics: Bleu: 0.9850,Correct answers 0.7610, State transitions score: 0.7959\n",
      "2020-11-23 22:45:31,626 INFO Validation perplexity: 1.04433\n",
      "2020-11-23 22:45:31,626 INFO Validation accuracy: 99.182\n",
      "2020-11-23 22:45:33,146 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:45:33,192 INFO number of examples: 3029\n",
      "2020-11-23 22:45:46,943 INFO Step 7500/10000; acc:  99.78; ppl:  1.01; xent: 0.01; lr: 0.00100; 9786/2108 tok/s;   5092 sec\n",
      "2020-11-23 22:45:46,945 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:45:46,955 INFO number of examples: 534\n",
      "2020-11-23 22:46:39,493 INFO Validation metrics: Bleu: 0.9862,Correct answers 0.7598, State transitions score: 0.7928\n",
      "2020-11-23 22:46:39,495 INFO Validation perplexity: 1.05156\n",
      "2020-11-23 22:46:39,495 INFO Validation accuracy: 99.1946\n",
      "2020-11-23 22:46:40,403 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:46:40,454 INFO number of examples: 3029\n",
      "2020-11-23 22:46:56,627 INFO Step 7600/10000; acc:  99.79; ppl:  1.01; xent: 0.01; lr: 0.00100; 9527/2058 tok/s;   5161 sec\n",
      "2020-11-23 22:46:56,629 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:46:56,643 INFO number of examples: 534\n",
      "2020-11-23 22:47:51,218 INFO Validation metrics: Bleu: 0.9866,Correct answers 0.7665, State transitions score: 0.7996\n",
      "2020-11-23 22:47:51,219 INFO Validation perplexity: 1.04737\n",
      "2020-11-23 22:47:51,219 INFO Validation accuracy: 99.2405\n",
      "2020-11-23 22:47:51,220 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:47:51,345 INFO number of examples: 3029\n",
      "2020-11-23 22:48:07,444 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:48:07,521 INFO number of examples: 3029\n",
      "2020-11-23 22:48:08,827 INFO Step 7700/10000; acc:  99.82; ppl:  1.01; xent: 0.01; lr: 0.00100; 9189/1977 tok/s;   5233 sec\n",
      "2020-11-23 22:48:08,829 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:48:08,841 INFO number of examples: 534\n",
      "2020-11-23 22:49:14,248 INFO Validation metrics: Bleu: 0.9871,Correct answers 0.7757, State transitions score: 0.8049\n",
      "2020-11-23 22:49:14,249 INFO Validation perplexity: 1.04999\n",
      "2020-11-23 22:49:14,250 INFO Validation accuracy: 99.2321\n",
      "2020-11-23 22:49:28,601 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:49:28,738 INFO number of examples: 3029\n",
      "2020-11-23 22:49:30,235 INFO Step 7800/10000; acc:  99.83; ppl:  1.01; xent: 0.01; lr: 0.00100; 8107/1753 tok/s;   5315 sec\n",
      "2020-11-23 22:49:30,237 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:49:30,249 INFO number of examples: 534\n",
      "2020-11-23 22:50:29,733 INFO Validation metrics: Bleu: 0.9876,Correct answers 0.7794, State transitions score: 0.8075\n",
      "2020-11-23 22:50:29,735 INFO Validation perplexity: 1.0435\n",
      "2020-11-23 22:50:29,735 INFO Validation accuracy: 99.2655\n",
      "2020-11-23 22:50:43,798 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:50:43,847 INFO number of examples: 3029\n",
      "2020-11-23 22:50:46,156 INFO Step 7900/10000; acc:  99.83; ppl:  1.01; xent: 0.01; lr: 0.00100; 8772/1887 tok/s;   5391 sec\n",
      "2020-11-23 22:50:46,158 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:50:46,171 INFO number of examples: 534\n",
      "2020-11-23 22:51:42,461 INFO Validation metrics: Bleu: 0.9877,Correct answers 0.7788, State transitions score: 0.8097\n",
      "2020-11-23 22:51:42,462 INFO Validation perplexity: 1.04731\n",
      "2020-11-23 22:51:42,462 INFO Validation accuracy: 99.253\n",
      "2020-11-23 22:51:56,400 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:51:56,448 INFO number of examples: 3029\n",
      "2020-11-23 22:51:59,690 INFO Step 8000/10000; acc:  99.82; ppl:  1.01; xent: 0.01; lr: 0.00100; 9026/1950 tok/s;   5464 sec\n",
      "2020-11-23 22:51:59,692 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:51:59,704 INFO number of examples: 534\n",
      "2020-11-23 22:52:58,945 INFO Validation metrics: Bleu: 0.9872,Correct answers 0.7714, State transitions score: 0.8048\n",
      "2020-11-23 22:52:58,947 INFO Validation perplexity: 1.04865\n",
      "2020-11-23 22:52:58,947 INFO Validation accuracy: 99.2697\n",
      "2020-11-23 22:52:58,961 INFO Saving checkpoint model_step_8000.pt\n",
      "2020-11-23 22:53:12,521 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:53:12,574 INFO number of examples: 3029\n",
      "2020-11-23 22:53:16,616 INFO Step 8100/10000; acc:  99.83; ppl:  1.01; xent: 0.01; lr: 0.00100; 8734/1877 tok/s;   5541 sec\n",
      "2020-11-23 22:53:16,618 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:53:16,631 INFO number of examples: 534\n",
      "2020-11-23 22:54:16,805 INFO Validation metrics: Bleu: 0.9862,Correct answers 0.7647, State transitions score: 0.7982\n",
      "2020-11-23 22:54:16,806 INFO Validation perplexity: 1.05362\n",
      "2020-11-23 22:54:16,807 INFO Validation accuracy: 99.2196\n",
      "2020-11-23 22:54:28,614 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:54:28,737 INFO number of examples: 3029\n",
      "2020-11-23 22:54:33,586 INFO Step 8200/10000; acc:  99.83; ppl:  1.01; xent: 0.01; lr: 0.00100; 8677/1867 tok/s;   5618 sec\n",
      "2020-11-23 22:54:33,588 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:54:33,598 INFO number of examples: 534\n",
      "2020-11-23 22:55:35,694 INFO Validation metrics: Bleu: 0.9873,Correct answers 0.7837, State transitions score: 0.8098\n",
      "2020-11-23 22:55:35,695 INFO Validation perplexity: 1.04295\n",
      "2020-11-23 22:55:35,695 INFO Validation accuracy: 99.278\n",
      "2020-11-23 22:55:46,463 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:55:46,521 INFO number of examples: 3029\n",
      "2020-11-23 22:55:52,884 INFO Step 8300/10000; acc:  99.85; ppl:  1.01; xent: 0.01; lr: 0.00100; 8366/1807 tok/s;   5697 sec\n",
      "2020-11-23 22:55:52,889 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:55:52,903 INFO number of examples: 534\n",
      "2020-11-23 22:56:54,352 INFO Validation metrics: Bleu: 0.9866,Correct answers 0.7629, State transitions score: 0.7977\n",
      "2020-11-23 22:56:54,353 INFO Validation perplexity: 1.04631\n",
      "2020-11-23 22:56:54,353 INFO Validation accuracy: 99.2238\n",
      "2020-11-23 22:57:06,312 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:57:06,370 INFO number of examples: 3029\n",
      "2020-11-23 22:57:13,437 INFO Step 8400/10000; acc:  99.84; ppl:  1.01; xent: 0.01; lr: 0.00100; 8268/1785 tok/s;   5778 sec\n",
      "2020-11-23 22:57:13,438 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:57:13,448 INFO number of examples: 534\n",
      "2020-11-23 22:58:14,708 INFO Validation metrics: Bleu: 0.9875,Correct answers 0.7855, State transitions score: 0.8098\n",
      "2020-11-23 22:58:14,709 INFO Validation perplexity: 1.04927\n",
      "2020-11-23 22:58:14,709 INFO Validation accuracy: 99.2572\n",
      "2020-11-23 22:58:25,546 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:58:25,604 INFO number of examples: 3029\n",
      "2020-11-23 22:58:33,485 INFO Step 8500/10000; acc:  99.83; ppl:  1.01; xent: 0.01; lr: 0.00100; 8335/1794 tok/s;   5858 sec\n",
      "2020-11-23 22:58:33,487 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:58:33,496 INFO number of examples: 534\n",
      "2020-11-23 22:59:34,740 INFO Validation metrics: Bleu: 0.9883,Correct answers 0.7862, State transitions score: 0.8176\n",
      "2020-11-23 22:59:34,741 INFO Validation perplexity: 1.04409\n",
      "2020-11-23 22:59:34,741 INFO Validation accuracy: 99.2405\n",
      "2020-11-23 22:59:42,960 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 22:59:43,082 INFO number of examples: 3029\n",
      "2020-11-23 22:59:51,805 INFO Step 8600/10000; acc:  99.84; ppl:  1.01; xent: 0.01; lr: 0.00100; 8467/1821 tok/s;   5936 sec\n",
      "2020-11-23 22:59:51,806 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 22:59:51,819 INFO number of examples: 534\n",
      "2020-11-23 23:00:55,304 INFO Validation metrics: Bleu: 0.9864,Correct answers 0.7659, State transitions score: 0.8027\n",
      "2020-11-23 23:00:55,305 INFO Validation perplexity: 1.04902\n",
      "2020-11-23 23:00:55,306 INFO Validation accuracy: 99.2321\n",
      "2020-11-23 23:01:04,345 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:01:04,396 INFO number of examples: 3029\n",
      "2020-11-23 23:01:13,190 INFO Step 8700/10000; acc:  99.85; ppl:  1.01; xent: 0.01; lr: 0.00100; 8176/1762 tok/s;   6018 sec\n",
      "2020-11-23 23:01:13,192 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:01:13,208 INFO number of examples: 534\n",
      "2020-11-23 23:02:16,120 INFO Validation metrics: Bleu: 0.9868,Correct answers 0.7757, State transitions score: 0.8044\n",
      "2020-11-23 23:02:16,123 INFO Validation perplexity: 1.05035\n",
      "2020-11-23 23:02:16,124 INFO Validation accuracy: 99.2446\n",
      "2020-11-23 23:02:24,254 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:02:24,305 INFO number of examples: 3029\n",
      "2020-11-23 23:02:34,780 INFO Step 8800/10000; acc:  99.85; ppl:  1.01; xent: 0.01; lr: 0.00100; 8158/1763 tok/s;   6099 sec\n",
      "2020-11-23 23:02:34,782 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:02:34,796 INFO number of examples: 534\n",
      "2020-11-23 23:03:40,942 INFO Validation metrics: Bleu: 0.9884,Correct answers 0.8039, State transitions score: 0.8281\n",
      "2020-11-23 23:03:40,943 INFO Validation perplexity: 1.03991\n",
      "2020-11-23 23:03:40,943 INFO Validation accuracy: 99.3406\n",
      "2020-11-23 23:03:47,535 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:03:47,583 INFO number of examples: 3029\n",
      "2020-11-23 23:03:59,482 INFO Step 8900/10000; acc:  99.85; ppl:  1.01; xent: 0.01; lr: 0.00100; 7818/1691 tok/s;   6184 sec\n",
      "2020-11-23 23:03:59,484 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:03:59,495 INFO number of examples: 534\n",
      "2020-11-23 23:05:07,638 INFO Validation metrics: Bleu: 0.9873,Correct answers 0.7941, State transitions score: 0.8206\n",
      "2020-11-23 23:05:07,639 INFO Validation perplexity: 1.04011\n",
      "2020-11-23 23:05:07,640 INFO Validation accuracy: 99.3198\n",
      "2020-11-23 23:05:15,608 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:05:15,670 INFO number of examples: 3029\n",
      "2020-11-23 23:05:32,616 INFO Step 9000/10000; acc:  99.87; ppl:  1.00; xent: 0.00; lr: 0.00100; 7188/1543 tok/s;   6277 sec\n",
      "2020-11-23 23:05:32,618 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:05:32,628 INFO number of examples: 534\n",
      "2020-11-23 23:06:42,641 INFO Validation metrics: Bleu: 0.9882,Correct answers 0.7825, State transitions score: 0.8099\n",
      "2020-11-23 23:06:42,643 INFO Validation perplexity: 1.05682\n",
      "2020-11-23 23:06:42,643 INFO Validation accuracy: 99.157\n",
      "2020-11-23 23:06:42,657 INFO Saving checkpoint model_step_9000.pt\n",
      "2020-11-23 23:06:47,130 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:06:47,257 INFO number of examples: 3029\n",
      "2020-11-23 23:06:59,583 INFO Step 9100/10000; acc:  99.83; ppl:  1.01; xent: 0.01; lr: 0.00100; 7765/1655 tok/s;   6364 sec\n",
      "2020-11-23 23:06:59,585 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:06:59,594 INFO number of examples: 534\n",
      "2020-11-23 23:08:08,809 INFO Validation metrics: Bleu: 0.9878,Correct answers 0.7831, State transitions score: 0.8137\n",
      "2020-11-23 23:08:08,810 INFO Validation perplexity: 1.04747\n",
      "2020-11-23 23:08:08,811 INFO Validation accuracy: 99.2613\n",
      "2020-11-23 23:08:11,991 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:08:12,041 INFO number of examples: 3029\n",
      "2020-11-23 23:08:24,527 INFO Step 9200/10000; acc:  99.83; ppl:  1.01; xent: 0.01; lr: 0.00100; 7805/1684 tok/s;   6449 sec\n",
      "2020-11-23 23:08:24,529 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:08:24,538 INFO number of examples: 534\n",
      "2020-11-23 23:09:33,320 INFO Validation metrics: Bleu: 0.9881,Correct answers 0.7782, State transitions score: 0.8120\n",
      "2020-11-23 23:09:33,321 INFO Validation perplexity: 1.04736\n",
      "2020-11-23 23:09:33,321 INFO Validation accuracy: 99.3239\n",
      "2020-11-23 23:09:35,947 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:09:36,004 INFO number of examples: 3029\n",
      "2020-11-23 23:09:49,229 INFO Step 9300/10000; acc:  99.87; ppl:  1.00; xent: 0.00; lr: 0.00100; 7856/1687 tok/s;   6534 sec\n",
      "2020-11-23 23:09:49,230 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:09:49,240 INFO number of examples: 534\n",
      "2020-11-23 23:10:43,008 INFO Validation metrics: Bleu: 0.9875,Correct answers 0.7868, State transitions score: 0.8165\n",
      "2020-11-23 23:10:43,009 INFO Validation perplexity: 1.04918\n",
      "2020-11-23 23:10:43,009 INFO Validation accuracy: 99.2446\n",
      "2020-11-23 23:10:44,653 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:10:44,705 INFO number of examples: 3029\n",
      "2020-11-23 23:10:58,571 INFO Step 9400/10000; acc:  99.87; ppl:  1.00; xent: 0.00; lr: 0.00100; 9589/2066 tok/s;   6603 sec\n",
      "2020-11-23 23:10:58,573 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:10:58,584 INFO number of examples: 534\n",
      "2020-11-23 23:11:53,881 INFO Validation metrics: Bleu: 0.9886,Correct answers 0.7880, State transitions score: 0.8196\n",
      "2020-11-23 23:11:53,882 INFO Validation perplexity: 1.05058\n",
      "2020-11-23 23:11:53,882 INFO Validation accuracy: 99.2488\n",
      "2020-11-23 23:11:55,640 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:11:55,827 INFO number of examples: 3029\n",
      "2020-11-23 23:12:11,889 INFO Step 9500/10000; acc:  99.87; ppl:  1.00; xent: 0.00; lr: 0.00100; 9055/1956 tok/s;   6676 sec\n",
      "2020-11-23 23:12:11,891 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:12:11,904 INFO number of examples: 534\n",
      "2020-11-23 23:13:07,898 INFO Validation metrics: Bleu: 0.9886,Correct answers 0.7892, State transitions score: 0.8175\n",
      "2020-11-23 23:13:07,899 INFO Validation perplexity: 1.04172\n",
      "2020-11-23 23:13:07,899 INFO Validation accuracy: 99.3239\n",
      "2020-11-23 23:13:07,900 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:13:07,952 INFO number of examples: 3029\n",
      "2020-11-23 23:13:22,484 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:13:22,620 INFO number of examples: 3029\n",
      "2020-11-23 23:13:23,442 INFO Step 9600/10000; acc:  99.87; ppl:  1.00; xent: 0.00; lr: 0.00100; 9272/1995 tok/s;   6748 sec\n",
      "2020-11-23 23:13:23,444 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:13:23,453 INFO number of examples: 534\n",
      "2020-11-23 23:14:16,927 INFO Validation metrics: Bleu: 0.9889,Correct answers 0.7953, State transitions score: 0.8260\n",
      "2020-11-23 23:14:16,928 INFO Validation perplexity: 1.04428\n",
      "2020-11-23 23:14:16,928 INFO Validation accuracy: 99.278\n",
      "2020-11-23 23:14:30,894 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:14:30,945 INFO number of examples: 3029\n",
      "2020-11-23 23:14:32,440 INFO Step 9700/10000; acc:  99.87; ppl:  1.00; xent: 0.00; lr: 0.00100; 9565/2069 tok/s;   6817 sec\n",
      "2020-11-23 23:14:32,441 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:14:32,456 INFO number of examples: 534\n",
      "2020-11-23 23:15:26,751 INFO Validation metrics: Bleu: 0.9877,Correct answers 0.7855, State transitions score: 0.8157\n",
      "2020-11-23 23:15:26,752 INFO Validation perplexity: 1.04754\n",
      "2020-11-23 23:15:26,753 INFO Validation accuracy: 99.3281\n",
      "2020-11-23 23:15:40,105 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:15:40,155 INFO number of examples: 3029\n",
      "2020-11-23 23:15:42,541 INFO Step 9800/10000; acc:  99.85; ppl:  1.01; xent: 0.01; lr: 0.00100; 9500/2044 tok/s;   6887 sec\n",
      "2020-11-23 23:15:42,543 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:15:42,558 INFO number of examples: 534\n",
      "2020-11-23 23:16:36,089 INFO Validation metrics: Bleu: 0.9890,Correct answers 0.8062, State transitions score: 0.8264\n",
      "2020-11-23 23:16:36,090 INFO Validation perplexity: 1.04127\n",
      "2020-11-23 23:16:36,090 INFO Validation accuracy: 99.3323\n",
      "2020-11-23 23:16:48,606 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:16:48,656 INFO number of examples: 3029\n",
      "2020-11-23 23:16:51,681 INFO Step 9900/10000; acc:  99.87; ppl:  1.00; xent: 0.00; lr: 0.00100; 9600/2074 tok/s;   6956 sec\n",
      "2020-11-23 23:16:51,683 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:16:51,697 INFO number of examples: 534\n",
      "2020-11-23 23:17:45,766 INFO Validation metrics: Bleu: 0.9886,Correct answers 0.7917, State transitions score: 0.8214\n",
      "2020-11-23 23:17:45,767 INFO Validation perplexity: 1.04098\n",
      "2020-11-23 23:17:45,768 INFO Validation accuracy: 99.349\n",
      "2020-11-23 23:17:57,513 INFO Loading dataset from processed/SimQA.train.0.pt\n",
      "2020-11-23 23:17:57,638 INFO number of examples: 3029\n",
      "2020-11-23 23:18:01,765 INFO Step 10000/10000; acc:  99.86; ppl:  1.00; xent: 0.00; lr: 0.00100; 9587/2060 tok/s;   7026 sec\n",
      "2020-11-23 23:18:01,766 INFO Loading dataset from processed/SimQA.valid.0.pt\n",
      "2020-11-23 23:18:01,780 INFO number of examples: 534\n",
      "2020-11-23 23:18:55,385 INFO Validation metrics: Bleu: 0.9879,Correct answers 0.8152, State transitions score: 0.8338\n",
      "2020-11-23 23:18:55,386 INFO Validation perplexity: 1.04896\n",
      "2020-11-23 23:18:55,387 INFO Validation accuracy: 99.3072\n",
      "2020-11-23 23:18:55,401 INFO Saving checkpoint model_step_10000.pt\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "from itertools import chain\n",
    "train_data_file = f\"processed/SimQA.train.0.pt\"\n",
    "valid_data_file = f\"processed/SimQA.valid.0.pt\"\n",
    "train_iter = onmt.inputters.inputter.DatasetLazyIter(dataset_paths=[train_data_file],\n",
    "                                                     fields=vocab_fields,\n",
    "                                                     batch_size=32,\n",
    "                                                     batch_size_multiple=1,\n",
    "                                                     batch_size_fn=None,\n",
    "                                                     device=device,\n",
    "                                                     is_train=True,\n",
    "                                                     repeat=True,\n",
    "                                                     pool_factor=8192)\n",
    "\n",
    "valid_iter = onmt.inputters.inputter.DatasetLazyIter(dataset_paths=[valid_data_file],\n",
    "                                                     fields=vocab_fields,\n",
    "                                                     batch_size=16,\n",
    "                                                     batch_size_multiple=1,\n",
    "                                                     batch_size_fn=None,\n",
    "                                                     device=device,\n",
    "                                                     is_train=False,\n",
    "                                                     repeat=False,\n",
    "                                                     pool_factor=8192)\n",
    "report_manager = onmt.utils.ReportMgr(report_every=100, tensorboard_writer=writer)\n",
    "\n",
    "src_reader = onmt.inputters.str2reader[\"text\"]\n",
    "tgt_reader = onmt.inputters.str2reader[\"text\"]\n",
    "scorer = onmt.translate.GNMTGlobalScorer(alpha=0.7,\n",
    "                                         beta=0.,\n",
    "                                         length_penalty=\"avg\",\n",
    "                                         coverage_penalty=\"none\")\n",
    "gpu = 0 if torch.cuda.is_available() else -1\n",
    "translator = onmt.translate.Translator(model=model,\n",
    "                                       fields=vocab_fields,\n",
    "                                       src_reader=src_reader,\n",
    "                                       tgt_reader=tgt_reader,\n",
    "                                       global_scorer=scorer,\n",
    "                                       gpu=gpu)\n",
    "builder = onmt.translate.TranslationBuilder(data=torch.load(valid_data_file),\n",
    "                                            fields=vocab_fields, has_tgt=True)\n",
    "\n",
    "metrics = [BleuScore(), CorrectAnswersScore(tgt_vocab)]\n",
    "model_saver = onmt.models.model_saver.ModelSaver(\"model\", model, None, vocab_fields, optim)\n",
    "trainer = SimMLETrainer(model=model,\n",
    "                    model_saver=model_saver,\n",
    "                    train_loss=loss,\n",
    "                    valid_loss=loss,\n",
    "                    optim=optim,\n",
    "                    report_manager=report_manager,\n",
    "                    metrics=metrics,\n",
    "                    translator=translator,\n",
    "                    translation_builder=builder)\n",
    "stats, metrics = trainer.train(train_iter=train_iter,\n",
    "              src_vocab=src_vocab,\n",
    "              train_steps=10000,\n",
    "              valid_iter=valid_iter,\n",
    "              valid_steps=100,\n",
    "              save_checkpoint_steps=1000)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}