onmt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Experimental Class for Smiles Enumeration, Iterator and SmilesIterator\n",
    "# adapted from Keras 1.2.2\n",
    "from rdkit import Chem\n",
    "import numpy as np\n",
    "import threading\n",
    "\n",
    "\n",
    "class Iterator(object):\n",
    "    \"\"\"Abstract base class for data iterators.\n",
    "    # Arguments\n",
    "        n: Integer, total number of samples in the dataset to loop over.\n",
    "        batch_size: Integer, size of a batch.\n",
    "        shuffle: Boolean, whether to shuffle the data between epochs.\n",
    "        seed: Random seeding for data shuffling.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, n, batch_size, shuffle, seed):\n",
    "        self.n = n\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "        self.batch_index = 0\n",
    "        self.total_batches_seen = 0\n",
    "        self.lock = threading.Lock()\n",
    "        self.index_generator = self._flow_index(n, batch_size, shuffle, seed)\n",
    "        if n < batch_size:\n",
    "            raise ValueError('Input data length is shorter than batch_size'\n",
    "                             'Adjust batch_size')\n",
    "\n",
    "    def reset(self):\n",
    "        self.batch_index = 0\n",
    "\n",
    "    def _flow_index(self, n, batch_size=32, shuffle=False, seed=None):\n",
    "        # Ensure self.batch_index is 0.\n",
    "        self.reset()\n",
    "        while 1:\n",
    "            if seed is not None:\n",
    "                np.random.seed(seed + self.total_batches_seen)\n",
    "            if self.batch_index == 0:\n",
    "                index_array = np.arange(n)\n",
    "                if shuffle:\n",
    "                    index_array = np.random.permutation(n)\n",
    "\n",
    "            current_index = (self.batch_index * batch_size) % n\n",
    "            if n > current_index + batch_size:\n",
    "                current_batch_size = batch_size\n",
    "                self.batch_index += 1\n",
    "            else:\n",
    "                current_batch_size = n - current_index\n",
    "                self.batch_index = 0\n",
    "            self.total_batches_seen += 1\n",
    "            yield (index_array[current_index: current_index +\n",
    "                               current_batch_size],\n",
    "                   current_index, current_batch_size)\n",
    "\n",
    "    def __iter__(self):\n",
    "        # Needed if we want to do something like:\n",
    "        # for x, y in data_gen.flow(...):\n",
    "        return self\n",
    "\n",
    "    def __next__(self, *args, **kwargs):\n",
    "        return self.next(*args, **kwargs)\n",
    "\n",
    "\n",
    "class SmilesIterator(Iterator):\n",
    "    \"\"\"Iterator yielding data from a SMILES array.\n",
    "    # Arguments\n",
    "        x: Numpy array of SMILES input data.\n",
    "        y: Numpy array of targets data.\n",
    "        smiles_data_generator: Instance of `SmilesEnumerator`\n",
    "            to use for random SMILES generation.\n",
    "        batch_size: Integer, size of a batch.\n",
    "        shuffle: Boolean, whether to shuffle the data between epochs.\n",
    "        seed: Random seed for data shuffling.\n",
    "        dtype: dtype to use for returned batch.\n",
    "        Set to keras.backend.floatx if using Keras\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, x, y, smiles_data_generator,\n",
    "                 batch_size=32, shuffle=False, seed=None,\n",
    "                 dtype=np.float32\n",
    "                 ):\n",
    "        if y is not None and len(x) != len(y):\n",
    "            raise ValueError('X (images tensor) and y (labels) '\n",
    "                             'should have the same length. '\n",
    "                             'Found: X.shape = %s, y.shape = %s' %\n",
    "                             (np.asarray(x).shape, np.asarray(y).shape))\n",
    "\n",
    "        self.x = np.asarray(x)\n",
    "\n",
    "        if y is not None:\n",
    "            self.y = np.asarray(y)\n",
    "        else:\n",
    "            self.y = None\n",
    "        self.smiles_data_generator = smiles_data_generator\n",
    "        self.dtype = dtype\n",
    "        super(SmilesIterator, self).__init__(x.shape[0], batch_size, shuffle,\n",
    "                                             seed)\n",
    "\n",
    "    def next(self):\n",
    "        \"\"\"For python 2.x.\n",
    "        # Returns\n",
    "            The next batch.\n",
    "        \"\"\"\n",
    "        # Keeps under lock only the mechanism which advances\n",
    "        # the indexing of each batch.\n",
    "        with self.lock:\n",
    "            index_array, current_index, current_batch_size =\\\n",
    "                next(self.index_generator)\n",
    "        # The transformation of images is not under thread lock\n",
    "        # so it can be done in parallel\n",
    "        batch_x = np.zeros(\n",
    "            tuple([current_batch_size] + [self.smiles_data_generator.pad,\n",
    "                                          self.smiles_data_generator._charlen]),\n",
    "            dtype=self.dtype)\n",
    "        for i, j in enumerate(index_array):\n",
    "            smiles = self.x[j:j + 1]\n",
    "            x = self.smiles_data_generator.transform(smiles)\n",
    "            batch_x[i] = x\n",
    "\n",
    "        if self.y is None:\n",
    "            return batch_x\n",
    "        batch_y = self.y[index_array]\n",
    "        return batch_x, batch_y\n",
    "\n",
    "\n",
    "class SmilesEnumerator(object):\n",
    "    \"\"\"SMILES Enumerator, vectorizer and devectorizer\n",
    "    #Arguments\n",
    "        charset: string containing the characters for the vectorization\n",
    "          can also be generated via the .fit() method\n",
    "        pad: Length of the vectorization\n",
    "        leftpad: Add spaces to the left of the SMILES\n",
    "        isomericSmiles: Generate SMILES containing information about stereogenic centers\n",
    "        enum: Enumerate the SMILES during transform\n",
    "        canonical: use canonical SMILES during transform (overrides enum)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, charset='@C)(=cOn1S2/H[N]\\\\', pad=120, leftpad=True,\n",
    "                 isomericSmiles=True, enum=True,\n",
    "                 canonical=False):\n",
    "        self._charset = None\n",
    "        self.charset = charset\n",
    "        self.pad = pad\n",
    "        self.leftpad = leftpad\n",
    "        self.isomericSmiles = isomericSmiles\n",
    "        self.enumerate = enum\n",
    "        self.canonical = canonical\n",
    "\n",
    "    @property\n",
    "    def charset(self):\n",
    "        return self._charset\n",
    "\n",
    "    @charset.setter\n",
    "    def charset(self, charset):\n",
    "        self._charset = charset\n",
    "        self._charlen = len(charset)\n",
    "        self._char_to_int = dict((c, i) for i, c in enumerate(charset))\n",
    "        self._int_to_char = dict((i, c) for i, c in enumerate(charset))\n",
    "\n",
    "    def fit(self, smiles, extra_chars=[], extra_pad=5):\n",
    "        \"\"\"Performs extraction of the charset and length of a SMILES datasets\n",
    "        and sets self.pad and self.charset\n",
    "        #Arguments\n",
    "            smiles: Numpy array or Pandas series containing smiles as strings\n",
    "            extra_chars: List of extra chars to add to the charset\n",
    "            (e.g. \"\\\\\\\\\" when \"/\" is present)\n",
    "            extra_pad: Extra padding to add before or after the\n",
    "            SMILES vectorization\n",
    "        \"\"\"\n",
    "        charset = set(\"\".join(list(smiles)))\n",
    "        self.charset = \"\".join(charset.union(set(extra_chars)))\n",
    "        self.pad = max([len(smile) for smile in smiles]) + extra_pad\n",
    "\n",
    "    def randomize_smiles(self, smiles):\n",
    "        \"\"\"Perform a randomization of a SMILES string\n",
    "        must be RDKit sanitizable\"\"\"\n",
    "        m = Chem.MolFromSmiles(smiles)\n",
    "        ans = list(range(m.GetNumAtoms()))\n",
    "        np.random.shuffle(ans)\n",
    "        nm = Chem.RenumberAtoms(m, ans)\n",
    "        return Chem.MolToSmiles(nm, canonical=self.canonical,\n",
    "                                isomericSmiles=self.isomericSmiles)\n",
    "\n",
    "    def transform(self, smiles):\n",
    "        \"\"\"Perform an enumeration (randomization) and vectorization of\n",
    "        a Numpy array of smiles strings\n",
    "        #Arguments\n",
    "            smiles: Numpy array or Pandas series containing smiles as strings\n",
    "        \"\"\"\n",
    "        one_hot = np.zeros((smiles.shape[0], self.pad, self._charlen), dtype=np.int8)\n",
    "\n",
    "        for i, ss in enumerate(smiles):\n",
    "            if self.enumerate: ss = self.randomize_smiles(ss)\n",
    "            for j, c in enumerate(ss):\n",
    "                one_hot[i, j, self._char_to_int[c]] = 1\n",
    "        return one_hot\n",
    "\n",
    "    def reverse_transform(self, vect):\n",
    "        \"\"\" Performs a conversion of a vectorized SMILES to a smiles strings\n",
    "        charset must be the same as used for vectorization.\n",
    "        #Arguments\n",
    "            vect: Numpy array of vectorized SMILES.\n",
    "        \"\"\"\n",
    "        smiles = []\n",
    "        for v in vect:\n",
    "            # mask v\n",
    "            v = v[v.sum(axis=1) == 1]\n",
    "            # Find one hot encoded index with argmax, translate to char\n",
    "            # and join to string\n",
    "            smile = \"\".join(self._int_to_char[i] for i in v.argmax(axis=1))\n",
    "            smiles.append(smile)\n",
    "        return np.array(smiles)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2'\n",
      " 'CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2'\n",
      " 'CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2']\n",
      "Unbalanced generation of batches\n"
     ]
    }
   ],
   "source": [
    "smiles = np.array([\"CCC(=O)O[C@@]1(CC[NH+](C[C@H]1CC=C)C)c2ccccc2\",\n",
    "                       \"CCC[S@@](=O)c1ccc2c(c1)[nH]/c(=N/C(=O)OC)/[nH]2\"] * 10\n",
    "                      )\n",
    "# Test canonical SMILES vectorization\n",
    "sm_en = SmilesEnumerator(canonical=True, enum=False)\n",
    "sm_en.fit(smiles, extra_chars=[\"\\\\\"])\n",
    "v = sm_en.transform(smiles)\n",
    "transformed = sm_en.reverse_transform(v)\n",
    "print(transformed)\n",
    "if len(set(transformed)) > 2:\n",
    "    print(\"Too many different canonical SMILES generated\")\n",
    "\n",
    "# Test enumeration\n",
    "sm_en.canonical = False\n",
    "sm_en.enumerate = True\n",
    "v2 = sm_en.transform(smiles)\n",
    "transformed = sm_en.reverse_transform(v2)\n",
    "if len(set(transformed)) < 3: print(\"Too few enumerated SMILES generated\")\n",
    "\n",
    "# Reconstruction\n",
    "reconstructed = sm_en.reverse_transform(v[0:5])\n",
    "for i, smile in enumerate(reconstructed):\n",
    "    if smile != smiles[i]:\n",
    "        print(\"Error in reconstruction %s %s\" % (smile, smiles[i]))\n",
    "        break\n",
    "\n",
    "# test Pandas\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(smiles)\n",
    "v = sm_en.transform(df[0])\n",
    "if v.shape != (20, 52, 18): print(\"Possible error in pandas use\")\n",
    "\n",
    "# BUG, when batchsize > x.shape[0], then it only returns x.shape[0]!\n",
    "# Test batch generation\n",
    "sm_it = SmilesIterator(smiles, np.array([1, 2] * 10), sm_en, batch_size=10,\n",
    "                       shuffle=True)\n",
    "X, y = sm_it.next()\n",
    "if sum(y == 1) - sum(y == 2) > 1:\n",
    "    print(\"Unbalanced generation of batches\")\n",
    "if len(X) != 10: print(\"Error in batchsize generation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from onmt.train_single import main as single_main\n",
    "from onmt.new_opts import Opts, PreprocessorOpts\n",
    "from onmt.preprocess import build_save_dataset, build_save_vocab\n",
    "import onmt.inputters as inputters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def separate_data(data_path, name_to_save, mode):\n",
    "    lines = open(data_path).read().split('\\n')\n",
    "    with open(name_to_save + \"_src-\" + mode +\".txt\", \"w\") as f1, open(name_to_save + \"_tgt-\" + mode + \".txt\", \"w\") as f2:\n",
    "        for line in lines[3: len(lines) - 1]:\n",
    "            input_text, target_text, *c = line.split('\\t')\n",
    "            input_text,_ = input_text.split('>')\n",
    "            input_text = input_text.split()\n",
    "            target_text = target_text.split()\n",
    "            f1.write(\" \".join(input_text) + \" \\n\")\n",
    "            f2.write(\" \".join(target_text) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"data/US_patents_1976-Sep2016_1product_reactions_train.csv\"\n",
    "name_to_save = \"data/USP\"\n",
    "mode = \"train\"\n",
    "separate_data(data_path, name_to_save, mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"data/US_patents_1976-Sep2016_1product_reactions_valid.csv\"\n",
    "name_to_save = \"data/USP\"\n",
    "mode = \"valid\"\n",
    "separate_data(data_path, name_to_save, mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = PreprocessorOpts(train_src=name_to_save + \"_src-train.txt\", \n",
    "                  train_tgt=name_to_save + \"_tgt-train.txt\", \n",
    "                  valid_src=name_to_save + \"_src-valid.txt\",\n",
    "                  valid_tgt=name_to_save + \"_tgt-valid.txt\",\n",
    "                  save_data=\"data/USP\")\n",
    "\n",
    "src_nfeats = inputters.get_num_features(\n",
    "    opt.data_type, opt.train_src, 'src')\n",
    "\n",
    "tgt_nfeats = inputters.get_num_features(\n",
    "    opt.data_type, opt.train_tgt, 'tgt')\n",
    "\n",
    "fields = inputters.get_fields(opt.data_type, src_nfeats, tgt_nfeats)\n",
    "\n",
    "train_dataset_files = build_save_dataset('train', fields, opt)\n",
    "\n",
    "build_save_dataset('valid', fields, opt)\n",
    "build_save_vocab(train_dataset_files, fields, opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 17:33:55,111 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 17:33:55,113 INFO]  * vocabulary size. source = 178; target = 101\n",
      "[2019-01-07 17:33:55,114 INFO] Building model...\n",
      "[2019-01-07 17:33:55,275 INFO] NMTModel(\n",
      "  (encoder): CNNEncoder(\n",
      "    (embeddings): Embeddings(\n",
      "      (make_embedding): Sequential(\n",
      "        (emb_luts): Elementwise(\n",
      "          (0): Embedding(178, 500, padding_idx=1)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (linear): Linear(in_features=500, out_features=500, bias=True)\n",
      "    (cnn): StackedCNN(\n",
      "      (layers): ModuleList(\n",
      "        (0): GatedConv(\n",
      "          (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))\n",
      "          (dropout): Dropout(p=0.3)\n",
      "        )\n",
      "        (1): GatedConv(\n",
      "          (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))\n",
      "          (dropout): Dropout(p=0.3)\n",
      "        )\n",
      "        (2): GatedConv(\n",
      "          (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))\n",
      "          (dropout): Dropout(p=0.3)\n",
      "        )\n",
      "        (3): GatedConv(\n",
      "          (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0))\n",
      "          (dropout): Dropout(p=0.3)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (decoder): CNNDecoder(\n",
      "    (embeddings): Embeddings(\n",
      "      (make_embedding): Sequential(\n",
      "        (emb_luts): Elementwise(\n",
      "          (0): Embedding(101, 500, padding_idx=1)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (linear): Linear(in_features=500, out_features=500, bias=True)\n",
      "    (conv_layers): ModuleList(\n",
      "      (0): GatedConv(\n",
      "        (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1))\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "      (1): GatedConv(\n",
      "        (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1))\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "      (2): GatedConv(\n",
      "        (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1))\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "      (3): GatedConv(\n",
      "        (conv): WeightNormConv2d(500, 1000, kernel_size=(3, 1), stride=(1, 1))\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "    )\n",
      "    (attn_layers): ModuleList(\n",
      "      (0): ConvMultiStepAttention(\n",
      "        (linear_in): Linear(in_features=500, out_features=500, bias=True)\n",
      "      )\n",
      "      (1): ConvMultiStepAttention(\n",
      "        (linear_in): Linear(in_features=500, out_features=500, bias=True)\n",
      "      )\n",
      "      (2): ConvMultiStepAttention(\n",
      "        (linear_in): Linear(in_features=500, out_features=500, bias=True)\n",
      "      )\n",
      "      (3): ConvMultiStepAttention(\n",
      "        (linear_in): Linear(in_features=500, out_features=500, bias=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (generator): Sequential(\n",
      "    (0): Linear(in_features=500, out_features=101, bias=True)\n",
      "    (1): LogSoftmax()\n",
      "  )\n",
      ")\n",
      "[2019-01-07 17:33:55,277 INFO] encoder: 6347500\n",
      "[2019-01-07 17:33:55,277 INFO] decoder: 7361601\n",
      "[2019-01-07 17:33:55,278 INFO] * number of parameters: 13709101\n",
      "[2019-01-07 17:33:55,280 INFO] Starting training on GPU: [0, 1]\n",
      "[2019-01-07 17:33:55,280 INFO] Start training...\n",
      "[2019-01-07 17:33:56,332 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "/home/phillnik/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/functional.py:1332: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n",
      "[2019-01-07 17:34:04,053 INFO] Step 50/100000; acc:  38.91; ppl: 11.49; xent: 2.44; lr: 0.00100; 14815/13468 tok/s;      8 sec\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-8acd5c1ebc0f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0menc_layers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m           )\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0msingle_main\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/train_single.py\u001b[0m in \u001b[0;36mmain\u001b[0;34m(opt, device_id)\u001b[0m\n\u001b[1;32m    143\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Starting training on CPU, could be very slow'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m     trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,\n\u001b[0;32m--> 145\u001b[0;31m                   opt.valid_steps)\n\u001b[0m\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensorboard\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, train_iter_fct, valid_iter_fct, train_steps, valid_steps)\u001b[0m\n\u001b[1;32m    175\u001b[0m                         self._gradient_accumulation(\n\u001b[1;32m    176\u001b[0m                             \u001b[0mtrue_batchs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnormalization\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_stats\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m                             report_stats)\n\u001b[0m\u001b[1;32m    178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m                         report_stats = self._maybe_report_training(\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/trainer.py\u001b[0m in \u001b[0;36m_gradient_accumulation\u001b[0;34m(self, true_batchs, normalization, total_stats, report_stats)\u001b[0m\n\u001b[1;32m    283\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m                 \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattns\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 285\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_lengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m                 \u001b[0;31m# 3. Compute loss in shards for memory efficiency.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/models/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, src, tgt, lengths)\u001b[0m\n\u001b[1;32m     44\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory_bank\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menc_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         dec_out, attns = self.decoder(tgt, memory_bank,\n\u001b[0;32m---> 46\u001b[0;31m                                       memory_lengths=lengths)\n\u001b[0m\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdec_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/decoders/cnn_decoder.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, tgt, memory_bank, memory_lengths, step)\u001b[0m\n\u001b[1;32m    110\u001b[0m                           self.cnn_kernel_width - 1, 1)\n\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m         \u001b[0mpad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype_as\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m         \u001b[0mbase_target_emb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "from onmt.train_single import main as single_main\n",
    "from onmt.new_opts import Opts, PreprocessorOpts\n",
    "from onmt.preprocess import build_save_dataset, build_save_vocab\n",
    "import onmt.inputters as inputters\n",
    "\n",
    "opt = Opts(\n",
    "        encoder_type=\"cnn\", \n",
    "        decoder_type=\"cnn\",\n",
    "        data=\"data/USP\", \n",
    "        save_model=\"CNN\", \n",
    "        epochs=10, \n",
    "        batch_size=64, \n",
    "        gpu_ranks=[0, 1], \n",
    "        valid_steps=100, \n",
    "        valid_batch_size=1000, \n",
    "        decay_steps=1000, \n",
    "        start_decay_steps=10000, \n",
    "        learning_rate_decay=0.9, \n",
    "        optim=\"adam\",\n",
    "        learning_rate=0.001,\n",
    "        save_checkpoint_steps=1000,\n",
    "        dec_layers=4,\n",
    "        enc_layers=4\n",
    "          )\n",
    "single_main(opt, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "python  train.py -data /tmp/de2/data -save_model /tmp/extra \\\n",
    "        -layers 6 -rnn_size 512 -word_vec_size 512 -transformer_ff 2048 -heads 8  \\\n",
    "        -encoder_type transformer -decoder_type transformer -position_encoding \\\n",
    "        -train_steps 200000  -max_generator_batches 2 -dropout 0.1 \\\n",
    "        -batch_size 4096 -batch_type tokens -normalization tokens  -accum_count 2 \\\n",
    "        -optim adam -adam_beta2 0.998 -decay_method noam -warmup_steps 8000 -learning_rate 2 \\\n",
    "        -max_grad_norm 0 -param_init 0  -param_init_glorot \\\n",
    "        -label_smoothing 0.1 -valid_steps 10000 -save_checkpoint_steps 10000 \\\n",
    "        -world_size 4 -gpu_ranks 0 1 2 3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 21:54:51,391 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 21:54:51,392 INFO]  * vocabulary size. source = 178; target = 101\n",
      "[2019-01-07 21:54:51,394 INFO] Building model...\n",
      "[2019-01-07 21:54:52,036 INFO] NMTModel(\n",
      "  (encoder): TransformerEncoder(\n",
      "    (embeddings): Embeddings(\n",
      "      (make_embedding): Sequential(\n",
      "        (emb_luts): Elementwise(\n",
      "          (0): Embedding(178, 256, padding_idx=1)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (transformer): ModuleList(\n",
      "      (0): TransformerEncoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "      (1): TransformerEncoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "      (2): TransformerEncoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "      (3): TransformerEncoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (dropout): Dropout(p=0.3)\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "  )\n",
      "  (decoder): TransformerDecoder(\n",
      "    (embeddings): Embeddings(\n",
      "      (make_embedding): Sequential(\n",
      "        (emb_luts): Elementwise(\n",
      "          (0): Embedding(101, 256, padding_idx=1)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (transformer_layers): ModuleList(\n",
      "      (0): TransformerDecoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (context_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm_1): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (layer_norm_2): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (drop): Dropout(p=0.3)\n",
      "      )\n",
      "      (1): TransformerDecoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (context_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm_1): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (layer_norm_2): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (drop): Dropout(p=0.3)\n",
      "      )\n",
      "      (2): TransformerDecoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (context_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm_1): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (layer_norm_2): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (drop): Dropout(p=0.3)\n",
      "      )\n",
      "      (3): TransformerDecoderLayer(\n",
      "        (self_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (context_attn): MultiHeadedAttention(\n",
      "          (linear_keys): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_values): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (linear_query): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (softmax): Softmax()\n",
      "          (dropout): Dropout(p=0.3)\n",
      "          (final_linear): Linear(in_features=256, out_features=256, bias=True)\n",
      "        )\n",
      "        (feed_forward): PositionwiseFeedForward(\n",
      "          (w_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (w_2): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "          (dropout_1): Dropout(p=0.3)\n",
      "          (relu): ReLU()\n",
      "          (dropout_2): Dropout(p=0.3)\n",
      "        )\n",
      "        (layer_norm_1): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (layer_norm_2): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "        (drop): Dropout(p=0.3)\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
      "  )\n",
      "  (generator): Sequential(\n",
      "    (0): Linear(in_features=256, out_features=101, bias=True)\n",
      "    (1): LogSoftmax()\n",
      "  )\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 21:54:52,039 INFO] encoder: 1629184\n",
      "[2019-01-07 21:54:52,040 INFO] decoder: 2690149\n",
      "[2019-01-07 21:54:52,041 INFO] * number of parameters: 4319333\n",
      "[2019-01-07 21:54:52,043 INFO] Starting training on GPU: [0]\n",
      "[2019-01-07 21:54:52,044 INFO] Start training...\n",
      "[2019-01-07 21:54:53,109 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 21:54:56,005 INFO] Step 50/100000; acc:  26.89; ppl: 26.45; xent: 3.28; lr: 0.00100; 36849/33702 tok/s;      3 sec\n",
      "[2019-01-07 21:54:58,729 INFO] Step 100/100000; acc:  40.97; ppl:  7.60; xent: 2.03; lr: 0.00100; 41537/37912 tok/s;      6 sec\n",
      "[2019-01-07 21:54:58,799 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:55:06,007 INFO] Validation perplexity: 5.93472\n",
      "[2019-01-07 21:55:06,008 INFO] Validation accuracy: 44.3341\n",
      "[2019-01-07 21:55:08,753 INFO] Step 150/100000; acc:  46.09; ppl:  5.57; xent: 1.72; lr: 0.00100; 11109/9980 tok/s;     16 sec\n",
      "[2019-01-07 21:55:11,439 INFO] Step 200/100000; acc:  49.05; ppl:  5.00; xent: 1.61; lr: 0.00100; 40430/37361 tok/s;     18 sec\n",
      "[2019-01-07 21:55:11,509 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:55:18,710 INFO] Validation perplexity: 4.68674\n",
      "[2019-01-07 21:55:18,711 INFO] Validation accuracy: 50.9161\n",
      "[2019-01-07 21:55:21,413 INFO] Step 250/100000; acc:  51.34; ppl:  4.70; xent: 1.55; lr: 0.00100; 10914/10092 tok/s;     28 sec\n",
      "[2019-01-07 21:55:24,102 INFO] Step 300/100000; acc:  52.60; ppl:  4.47; xent: 1.50; lr: 0.00100; 41957/37935 tok/s;     31 sec\n",
      "[2019-01-07 21:55:24,171 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:55:31,369 INFO] Validation perplexity: 4.29296\n",
      "[2019-01-07 21:55:31,370 INFO] Validation accuracy: 52.9335\n",
      "[2019-01-07 21:55:34,097 INFO] Step 350/100000; acc:  53.25; ppl:  4.31; xent: 1.46; lr: 0.00100; 11073/10114 tok/s;     41 sec\n",
      "[2019-01-07 21:55:36,788 INFO] Step 400/100000; acc:  54.01; ppl:  4.15; xent: 1.42; lr: 0.00100; 40829/37216 tok/s;     44 sec\n",
      "[2019-01-07 21:55:36,858 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:55:44,052 INFO] Validation perplexity: 3.90826\n",
      "[2019-01-07 21:55:44,053 INFO] Validation accuracy: 55.3321\n",
      "[2019-01-07 21:55:46,756 INFO] Step 450/100000; acc:  55.54; ppl:  3.91; xent: 1.36; lr: 0.00100; 10435/9669 tok/s;     54 sec\n",
      "[2019-01-07 21:55:49,473 INFO] Step 500/100000; acc:  55.48; ppl:  3.89; xent: 1.36; lr: 0.00100; 42257/38255 tok/s;     56 sec\n",
      "[2019-01-07 21:55:49,542 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:55:56,742 INFO] Validation perplexity: 3.66664\n",
      "[2019-01-07 21:55:56,743 INFO] Validation accuracy: 57.2167\n",
      "[2019-01-07 21:55:59,448 INFO] Step 550/100000; acc:  56.98; ppl:  3.71; xent: 1.31; lr: 0.00100; 10643/9698 tok/s;     66 sec\n",
      "[2019-01-07 21:56:02,170 INFO] Step 600/100000; acc:  56.82; ppl:  3.69; xent: 1.30; lr: 0.00100; 41933/38188 tok/s;     69 sec\n",
      "[2019-01-07 21:56:02,240 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:56:09,442 INFO] Validation perplexity: 3.40757\n",
      "[2019-01-07 21:56:09,443 INFO] Validation accuracy: 58.7242\n",
      "[2019-01-07 21:56:12,192 INFO] Step 650/100000; acc:  57.45; ppl:  3.59; xent: 1.28; lr: 0.00100; 11261/10294 tok/s;     79 sec\n",
      "[2019-01-07 21:56:14,906 INFO] Step 700/100000; acc:  57.89; ppl:  3.51; xent: 1.25; lr: 0.00100; 39797/36524 tok/s;     82 sec\n",
      "[2019-01-07 21:56:14,975 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:56:22,175 INFO] Validation perplexity: 3.28957\n",
      "[2019-01-07 21:56:22,177 INFO] Validation accuracy: 59.888\n",
      "[2019-01-07 21:56:24,924 INFO] Step 750/100000; acc:  57.39; ppl:  3.54; xent: 1.26; lr: 0.00100; 11183/10226 tok/s;     92 sec\n",
      "[2019-01-07 21:56:27,600 INFO] Step 800/100000; acc:  59.01; ppl:  3.38; xent: 1.22; lr: 0.00100; 40177/36799 tok/s;     94 sec\n",
      "[2019-01-07 21:56:27,670 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:56:34,870 INFO] Validation perplexity: 3.12851\n",
      "[2019-01-07 21:56:34,872 INFO] Validation accuracy: 61.6423\n",
      "[2019-01-07 21:56:37,570 INFO] Step 850/100000; acc:  58.87; ppl:  3.35; xent: 1.21; lr: 0.00100; 10669/9863 tok/s;    104 sec\n",
      "[2019-01-07 21:56:40,306 INFO] Step 900/100000; acc:  58.73; ppl:  3.37; xent: 1.21; lr: 0.00100; 41228/37348 tok/s;    107 sec\n",
      "[2019-01-07 21:56:40,376 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:56:47,579 INFO] Validation perplexity: 3.07434\n",
      "[2019-01-07 21:56:47,581 INFO] Validation accuracy: 61.0603\n",
      "[2019-01-07 21:56:50,307 INFO] Step 950/100000; acc:  59.35; ppl:  3.31; xent: 1.20; lr: 0.00100; 10935/10020 tok/s;    117 sec\n",
      "[2019-01-07 21:56:52,997 INFO] Step 1000/100000; acc:  59.36; ppl:  3.29; xent: 1.19; lr: 0.00100; 41456/37663 tok/s;    120 sec\n",
      "[2019-01-07 21:56:53,067 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:57:00,270 INFO] Validation perplexity: 2.95285\n",
      "[2019-01-07 21:57:00,272 INFO] Validation accuracy: 62.7224\n",
      "[2019-01-07 21:57:00,273 INFO] Saving checkpoint CNN_step_1000.pt\n",
      "[2019-01-07 21:57:03,097 INFO] Step 1050/100000; acc:  59.99; ppl:  3.23; xent: 1.17; lr: 0.00100; 10760/9922 tok/s;    130 sec\n",
      "[2019-01-07 21:57:05,821 INFO] Step 1100/100000; acc:  60.76; ppl:  3.12; xent: 1.14; lr: 0.00100; 41265/37332 tok/s;    133 sec\n",
      "[2019-01-07 21:57:05,890 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:57:13,092 INFO] Validation perplexity: 2.91191\n",
      "[2019-01-07 21:57:13,094 INFO] Validation accuracy: 62.711\n",
      "[2019-01-07 21:57:15,824 INFO] Step 1150/100000; acc:  60.63; ppl:  3.13; xent: 1.14; lr: 0.00100; 10750/9828 tok/s;    143 sec\n",
      "[2019-01-07 21:57:18,535 INFO] Step 1200/100000; acc:  61.08; ppl:  3.07; xent: 1.12; lr: 0.00100; 41659/38236 tok/s;    145 sec\n",
      "[2019-01-07 21:57:18,604 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:57:25,811 INFO] Validation perplexity: 2.90392\n",
      "[2019-01-07 21:57:25,813 INFO] Validation accuracy: 62.9404\n",
      "[2019-01-07 21:57:28,555 INFO] Step 1250/100000; acc:  60.37; ppl:  3.14; xent: 1.14; lr: 0.00100; 11185/10153 tok/s;    155 sec\n",
      "[2019-01-07 21:57:31,251 INFO] Step 1300/100000; acc:  60.70; ppl:  3.09; xent: 1.13; lr: 0.00100; 40176/36899 tok/s;    158 sec\n",
      "[2019-01-07 21:57:31,320 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:57:38,535 INFO] Validation perplexity: 2.85286\n",
      "[2019-01-07 21:57:38,536 INFO] Validation accuracy: 63.3117\n",
      "[2019-01-07 21:57:41,233 INFO] Step 1350/100000; acc:  61.99; ppl:  2.98; xent: 1.09; lr: 0.00100; 10730/9744 tok/s;    168 sec\n",
      "[2019-01-07 21:57:43,946 INFO] Step 1400/100000; acc:  61.32; ppl:  3.03; xent: 1.11; lr: 0.00100; 41547/38201 tok/s;    171 sec\n",
      "[2019-01-07 21:57:44,018 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:57:51,227 INFO] Validation perplexity: 2.80554\n",
      "[2019-01-07 21:57:51,229 INFO] Validation accuracy: 63.7076\n",
      "[2019-01-07 21:57:53,991 INFO] Step 1450/100000; acc:  61.97; ppl:  2.96; xent: 1.09; lr: 0.00100; 11299/10297 tok/s;    181 sec\n",
      "[2019-01-07 21:57:56,695 INFO] Step 1500/100000; acc:  62.32; ppl:  2.95; xent: 1.08; lr: 0.00100; 39382/36106 tok/s;    184 sec\n",
      "[2019-01-07 21:57:56,765 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:58:03,983 INFO] Validation perplexity: 2.80182\n",
      "[2019-01-07 21:58:03,985 INFO] Validation accuracy: 63.876\n",
      "[2019-01-07 21:58:06,704 INFO] Step 1550/100000; acc:  61.96; ppl:  2.96; xent: 1.08; lr: 0.00100; 11190/10206 tok/s;    194 sec\n",
      "[2019-01-07 21:58:09,410 INFO] Step 1600/100000; acc:  62.44; ppl:  2.91; xent: 1.07; lr: 0.00100; 39871/36686 tok/s;    196 sec\n",
      "[2019-01-07 21:58:09,479 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:58:16,706 INFO] Validation perplexity: 2.72136\n",
      "[2019-01-07 21:58:16,707 INFO] Validation accuracy: 64.8359\n",
      "[2019-01-07 21:58:19,445 INFO] Step 1650/100000; acc:  62.89; ppl:  2.89; xent: 1.06; lr: 0.00100; 10795/9983 tok/s;    206 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 21:58:22,149 INFO] Step 1700/100000; acc:  62.65; ppl:  2.89; xent: 1.06; lr: 0.00100; 41502/37438 tok/s;    209 sec\n",
      "[2019-01-07 21:58:22,218 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:58:29,440 INFO] Validation perplexity: 2.61999\n",
      "[2019-01-07 21:58:29,441 INFO] Validation accuracy: 65.7953\n",
      "[2019-01-07 21:58:32,201 INFO] Step 1750/100000; acc:  62.83; ppl:  2.88; xent: 1.06; lr: 0.00100; 10959/9890 tok/s;    219 sec\n",
      "[2019-01-07 21:58:34,920 INFO] Step 1800/100000; acc:  63.35; ppl:  2.82; xent: 1.04; lr: 0.00100; 40778/37577 tok/s;    222 sec\n",
      "[2019-01-07 21:58:34,990 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:58:42,220 INFO] Validation perplexity: 2.64952\n",
      "[2019-01-07 21:58:42,221 INFO] Validation accuracy: 65.5346\n",
      "[2019-01-07 21:58:44,891 INFO] Step 1850/100000; acc:  63.00; ppl:  2.84; xent: 1.04; lr: 0.00100; 10934/9917 tok/s;    232 sec\n",
      "[2019-01-07 21:58:47,625 INFO] Step 1900/100000; acc:  63.19; ppl:  2.86; xent: 1.05; lr: 0.00100; 40531/37412 tok/s;    235 sec\n",
      "[2019-01-07 21:58:47,694 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:58:54,913 INFO] Validation perplexity: 2.63845\n",
      "[2019-01-07 21:58:54,915 INFO] Validation accuracy: 65.3987\n",
      "[2019-01-07 21:58:57,629 INFO] Step 1950/100000; acc:  62.22; ppl:  2.90; xent: 1.07; lr: 0.00100; 10637/9887 tok/s;    245 sec\n",
      "[2019-01-07 21:59:00,333 INFO] Step 2000/100000; acc:  63.37; ppl:  2.83; xent: 1.04; lr: 0.00100; 42236/38111 tok/s;    247 sec\n",
      "[2019-01-07 21:59:00,404 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:59:07,629 INFO] Validation perplexity: 2.56187\n",
      "[2019-01-07 21:59:07,631 INFO] Validation accuracy: 66.5527\n",
      "[2019-01-07 21:59:07,632 INFO] Saving checkpoint CNN_step_2000.pt\n",
      "[2019-01-07 21:59:10,461 INFO] Step 2050/100000; acc:  64.02; ppl:  2.75; xent: 1.01; lr: 0.00100; 10670/9817 tok/s;    257 sec\n",
      "[2019-01-07 21:59:13,179 INFO] Step 2100/100000; acc:  63.01; ppl:  2.85; xent: 1.05; lr: 0.00100; 41291/37570 tok/s;    260 sec\n",
      "[2019-01-07 21:59:13,249 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:59:20,477 INFO] Validation perplexity: 2.52472\n",
      "[2019-01-07 21:59:20,478 INFO] Validation accuracy: 67.026\n",
      "[2019-01-07 21:59:23,147 INFO] Step 2150/100000; acc:  63.96; ppl:  2.79; xent: 1.02; lr: 0.00100; 10823/9820 tok/s;    270 sec\n",
      "[2019-01-07 21:59:25,890 INFO] Step 2200/100000; acc:  63.82; ppl:  2.78; xent: 1.02; lr: 0.00100; 40739/37131 tok/s;    273 sec\n",
      "[2019-01-07 21:59:25,959 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:59:33,184 INFO] Validation perplexity: 2.49457\n",
      "[2019-01-07 21:59:33,186 INFO] Validation accuracy: 67.3262\n",
      "[2019-01-07 21:59:35,908 INFO] Step 2250/100000; acc:  63.61; ppl:  2.78; xent: 1.02; lr: 0.00100; 10926/10064 tok/s;    283 sec\n",
      "[2019-01-07 21:59:38,618 INFO] Step 2300/100000; acc:  64.03; ppl:  2.77; xent: 1.02; lr: 0.00100; 41166/37391 tok/s;    286 sec\n",
      "[2019-01-07 21:59:38,687 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:59:45,914 INFO] Validation perplexity: 2.48026\n",
      "[2019-01-07 21:59:45,916 INFO] Validation accuracy: 67.3976\n",
      "[2019-01-07 21:59:48,602 INFO] Step 2350/100000; acc:  64.67; ppl:  2.70; xent: 0.99; lr: 0.00100; 10409/9352 tok/s;    295 sec\n",
      "[2019-01-07 21:59:51,354 INFO] Step 2400/100000; acc:  64.05; ppl:  2.74; xent: 1.01; lr: 0.00100; 41920/38920 tok/s;    298 sec\n",
      "[2019-01-07 21:59:51,424 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 21:59:58,654 INFO] Validation perplexity: 2.48283\n",
      "[2019-01-07 21:59:58,655 INFO] Validation accuracy: 67.436\n",
      "[2019-01-07 22:00:01,355 INFO] Step 2450/100000; acc:  64.49; ppl:  2.71; xent: 1.00; lr: 0.00100; 10890/9953 tok/s;    308 sec\n",
      "[2019-01-07 22:00:04,048 INFO] Step 2500/100000; acc:  64.14; ppl:  2.76; xent: 1.02; lr: 0.00100; 41371/37750 tok/s;    311 sec\n",
      "[2019-01-07 22:00:04,118 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:00:11,340 INFO] Validation perplexity: 2.45801\n",
      "[2019-01-07 22:00:11,342 INFO] Validation accuracy: 67.9819\n",
      "[2019-01-07 22:00:14,126 INFO] Step 2550/100000; acc:  64.25; ppl:  2.72; xent: 1.00; lr: 0.00100; 11610/10675 tok/s;    321 sec\n",
      "[2019-01-07 22:00:16,821 INFO] Step 2600/100000; acc:  65.36; ppl:  2.64; xent: 0.97; lr: 0.00100; 38533/34808 tok/s;    324 sec\n",
      "[2019-01-07 22:00:16,890 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:00:24,113 INFO] Validation perplexity: 2.44866\n",
      "[2019-01-07 22:00:24,115 INFO] Validation accuracy: 67.4855\n",
      "[2019-01-07 22:00:26,821 INFO] Step 2650/100000; acc:  65.13; ppl:  2.67; xent: 0.98; lr: 0.00100; 10836/9879 tok/s;    334 sec\n",
      "[2019-01-07 22:00:29,534 INFO] Step 2700/100000; acc:  64.51; ppl:  2.71; xent: 1.00; lr: 0.00100; 41080/37647 tok/s;    336 sec\n",
      "[2019-01-07 22:00:29,604 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:00:36,846 INFO] Validation perplexity: 2.43481\n",
      "[2019-01-07 22:00:36,848 INFO] Validation accuracy: 67.8515\n",
      "[2019-01-07 22:00:39,620 INFO] Step 2750/100000; acc:  64.78; ppl:  2.68; xent: 0.98; lr: 0.00100; 10975/9916 tok/s;    347 sec\n",
      "[2019-01-07 22:00:42,304 INFO] Step 2800/100000; acc:  65.30; ppl:  2.64; xent: 0.97; lr: 0.00100; 40264/37454 tok/s;    349 sec\n",
      "[2019-01-07 22:00:42,375 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:00:49,604 INFO] Validation perplexity: 2.40933\n",
      "[2019-01-07 22:00:49,605 INFO] Validation accuracy: 68.0807\n",
      "[2019-01-07 22:00:52,340 INFO] Step 2850/100000; acc:  65.35; ppl:  2.64; xent: 0.97; lr: 0.00100; 11132/10215 tok/s;    359 sec\n",
      "[2019-01-07 22:00:55,047 INFO] Step 2900/100000; acc:  64.66; ppl:  2.69; xent: 0.99; lr: 0.00100; 40072/36604 tok/s;    362 sec\n",
      "[2019-01-07 22:00:55,117 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:01:02,343 INFO] Validation perplexity: 2.46509\n",
      "[2019-01-07 22:01:02,345 INFO] Validation accuracy: 67.4626\n",
      "[2019-01-07 22:01:05,095 INFO] Step 2950/100000; acc:  64.81; ppl:  2.68; xent: 0.98; lr: 0.00100; 11019/10090 tok/s;    372 sec\n",
      "[2019-01-07 22:01:07,590 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:01:09,526 INFO] Step 3000/100000; acc:  65.70; ppl:  2.61; xent: 0.96; lr: 0.00100; 24000/22109 tok/s;    376 sec\n",
      "[2019-01-07 22:01:09,597 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:01:16,813 INFO] Validation perplexity: 2.39924\n",
      "[2019-01-07 22:01:16,815 INFO] Validation accuracy: 68.2826\n",
      "[2019-01-07 22:01:16,817 INFO] Saving checkpoint CNN_step_3000.pt\n",
      "[2019-01-07 22:01:19,619 INFO] Step 3050/100000; acc:  65.58; ppl:  2.60; xent: 0.96; lr: 0.00100; 10738/9670 tok/s;    387 sec\n",
      "[2019-01-07 22:01:22,345 INFO] Step 3100/100000; acc:  64.74; ppl:  2.67; xent: 0.98; lr: 0.00100; 41913/38029 tok/s;    389 sec\n",
      "[2019-01-07 22:01:22,415 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:01:29,643 INFO] Validation perplexity: 2.36138\n",
      "[2019-01-07 22:01:29,645 INFO] Validation accuracy: 68.9263\n",
      "[2019-01-07 22:01:32,319 INFO] Step 3150/100000; acc:  66.05; ppl:  2.58; xent: 0.95; lr: 0.00100; 10848/9917 tok/s;    399 sec\n",
      "[2019-01-07 22:01:35,026 INFO] Step 3200/100000; acc:  65.75; ppl:  2.61; xent: 0.96; lr: 0.00100; 40540/37437 tok/s;    402 sec\n",
      "[2019-01-07 22:01:35,096 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:01:42,324 INFO] Validation perplexity: 2.35066\n",
      "[2019-01-07 22:01:42,325 INFO] Validation accuracy: 69.2533\n",
      "[2019-01-07 22:01:45,045 INFO] Step 3250/100000; acc:  65.69; ppl:  2.60; xent: 0.95; lr: 0.00100; 11015/10137 tok/s;    412 sec\n",
      "[2019-01-07 22:01:47,808 INFO] Step 3300/100000; acc:  65.89; ppl:  2.61; xent: 0.96; lr: 0.00100; 40801/37230 tok/s;    415 sec\n",
      "[2019-01-07 22:01:47,877 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:01:55,104 INFO] Validation perplexity: 2.34577\n",
      "[2019-01-07 22:01:55,106 INFO] Validation accuracy: 69.1426\n",
      "[2019-01-07 22:01:57,782 INFO] Step 3350/100000; acc:  66.11; ppl:  2.56; xent: 0.94; lr: 0.00100; 10735/9892 tok/s;    425 sec\n",
      "[2019-01-07 22:02:00,507 INFO] Step 3400/100000; acc:  65.94; ppl:  2.58; xent: 0.95; lr: 0.00100; 40085/36104 tok/s;    427 sec\n",
      "[2019-01-07 22:02:00,577 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:02:07,807 INFO] Validation perplexity: 2.37797\n",
      "[2019-01-07 22:02:07,809 INFO] Validation accuracy: 68.5458\n",
      "[2019-01-07 22:02:10,515 INFO] Step 3450/100000; acc:  66.39; ppl:  2.54; xent: 0.93; lr: 0.00100; 11080/10256 tok/s;    437 sec\n",
      "[2019-01-07 22:02:13,222 INFO] Step 3500/100000; acc:  65.91; ppl:  2.60; xent: 0.95; lr: 0.00100; 40033/36187 tok/s;    440 sec\n",
      "[2019-01-07 22:02:13,292 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:02:20,519 INFO] Validation perplexity: 2.39006\n",
      "[2019-01-07 22:02:20,520 INFO] Validation accuracy: 68.1474\n",
      "[2019-01-07 22:02:23,196 INFO] Step 3550/100000; acc:  66.25; ppl:  2.56; xent: 0.94; lr: 0.00100; 11072/10139 tok/s;    450 sec\n",
      "[2019-01-07 22:02:25,925 INFO] Step 3600/100000; acc:  65.93; ppl:  2.58; xent: 0.95; lr: 0.00100; 40903/36784 tok/s;    453 sec\n",
      "[2019-01-07 22:02:25,995 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:02:33,230 INFO] Validation perplexity: 2.37727\n",
      "[2019-01-07 22:02:33,231 INFO] Validation accuracy: 68.2151\n",
      "[2019-01-07 22:02:35,951 INFO] Step 3650/100000; acc:  66.55; ppl:  2.54; xent: 0.93; lr: 0.00100; 11084/10268 tok/s;    463 sec\n",
      "[2019-01-07 22:02:38,711 INFO] Step 3700/100000; acc:  65.43; ppl:  2.61; xent: 0.96; lr: 0.00100; 41833/38078 tok/s;    466 sec\n",
      "[2019-01-07 22:02:38,782 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:02:46,009 INFO] Validation perplexity: 2.36434\n",
      "[2019-01-07 22:02:46,011 INFO] Validation accuracy: 68.6694\n",
      "[2019-01-07 22:02:48,703 INFO] Step 3750/100000; acc:  66.75; ppl:  2.53; xent: 0.93; lr: 0.00100; 10379/9533 tok/s;    476 sec\n",
      "[2019-01-07 22:02:51,403 INFO] Step 3800/100000; acc:  66.56; ppl:  2.54; xent: 0.93; lr: 0.00100; 39884/37123 tok/s;    478 sec\n",
      "[2019-01-07 22:02:51,473 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:02:58,701 INFO] Validation perplexity: 2.34721\n",
      "[2019-01-07 22:02:58,703 INFO] Validation accuracy: 69.3216\n",
      "[2019-01-07 22:03:01,425 INFO] Step 3850/100000; acc:  66.20; ppl:  2.56; xent: 0.94; lr: 0.00100; 11204/10210 tok/s;    488 sec\n",
      "[2019-01-07 22:03:04,143 INFO] Step 3900/100000; acc:  66.06; ppl:  2.57; xent: 0.94; lr: 0.00100; 39695/36024 tok/s;    491 sec\n",
      "[2019-01-07 22:03:04,213 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:03:11,438 INFO] Validation perplexity: 2.29483\n",
      "[2019-01-07 22:03:11,440 INFO] Validation accuracy: 69.8147\n",
      "[2019-01-07 22:03:14,172 INFO] Step 3950/100000; acc:  66.55; ppl:  2.55; xent: 0.94; lr: 0.00100; 11484/10352 tok/s;    501 sec\n",
      "[2019-01-07 22:03:16,875 INFO] Step 4000/100000; acc:  66.60; ppl:  2.53; xent: 0.93; lr: 0.00100; 39808/36961 tok/s;    504 sec\n",
      "[2019-01-07 22:03:16,945 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:03:24,173 INFO] Validation perplexity: 2.26779\n",
      "[2019-01-07 22:03:24,175 INFO] Validation accuracy: 70.2909\n",
      "[2019-01-07 22:03:24,176 INFO] Saving checkpoint CNN_step_4000.pt\n",
      "[2019-01-07 22:03:27,019 INFO] Step 4050/100000; acc:  66.72; ppl:  2.53; xent: 0.93; lr: 0.00100; 10829/9829 tok/s;    514 sec\n",
      "[2019-01-07 22:03:29,751 INFO] Step 4100/100000; acc:  66.61; ppl:  2.52; xent: 0.93; lr: 0.00100; 40310/36725 tok/s;    517 sec\n",
      "[2019-01-07 22:03:29,821 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:03:37,057 INFO] Validation perplexity: 2.27761\n",
      "[2019-01-07 22:03:37,059 INFO] Validation accuracy: 69.7476\n",
      "[2019-01-07 22:03:39,800 INFO] Step 4150/100000; acc:  66.44; ppl:  2.53; xent: 0.93; lr: 0.00100; 11230/10261 tok/s;    527 sec\n",
      "[2019-01-07 22:03:42,562 INFO] Step 4200/100000; acc:  66.47; ppl:  2.53; xent: 0.93; lr: 0.00100; 41162/37469 tok/s;    529 sec\n",
      "[2019-01-07 22:03:42,632 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:03:49,857 INFO] Validation perplexity: 2.2951\n",
      "[2019-01-07 22:03:49,859 INFO] Validation accuracy: 69.8405\n",
      "[2019-01-07 22:03:52,558 INFO] Step 4250/100000; acc:  66.49; ppl:  2.53; xent: 0.93; lr: 0.00100; 10612/9763 tok/s;    539 sec\n",
      "[2019-01-07 22:03:55,271 INFO] Step 4300/100000; acc:  66.40; ppl:  2.52; xent: 0.93; lr: 0.00100; 40397/36601 tok/s;    542 sec\n",
      "[2019-01-07 22:03:55,340 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:04:02,569 INFO] Validation perplexity: 2.27755\n",
      "[2019-01-07 22:04:02,570 INFO] Validation accuracy: 69.8133\n",
      "[2019-01-07 22:04:05,291 INFO] Step 4350/100000; acc:  66.80; ppl:  2.51; xent: 0.92; lr: 0.00100; 10807/10080 tok/s;    552 sec\n",
      "[2019-01-07 22:04:08,032 INFO] Step 4400/100000; acc:  66.83; ppl:  2.50; xent: 0.91; lr: 0.00100; 42086/37814 tok/s;    555 sec\n",
      "[2019-01-07 22:04:08,102 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:04:15,333 INFO] Validation perplexity: 2.25252\n",
      "[2019-01-07 22:04:15,335 INFO] Validation accuracy: 70.3852\n",
      "[2019-01-07 22:04:18,053 INFO] Step 4450/100000; acc:  67.25; ppl:  2.47; xent: 0.91; lr: 0.00100; 11104/10084 tok/s;    565 sec\n",
      "[2019-01-07 22:04:20,752 INFO] Step 4500/100000; acc:  66.86; ppl:  2.52; xent: 0.92; lr: 0.00100; 39839/36703 tok/s;    568 sec\n",
      "[2019-01-07 22:04:20,824 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:04:28,062 INFO] Validation perplexity: 2.31587\n",
      "[2019-01-07 22:04:28,064 INFO] Validation accuracy: 69.1292\n",
      "[2019-01-07 22:04:30,785 INFO] Step 4550/100000; acc:  67.23; ppl:  2.47; xent: 0.90; lr: 0.00100; 10806/9859 tok/s;    578 sec\n",
      "[2019-01-07 22:04:33,499 INFO] Step 4600/100000; acc:  67.17; ppl:  2.49; xent: 0.91; lr: 0.00100; 40530/37782 tok/s;    580 sec\n",
      "[2019-01-07 22:04:33,568 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:04:40,794 INFO] Validation perplexity: 2.27948\n",
      "[2019-01-07 22:04:40,795 INFO] Validation accuracy: 69.9216\n",
      "[2019-01-07 22:04:43,521 INFO] Step 4650/100000; acc:  67.14; ppl:  2.48; xent: 0.91; lr: 0.00100; 11307/10267 tok/s;    590 sec\n",
      "[2019-01-07 22:04:46,211 INFO] Step 4700/100000; acc:  67.59; ppl:  2.45; xent: 0.90; lr: 0.00100; 38103/34389 tok/s;    593 sec\n",
      "[2019-01-07 22:04:46,281 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:04:53,509 INFO] Validation perplexity: 2.32959\n",
      "[2019-01-07 22:04:53,511 INFO] Validation accuracy: 69.229\n",
      "[2019-01-07 22:04:56,236 INFO] Step 4750/100000; acc:  67.12; ppl:  2.47; xent: 0.91; lr: 0.00100; 11227/10332 tok/s;    603 sec\n",
      "[2019-01-07 22:04:58,962 INFO] Step 4800/100000; acc:  67.28; ppl:  2.47; xent: 0.90; lr: 0.00100; 40678/36830 tok/s;    606 sec\n",
      "[2019-01-07 22:04:59,032 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:05:06,262 INFO] Validation perplexity: 2.23538\n",
      "[2019-01-07 22:05:06,264 INFO] Validation accuracy: 70.7966\n",
      "[2019-01-07 22:05:08,987 INFO] Step 4850/100000; acc:  67.53; ppl:  2.47; xent: 0.90; lr: 0.00100; 10848/9882 tok/s;    616 sec\n",
      "[2019-01-07 22:05:11,739 INFO] Step 4900/100000; acc:  66.43; ppl:  2.53; xent: 0.93; lr: 0.00100; 40167/37219 tok/s;    619 sec\n",
      "[2019-01-07 22:05:11,812 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:05:19,049 INFO] Validation perplexity: 2.33894\n",
      "[2019-01-07 22:05:19,051 INFO] Validation accuracy: 68.9478\n",
      "[2019-01-07 22:05:21,785 INFO] Step 4950/100000; acc:  67.03; ppl:  2.50; xent: 0.92; lr: 0.00100; 11531/10542 tok/s;    629 sec\n",
      "[2019-01-07 22:05:24,499 INFO] Step 5000/100000; acc:  67.89; ppl:  2.43; xent: 0.89; lr: 0.00100; 39583/36195 tok/s;    631 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:05:24,572 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:05:31,824 INFO] Validation perplexity: 2.2783\n",
      "[2019-01-07 22:05:31,825 INFO] Validation accuracy: 69.8165\n",
      "[2019-01-07 22:05:31,827 INFO] Saving checkpoint CNN_step_5000.pt\n",
      "[2019-01-07 22:05:34,682 INFO] Step 5050/100000; acc:  67.30; ppl:  2.48; xent: 0.91; lr: 0.00100; 10899/9967 tok/s;    642 sec\n",
      "[2019-01-07 22:05:37,381 INFO] Step 5100/100000; acc:  67.66; ppl:  2.45; xent: 0.90; lr: 0.00100; 38951/35450 tok/s;    644 sec\n",
      "[2019-01-07 22:05:37,453 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:05:44,700 INFO] Validation perplexity: 2.24711\n",
      "[2019-01-07 22:05:44,702 INFO] Validation accuracy: 70.6391\n",
      "[2019-01-07 22:05:47,442 INFO] Step 5150/100000; acc:  67.00; ppl:  2.49; xent: 0.91; lr: 0.00100; 11233/10143 tok/s;    654 sec\n",
      "[2019-01-07 22:05:50,136 INFO] Step 5200/100000; acc:  67.74; ppl:  2.44; xent: 0.89; lr: 0.00100; 40314/37309 tok/s;    657 sec\n",
      "[2019-01-07 22:05:50,206 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:05:57,446 INFO] Validation perplexity: 2.34123\n",
      "[2019-01-07 22:05:57,448 INFO] Validation accuracy: 68.807\n",
      "[2019-01-07 22:06:00,163 INFO] Step 5250/100000; acc:  67.30; ppl:  2.47; xent: 0.90; lr: 0.00100; 11058/10096 tok/s;    667 sec\n",
      "[2019-01-07 22:06:02,873 INFO] Step 5300/100000; acc:  67.63; ppl:  2.45; xent: 0.90; lr: 0.00100; 39321/35747 tok/s;    670 sec\n",
      "[2019-01-07 22:06:02,944 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:06:10,182 INFO] Validation perplexity: 2.24063\n",
      "[2019-01-07 22:06:10,184 INFO] Validation accuracy: 70.9794\n",
      "[2019-01-07 22:06:12,932 INFO] Step 5350/100000; acc:  67.61; ppl:  2.44; xent: 0.89; lr: 0.00100; 11333/10298 tok/s;    680 sec\n",
      "[2019-01-07 22:06:15,652 INFO] Step 5400/100000; acc:  67.69; ppl:  2.44; xent: 0.89; lr: 0.00100; 40193/37179 tok/s;    683 sec\n",
      "[2019-01-07 22:06:15,723 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:06:22,958 INFO] Validation perplexity: 2.23892\n",
      "[2019-01-07 22:06:22,960 INFO] Validation accuracy: 70.5877\n",
      "[2019-01-07 22:06:25,658 INFO] Step 5450/100000; acc:  67.34; ppl:  2.47; xent: 0.90; lr: 0.00100; 11265/10203 tok/s;    693 sec\n",
      "[2019-01-07 22:06:28,431 INFO] Step 5500/100000; acc:  67.63; ppl:  2.45; xent: 0.90; lr: 0.00100; 39790/36405 tok/s;    695 sec\n",
      "[2019-01-07 22:06:28,501 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:06:35,744 INFO] Validation perplexity: 2.23127\n",
      "[2019-01-07 22:06:35,746 INFO] Validation accuracy: 70.6958\n",
      "[2019-01-07 22:06:38,469 INFO] Step 5550/100000; acc:  68.25; ppl:  2.40; xent: 0.88; lr: 0.00100; 11120/10162 tok/s;    705 sec\n",
      "[2019-01-07 22:06:41,155 INFO] Step 5600/100000; acc:  67.92; ppl:  2.43; xent: 0.89; lr: 0.00100; 40127/36318 tok/s;    708 sec\n",
      "[2019-01-07 22:06:41,225 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:06:48,453 INFO] Validation perplexity: 2.19248\n",
      "[2019-01-07 22:06:48,455 INFO] Validation accuracy: 71.5629\n",
      "[2019-01-07 22:06:51,180 INFO] Step 5650/100000; acc:  67.73; ppl:  2.44; xent: 0.89; lr: 0.00100; 10952/10149 tok/s;    718 sec\n",
      "[2019-01-07 22:06:53,903 INFO] Step 5700/100000; acc:  67.52; ppl:  2.45; xent: 0.90; lr: 0.00100; 40698/36867 tok/s;    721 sec\n",
      "[2019-01-07 22:06:53,973 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:07:01,206 INFO] Validation perplexity: 2.18341\n",
      "[2019-01-07 22:07:01,208 INFO] Validation accuracy: 71.3777\n",
      "[2019-01-07 22:07:03,883 INFO] Step 5750/100000; acc:  67.97; ppl:  2.41; xent: 0.88; lr: 0.00100; 10765/9938 tok/s;    731 sec\n",
      "[2019-01-07 22:07:06,641 INFO] Step 5800/100000; acc:  67.82; ppl:  2.44; xent: 0.89; lr: 0.00100; 40719/37091 tok/s;    734 sec\n",
      "[2019-01-07 22:07:06,712 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:07:13,937 INFO] Validation perplexity: 2.29411\n",
      "[2019-01-07 22:07:13,939 INFO] Validation accuracy: 69.6632\n",
      "[2019-01-07 22:07:16,658 INFO] Step 5850/100000; acc:  67.95; ppl:  2.42; xent: 0.88; lr: 0.00100; 10687/9968 tok/s;    744 sec\n",
      "[2019-01-07 22:07:19,400 INFO] Step 5900/100000; acc:  67.29; ppl:  2.46; xent: 0.90; lr: 0.00100; 39672/35791 tok/s;    746 sec\n",
      "[2019-01-07 22:07:19,472 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:07:26,728 INFO] Validation perplexity: 2.22578\n",
      "[2019-01-07 22:07:26,730 INFO] Validation accuracy: 70.4698\n",
      "[2019-01-07 22:07:30,241 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:07:31,188 INFO] Step 5950/100000; acc:  68.54; ppl:  2.38; xent: 0.87; lr: 0.00100; 9305/8602 tok/s;    758 sec\n",
      "[2019-01-07 22:07:33,882 INFO] Step 6000/100000; acc:  68.13; ppl:  2.41; xent: 0.88; lr: 0.00100; 40322/36469 tok/s;    761 sec\n",
      "[2019-01-07 22:07:33,953 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:07:41,170 INFO] Validation perplexity: 2.19373\n",
      "[2019-01-07 22:07:41,172 INFO] Validation accuracy: 71.4882\n",
      "[2019-01-07 22:07:41,173 INFO] Saving checkpoint CNN_step_6000.pt\n",
      "[2019-01-07 22:07:43,991 INFO] Step 6050/100000; acc:  67.81; ppl:  2.41; xent: 0.88; lr: 0.00100; 11134/10126 tok/s;    771 sec\n",
      "[2019-01-07 22:07:46,709 INFO] Step 6100/100000; acc:  68.31; ppl:  2.39; xent: 0.87; lr: 0.00100; 40758/37210 tok/s;    774 sec\n",
      "[2019-01-07 22:07:46,780 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:07:54,005 INFO] Validation perplexity: 2.20861\n",
      "[2019-01-07 22:07:54,007 INFO] Validation accuracy: 71.2559\n",
      "[2019-01-07 22:07:56,756 INFO] Step 6150/100000; acc:  68.47; ppl:  2.39; xent: 0.87; lr: 0.00100; 10793/9869 tok/s;    784 sec\n",
      "[2019-01-07 22:07:59,500 INFO] Step 6200/100000; acc:  67.66; ppl:  2.43; xent: 0.89; lr: 0.00100; 41484/38216 tok/s;    786 sec\n",
      "[2019-01-07 22:07:59,570 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:08:06,798 INFO] Validation perplexity: 2.18391\n",
      "[2019-01-07 22:08:06,799 INFO] Validation accuracy: 71.7272\n",
      "[2019-01-07 22:08:09,503 INFO] Step 6250/100000; acc:  68.52; ppl:  2.37; xent: 0.86; lr: 0.00100; 10748/9817 tok/s;    796 sec\n",
      "[2019-01-07 22:08:12,230 INFO] Step 6300/100000; acc:  68.45; ppl:  2.39; xent: 0.87; lr: 0.00100; 40570/37291 tok/s;    799 sec\n",
      "[2019-01-07 22:08:12,300 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:08:19,530 INFO] Validation perplexity: 2.16787\n",
      "[2019-01-07 22:08:19,532 INFO] Validation accuracy: 71.9814\n",
      "[2019-01-07 22:08:22,275 INFO] Step 6350/100000; acc:  68.26; ppl:  2.40; xent: 0.88; lr: 0.00100; 10996/9983 tok/s;    809 sec\n",
      "[2019-01-07 22:08:24,967 INFO] Step 6400/100000; acc:  68.69; ppl:  2.36; xent: 0.86; lr: 0.00100; 39794/36557 tok/s;    812 sec\n",
      "[2019-01-07 22:08:25,041 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:08:32,279 INFO] Validation perplexity: 2.23918\n",
      "[2019-01-07 22:08:32,280 INFO] Validation accuracy: 70.239\n",
      "[2019-01-07 22:08:35,004 INFO] Step 6450/100000; acc:  68.67; ppl:  2.38; xent: 0.87; lr: 0.00100; 10987/9891 tok/s;    822 sec\n",
      "[2019-01-07 22:08:37,705 INFO] Step 6500/100000; acc:  68.16; ppl:  2.42; xent: 0.88; lr: 0.00100; 40361/37019 tok/s;    825 sec\n",
      "[2019-01-07 22:08:37,775 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:08:45,007 INFO] Validation perplexity: 2.18429\n",
      "[2019-01-07 22:08:45,008 INFO] Validation accuracy: 71.5033\n",
      "[2019-01-07 22:08:47,728 INFO] Step 6550/100000; acc:  68.71; ppl:  2.37; xent: 0.86; lr: 0.00100; 11206/10147 tok/s;    835 sec\n",
      "[2019-01-07 22:08:50,447 INFO] Step 6600/100000; acc:  68.92; ppl:  2.36; xent: 0.86; lr: 0.00100; 40938/37814 tok/s;    837 sec\n",
      "[2019-01-07 22:08:50,518 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:08:57,760 INFO] Validation perplexity: 2.20596\n",
      "[2019-01-07 22:08:57,761 INFO] Validation accuracy: 71.0379\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:09:00,531 INFO] Step 6650/100000; acc:  67.97; ppl:  2.42; xent: 0.88; lr: 0.00100; 11414/10479 tok/s;    847 sec\n",
      "[2019-01-07 22:09:03,206 INFO] Step 6700/100000; acc:  68.67; ppl:  2.37; xent: 0.86; lr: 0.00100; 39800/35938 tok/s;    850 sec\n",
      "[2019-01-07 22:09:03,276 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:09:10,518 INFO] Validation perplexity: 2.21766\n",
      "[2019-01-07 22:09:10,519 INFO] Validation accuracy: 71.0918\n",
      "[2019-01-07 22:09:13,241 INFO] Step 6750/100000; acc:  69.15; ppl:  2.35; xent: 0.86; lr: 0.00100; 10681/10020 tok/s;    860 sec\n",
      "[2019-01-07 22:09:15,952 INFO] Step 6800/100000; acc:  68.46; ppl:  2.39; xent: 0.87; lr: 0.00100; 39934/36163 tok/s;    863 sec\n",
      "[2019-01-07 22:09:16,022 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:09:23,255 INFO] Validation perplexity: 2.18417\n",
      "[2019-01-07 22:09:23,257 INFO] Validation accuracy: 71.2082\n",
      "[2019-01-07 22:09:25,983 INFO] Step 6850/100000; acc:  68.44; ppl:  2.38; xent: 0.87; lr: 0.00100; 11104/10148 tok/s;    873 sec\n",
      "[2019-01-07 22:09:28,678 INFO] Step 6900/100000; acc:  68.64; ppl:  2.39; xent: 0.87; lr: 0.00100; 41072/37417 tok/s;    876 sec\n",
      "[2019-01-07 22:09:28,748 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:09:35,989 INFO] Validation perplexity: 2.18442\n",
      "[2019-01-07 22:09:35,990 INFO] Validation accuracy: 70.9567\n",
      "[2019-01-07 22:09:38,707 INFO] Step 6950/100000; acc:  68.74; ppl:  2.38; xent: 0.87; lr: 0.00100; 11052/10083 tok/s;    886 sec\n",
      "[2019-01-07 22:09:41,412 INFO] Step 7000/100000; acc:  68.61; ppl:  2.38; xent: 0.87; lr: 0.00100; 41152/37637 tok/s;    888 sec\n",
      "[2019-01-07 22:09:41,486 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:09:48,726 INFO] Validation perplexity: 2.18076\n",
      "[2019-01-07 22:09:48,728 INFO] Validation accuracy: 71.6248\n",
      "[2019-01-07 22:09:48,730 INFO] Saving checkpoint CNN_step_7000.pt\n",
      "[2019-01-07 22:09:51,560 INFO] Step 7050/100000; acc:  68.98; ppl:  2.34; xent: 0.85; lr: 0.00100; 10912/9867 tok/s;    898 sec\n",
      "[2019-01-07 22:09:54,245 INFO] Step 7100/100000; acc:  68.73; ppl:  2.36; xent: 0.86; lr: 0.00100; 40346/37110 tok/s;    901 sec\n",
      "[2019-01-07 22:09:54,315 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:10:01,555 INFO] Validation perplexity: 2.1729\n",
      "[2019-01-07 22:10:01,556 INFO] Validation accuracy: 71.6051\n",
      "[2019-01-07 22:10:04,326 INFO] Step 7150/100000; acc:  68.78; ppl:  2.36; xent: 0.86; lr: 0.00100; 11417/10412 tok/s;    911 sec\n",
      "[2019-01-07 22:10:07,027 INFO] Step 7200/100000; acc:  68.60; ppl:  2.37; xent: 0.86; lr: 0.00100; 39771/36111 tok/s;    914 sec\n",
      "[2019-01-07 22:10:07,097 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:10:14,333 INFO] Validation perplexity: 2.13195\n",
      "[2019-01-07 22:10:14,334 INFO] Validation accuracy: 72.5841\n",
      "[2019-01-07 22:10:17,028 INFO] Step 7250/100000; acc:  68.84; ppl:  2.36; xent: 0.86; lr: 0.00100; 10923/10060 tok/s;    924 sec\n",
      "[2019-01-07 22:10:19,711 INFO] Step 7300/100000; acc:  68.97; ppl:  2.35; xent: 0.85; lr: 0.00100; 39828/36682 tok/s;    927 sec\n",
      "[2019-01-07 22:10:19,784 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:10:27,026 INFO] Validation perplexity: 2.28735\n",
      "[2019-01-07 22:10:27,028 INFO] Validation accuracy: 69.9984\n",
      "[2019-01-07 22:10:29,771 INFO] Step 7350/100000; acc:  68.88; ppl:  2.35; xent: 0.85; lr: 0.00100; 11333/10229 tok/s;    937 sec\n",
      "[2019-01-07 22:10:32,505 INFO] Step 7400/100000; acc:  68.97; ppl:  2.35; xent: 0.85; lr: 0.00100; 42390/38627 tok/s;    939 sec\n",
      "[2019-01-07 22:10:32,577 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:10:39,820 INFO] Validation perplexity: 2.17951\n",
      "[2019-01-07 22:10:39,822 INFO] Validation accuracy: 71.4664\n",
      "[2019-01-07 22:10:42,511 INFO] Step 7450/100000; acc:  69.25; ppl:  2.32; xent: 0.84; lr: 0.00100; 10190/9483 tok/s;    949 sec\n",
      "[2019-01-07 22:10:45,244 INFO] Step 7500/100000; acc:  68.70; ppl:  2.36; xent: 0.86; lr: 0.00100; 41540/37643 tok/s;    952 sec\n",
      "[2019-01-07 22:10:45,314 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:10:52,549 INFO] Validation perplexity: 2.1399\n",
      "[2019-01-07 22:10:52,550 INFO] Validation accuracy: 72.6615\n",
      "[2019-01-07 22:10:55,254 INFO] Step 7550/100000; acc:  69.45; ppl:  2.32; xent: 0.84; lr: 0.00100; 10601/9735 tok/s;    962 sec\n",
      "[2019-01-07 22:10:57,977 INFO] Step 7600/100000; acc:  68.77; ppl:  2.36; xent: 0.86; lr: 0.00100; 41193/37710 tok/s;    965 sec\n",
      "[2019-01-07 22:10:58,048 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:11:05,275 INFO] Validation perplexity: 2.17026\n",
      "[2019-01-07 22:11:05,277 INFO] Validation accuracy: 71.9669\n",
      "[2019-01-07 22:11:08,022 INFO] Step 7650/100000; acc:  69.34; ppl:  2.33; xent: 0.85; lr: 0.00100; 10898/9881 tok/s;    975 sec\n",
      "[2019-01-07 22:11:10,719 INFO] Step 7700/100000; acc:  69.10; ppl:  2.33; xent: 0.85; lr: 0.00100; 41092/37575 tok/s;    978 sec\n",
      "[2019-01-07 22:11:10,790 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:11:18,025 INFO] Validation perplexity: 2.15203\n",
      "[2019-01-07 22:11:18,027 INFO] Validation accuracy: 71.9985\n",
      "[2019-01-07 22:11:20,754 INFO] Step 7750/100000; acc:  69.07; ppl:  2.34; xent: 0.85; lr: 0.00100; 10955/10021 tok/s;    988 sec\n",
      "[2019-01-07 22:11:23,436 INFO] Step 7800/100000; acc:  69.31; ppl:  2.32; xent: 0.84; lr: 0.00100; 40039/36453 tok/s;    990 sec\n",
      "[2019-01-07 22:11:23,506 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:11:30,733 INFO] Validation perplexity: 2.1448\n",
      "[2019-01-07 22:11:30,734 INFO] Validation accuracy: 72.1785\n",
      "[2019-01-07 22:11:33,488 INFO] Step 7850/100000; acc:  68.82; ppl:  2.36; xent: 0.86; lr: 0.00100; 11076/10215 tok/s;   1000 sec\n",
      "[2019-01-07 22:11:36,191 INFO] Step 7900/100000; acc:  68.58; ppl:  2.38; xent: 0.87; lr: 0.00100; 41584/38162 tok/s;   1003 sec\n",
      "[2019-01-07 22:11:36,262 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:11:43,495 INFO] Validation perplexity: 2.16875\n",
      "[2019-01-07 22:11:43,497 INFO] Validation accuracy: 72.0099\n",
      "[2019-01-07 22:11:46,216 INFO] Step 7950/100000; acc:  69.09; ppl:  2.34; xent: 0.85; lr: 0.00100; 11200/10208 tok/s;   1013 sec\n",
      "[2019-01-07 22:11:48,917 INFO] Step 8000/100000; acc:  69.61; ppl:  2.31; xent: 0.84; lr: 0.00100; 39025/35906 tok/s;   1016 sec\n",
      "[2019-01-07 22:11:48,987 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:11:56,216 INFO] Validation perplexity: 2.19998\n",
      "[2019-01-07 22:11:56,218 INFO] Validation accuracy: 71.3924\n",
      "[2019-01-07 22:11:56,219 INFO] Saving checkpoint CNN_step_8000.pt\n",
      "[2019-01-07 22:11:59,072 INFO] Step 8050/100000; acc:  68.96; ppl:  2.36; xent: 0.86; lr: 0.00100; 10872/9857 tok/s;   1026 sec\n",
      "[2019-01-07 22:12:01,798 INFO] Step 8100/100000; acc:  69.13; ppl:  2.34; xent: 0.85; lr: 0.00100; 40381/36495 tok/s;   1029 sec\n",
      "[2019-01-07 22:12:01,868 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:12:09,100 INFO] Validation perplexity: 2.11087\n",
      "[2019-01-07 22:12:09,101 INFO] Validation accuracy: 72.8577\n",
      "[2019-01-07 22:12:11,807 INFO] Step 8150/100000; acc:  69.47; ppl:  2.31; xent: 0.84; lr: 0.00100; 10918/10032 tok/s;   1039 sec\n",
      "[2019-01-07 22:12:14,519 INFO] Step 8200/100000; acc:  68.54; ppl:  2.37; xent: 0.86; lr: 0.00100; 42094/38348 tok/s;   1041 sec\n",
      "[2019-01-07 22:12:14,589 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:12:21,816 INFO] Validation perplexity: 2.13689\n",
      "[2019-01-07 22:12:21,817 INFO] Validation accuracy: 72.2706\n",
      "[2019-01-07 22:12:24,552 INFO] Step 8250/100000; acc:  69.39; ppl:  2.32; xent: 0.84; lr: 0.00100; 10661/9870 tok/s;   1051 sec\n",
      "[2019-01-07 22:12:27,224 INFO] Step 8300/100000; acc:  69.90; ppl:  2.29; xent: 0.83; lr: 0.00100; 40440/36008 tok/s;   1054 sec\n",
      "[2019-01-07 22:12:27,295 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:12:34,527 INFO] Validation perplexity: 2.18861\n",
      "[2019-01-07 22:12:34,529 INFO] Validation accuracy: 70.7779\n",
      "[2019-01-07 22:12:37,269 INFO] Step 8350/100000; acc:  68.88; ppl:  2.36; xent: 0.86; lr: 0.00100; 11217/10465 tok/s;   1064 sec\n",
      "[2019-01-07 22:12:39,988 INFO] Step 8400/100000; acc:  69.37; ppl:  2.32; xent: 0.84; lr: 0.00100; 42198/38390 tok/s;   1067 sec\n",
      "[2019-01-07 22:12:40,059 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:12:47,283 INFO] Validation perplexity: 2.20407\n",
      "[2019-01-07 22:12:47,285 INFO] Validation accuracy: 71.119\n",
      "[2019-01-07 22:12:50,029 INFO] Step 8450/100000; acc:  68.98; ppl:  2.35; xent: 0.85; lr: 0.00100; 10930/9909 tok/s;   1077 sec\n",
      "[2019-01-07 22:12:52,752 INFO] Step 8500/100000; acc:  69.69; ppl:  2.30; xent: 0.83; lr: 0.00100; 40621/37795 tok/s;   1080 sec\n",
      "[2019-01-07 22:12:52,822 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:13:00,054 INFO] Validation perplexity: 2.14101\n",
      "[2019-01-07 22:13:00,055 INFO] Validation accuracy: 72.3961\n",
      "[2019-01-07 22:13:02,763 INFO] Step 8550/100000; acc:  69.66; ppl:  2.30; xent: 0.83; lr: 0.00100; 10518/9445 tok/s;   1090 sec\n",
      "[2019-01-07 22:13:05,476 INFO] Step 8600/100000; acc:  69.58; ppl:  2.32; xent: 0.84; lr: 0.00100; 41225/37629 tok/s;   1092 sec\n",
      "[2019-01-07 22:13:05,546 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:13:12,772 INFO] Validation perplexity: 2.11614\n",
      "[2019-01-07 22:13:12,774 INFO] Validation accuracy: 72.7964\n",
      "[2019-01-07 22:13:15,499 INFO] Step 8650/100000; acc:  68.87; ppl:  2.35; xent: 0.85; lr: 0.00100; 10858/9951 tok/s;   1102 sec\n",
      "[2019-01-07 22:13:18,212 INFO] Step 8700/100000; acc:  69.52; ppl:  2.30; xent: 0.83; lr: 0.00100; 40803/37400 tok/s;   1105 sec\n",
      "[2019-01-07 22:13:18,283 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:13:25,519 INFO] Validation perplexity: 2.11744\n",
      "[2019-01-07 22:13:25,520 INFO] Validation accuracy: 72.6184\n",
      "[2019-01-07 22:13:28,265 INFO] Step 8750/100000; acc:  69.69; ppl:  2.31; xent: 0.84; lr: 0.00100; 10913/10127 tok/s;   1115 sec\n",
      "[2019-01-07 22:13:30,985 INFO] Step 8800/100000; acc:  69.42; ppl:  2.32; xent: 0.84; lr: 0.00100; 40397/37031 tok/s;   1118 sec\n",
      "[2019-01-07 22:13:31,055 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:13:38,286 INFO] Validation perplexity: 2.22336\n",
      "[2019-01-07 22:13:38,288 INFO] Validation accuracy: 71.0654\n",
      "[2019-01-07 22:13:40,999 INFO] Step 8850/100000; acc:  69.21; ppl:  2.33; xent: 0.84; lr: 0.00100; 10498/9473 tok/s;   1128 sec\n",
      "[2019-01-07 22:13:43,751 INFO] Step 8900/100000; acc:  69.17; ppl:  2.34; xent: 0.85; lr: 0.00100; 41059/37912 tok/s;   1131 sec\n",
      "[2019-01-07 22:13:43,821 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:13:51,086 INFO] Validation perplexity: 2.12414\n",
      "[2019-01-07 22:13:51,088 INFO] Validation accuracy: 72.7289\n",
      "[2019-01-07 22:13:52,854 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:13:55,545 INFO] Step 8950/100000; acc:  70.04; ppl:  2.28; xent: 0.82; lr: 0.00100; 9050/8273 tok/s;   1142 sec\n",
      "[2019-01-07 22:13:58,290 INFO] Step 9000/100000; acc:  69.34; ppl:  2.32; xent: 0.84; lr: 0.00100; 41567/37714 tok/s;   1145 sec\n",
      "[2019-01-07 22:13:58,360 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:14:05,575 INFO] Validation perplexity: 2.13425\n",
      "[2019-01-07 22:14:05,577 INFO] Validation accuracy: 71.9978\n",
      "[2019-01-07 22:14:05,578 INFO] Saving checkpoint CNN_step_9000.pt\n",
      "[2019-01-07 22:14:08,427 INFO] Step 9050/100000; acc:  69.38; ppl:  2.32; xent: 0.84; lr: 0.00100; 10997/9900 tok/s;   1155 sec\n",
      "[2019-01-07 22:14:11,130 INFO] Step 9100/100000; acc:  70.08; ppl:  2.28; xent: 0.82; lr: 0.00100; 39973/36827 tok/s;   1158 sec\n",
      "[2019-01-07 22:14:11,200 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:14:18,427 INFO] Validation perplexity: 2.11457\n",
      "[2019-01-07 22:14:18,429 INFO] Validation accuracy: 72.4712\n",
      "[2019-01-07 22:14:21,166 INFO] Step 9150/100000; acc:  69.67; ppl:  2.30; xent: 0.83; lr: 0.00100; 11134/10313 tok/s;   1168 sec\n",
      "[2019-01-07 22:14:23,845 INFO] Step 9200/100000; acc:  69.77; ppl:  2.29; xent: 0.83; lr: 0.00100; 40687/37008 tok/s;   1171 sec\n",
      "[2019-01-07 22:14:23,914 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:14:31,143 INFO] Validation perplexity: 2.14115\n",
      "[2019-01-07 22:14:31,145 INFO] Validation accuracy: 71.838\n",
      "[2019-01-07 22:14:33,876 INFO] Step 9250/100000; acc:  69.41; ppl:  2.32; xent: 0.84; lr: 0.00100; 11044/9977 tok/s;   1181 sec\n",
      "[2019-01-07 22:14:36,571 INFO] Step 9300/100000; acc:  70.18; ppl:  2.27; xent: 0.82; lr: 0.00100; 40862/37297 tok/s;   1183 sec\n",
      "[2019-01-07 22:14:36,641 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:14:43,868 INFO] Validation perplexity: 2.13986\n",
      "[2019-01-07 22:14:43,870 INFO] Validation accuracy: 71.9612\n",
      "[2019-01-07 22:14:46,561 INFO] Step 9350/100000; acc:  70.22; ppl:  2.27; xent: 0.82; lr: 0.00100; 10302/9555 tok/s;   1193 sec\n",
      "[2019-01-07 22:14:49,283 INFO] Step 9400/100000; acc:  69.73; ppl:  2.30; xent: 0.83; lr: 0.00100; 42800/38984 tok/s;   1196 sec\n",
      "[2019-01-07 22:14:49,353 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:14:56,579 INFO] Validation perplexity: 2.10092\n",
      "[2019-01-07 22:14:56,581 INFO] Validation accuracy: 72.955\n",
      "[2019-01-07 22:14:59,288 INFO] Step 9450/100000; acc:  70.03; ppl:  2.29; xent: 0.83; lr: 0.00100; 10519/9591 tok/s;   1206 sec\n",
      "[2019-01-07 22:15:02,004 INFO] Step 9500/100000; acc:  69.64; ppl:  2.30; xent: 0.83; lr: 0.00100; 41398/37740 tok/s;   1209 sec\n",
      "[2019-01-07 22:15:02,074 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:15:09,297 INFO] Validation perplexity: 2.10306\n",
      "[2019-01-07 22:15:09,298 INFO] Validation accuracy: 72.9225\n",
      "[2019-01-07 22:15:12,014 INFO] Step 9550/100000; acc:  69.61; ppl:  2.30; xent: 0.83; lr: 0.00100; 11487/10438 tok/s;   1219 sec\n",
      "[2019-01-07 22:15:14,718 INFO] Step 9600/100000; acc:  69.71; ppl:  2.30; xent: 0.83; lr: 0.00100; 40385/37042 tok/s;   1222 sec\n",
      "[2019-01-07 22:15:14,788 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:15:22,019 INFO] Validation perplexity: 2.20274\n",
      "[2019-01-07 22:15:22,020 INFO] Validation accuracy: 70.9479\n",
      "[2019-01-07 22:15:24,749 INFO] Step 9650/100000; acc:  69.52; ppl:  2.30; xent: 0.83; lr: 0.00100; 11098/10227 tok/s;   1232 sec\n",
      "[2019-01-07 22:15:27,415 INFO] Step 9700/100000; acc:  70.22; ppl:  2.26; xent: 0.81; lr: 0.00100; 39956/36745 tok/s;   1234 sec\n",
      "[2019-01-07 22:15:27,486 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:15:34,721 INFO] Validation perplexity: 2.14006\n",
      "[2019-01-07 22:15:34,723 INFO] Validation accuracy: 72.0115\n",
      "[2019-01-07 22:15:37,443 INFO] Step 9750/100000; acc:  70.20; ppl:  2.27; xent: 0.82; lr: 0.00100; 10474/9663 tok/s;   1244 sec\n",
      "[2019-01-07 22:15:40,202 INFO] Step 9800/100000; acc:  69.21; ppl:  2.33; xent: 0.84; lr: 0.00100; 41994/37701 tok/s;   1247 sec\n",
      "[2019-01-07 22:15:40,275 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:15:47,509 INFO] Validation perplexity: 2.127\n",
      "[2019-01-07 22:15:47,511 INFO] Validation accuracy: 72.5156\n",
      "[2019-01-07 22:15:50,235 INFO] Step 9850/100000; acc:  69.80; ppl:  2.29; xent: 0.83; lr: 0.00100; 10640/9817 tok/s;   1257 sec\n",
      "[2019-01-07 22:15:52,940 INFO] Step 9900/100000; acc:  69.90; ppl:  2.29; xent: 0.83; lr: 0.00100; 42152/37971 tok/s;   1260 sec\n",
      "[2019-01-07 22:15:53,010 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:16:00,239 INFO] Validation perplexity: 2.14836\n",
      "[2019-01-07 22:16:00,240 INFO] Validation accuracy: 72.215\n",
      "[2019-01-07 22:16:02,939 INFO] Step 9950/100000; acc:  70.14; ppl:  2.28; xent: 0.82; lr: 0.00100; 10564/9782 tok/s;   1270 sec\n",
      "[2019-01-07 22:16:05,678 INFO] Step 10000/100000; acc:  69.73; ppl:  2.30; xent: 0.83; lr: 0.00090; 41483/37417 tok/s;   1273 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:16:05,749 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:16:12,979 INFO] Validation perplexity: 2.12657\n",
      "[2019-01-07 22:16:12,981 INFO] Validation accuracy: 72.5541\n",
      "[2019-01-07 22:16:12,983 INFO] Saving checkpoint CNN_step_10000.pt\n",
      "[2019-01-07 22:16:15,811 INFO] Step 10050/100000; acc:  70.10; ppl:  2.27; xent: 0.82; lr: 0.00090; 10676/9886 tok/s;   1283 sec\n",
      "[2019-01-07 22:16:18,536 INFO] Step 10100/100000; acc:  70.04; ppl:  2.27; xent: 0.82; lr: 0.00090; 42415/38672 tok/s;   1285 sec\n",
      "[2019-01-07 22:16:18,606 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:16:25,838 INFO] Validation perplexity: 2.12536\n",
      "[2019-01-07 22:16:25,839 INFO] Validation accuracy: 72.5653\n",
      "[2019-01-07 22:16:28,560 INFO] Step 10150/100000; acc:  69.79; ppl:  2.29; xent: 0.83; lr: 0.00090; 11077/10024 tok/s;   1295 sec\n",
      "[2019-01-07 22:16:31,227 INFO] Step 10200/100000; acc:  70.34; ppl:  2.26; xent: 0.81; lr: 0.00090; 39955/37092 tok/s;   1298 sec\n",
      "[2019-01-07 22:16:31,297 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:16:38,527 INFO] Validation perplexity: 2.09591\n",
      "[2019-01-07 22:16:38,529 INFO] Validation accuracy: 73.1324\n",
      "[2019-01-07 22:16:41,258 INFO] Step 10250/100000; acc:  70.39; ppl:  2.25; xent: 0.81; lr: 0.00090; 10921/9810 tok/s;   1308 sec\n",
      "[2019-01-07 22:16:43,954 INFO] Step 10300/100000; acc:  70.32; ppl:  2.25; xent: 0.81; lr: 0.00090; 40489/37393 tok/s;   1311 sec\n",
      "[2019-01-07 22:16:44,025 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:16:51,256 INFO] Validation perplexity: 2.11765\n",
      "[2019-01-07 22:16:51,258 INFO] Validation accuracy: 72.5004\n",
      "[2019-01-07 22:16:54,038 INFO] Step 10350/100000; acc:  70.21; ppl:  2.26; xent: 0.82; lr: 0.00090; 11442/10491 tok/s;   1321 sec\n",
      "[2019-01-07 22:16:56,711 INFO] Step 10400/100000; acc:  70.72; ppl:  2.25; xent: 0.81; lr: 0.00090; 39530/35682 tok/s;   1324 sec\n",
      "[2019-01-07 22:16:56,782 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:17:04,014 INFO] Validation perplexity: 2.17772\n",
      "[2019-01-07 22:17:04,016 INFO] Validation accuracy: 71.0349\n",
      "[2019-01-07 22:17:06,717 INFO] Step 10450/100000; acc:  70.09; ppl:  2.27; xent: 0.82; lr: 0.00090; 11399/10473 tok/s;   1334 sec\n",
      "[2019-01-07 22:17:09,432 INFO] Step 10500/100000; acc:  70.29; ppl:  2.24; xent: 0.81; lr: 0.00090; 39969/36657 tok/s;   1336 sec\n",
      "[2019-01-07 22:17:09,502 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:17:16,741 INFO] Validation perplexity: 2.06892\n",
      "[2019-01-07 22:17:16,743 INFO] Validation accuracy: 73.0716\n",
      "[2019-01-07 22:17:19,460 INFO] Step 10550/100000; acc:  70.26; ppl:  2.26; xent: 0.81; lr: 0.00090; 10645/9870 tok/s;   1346 sec\n",
      "[2019-01-07 22:17:22,182 INFO] Step 10600/100000; acc:  70.18; ppl:  2.26; xent: 0.82; lr: 0.00090; 40403/36751 tok/s;   1349 sec\n",
      "[2019-01-07 22:17:22,253 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:17:29,485 INFO] Validation perplexity: 2.05995\n",
      "[2019-01-07 22:17:29,486 INFO] Validation accuracy: 73.8438\n",
      "[2019-01-07 22:17:32,225 INFO] Step 10650/100000; acc:  70.05; ppl:  2.27; xent: 0.82; lr: 0.00090; 11032/9874 tok/s;   1359 sec\n",
      "[2019-01-07 22:17:34,941 INFO] Step 10700/100000; acc:  70.66; ppl:  2.23; xent: 0.80; lr: 0.00090; 41202/37859 tok/s;   1362 sec\n",
      "[2019-01-07 22:17:35,012 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:17:42,244 INFO] Validation perplexity: 2.09966\n",
      "[2019-01-07 22:17:42,246 INFO] Validation accuracy: 72.8129\n",
      "[2019-01-07 22:17:44,953 INFO] Step 10750/100000; acc:  70.37; ppl:  2.24; xent: 0.80; lr: 0.00090; 10791/9774 tok/s;   1372 sec\n",
      "[2019-01-07 22:17:47,646 INFO] Step 10800/100000; acc:  70.77; ppl:  2.23; xent: 0.80; lr: 0.00090; 40131/37186 tok/s;   1375 sec\n",
      "[2019-01-07 22:17:47,717 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:17:54,948 INFO] Validation perplexity: 2.08216\n",
      "[2019-01-07 22:17:54,950 INFO] Validation accuracy: 73.217\n",
      "[2019-01-07 22:17:57,673 INFO] Step 10850/100000; acc:  69.75; ppl:  2.30; xent: 0.83; lr: 0.00090; 11198/10379 tok/s;   1385 sec\n",
      "[2019-01-07 22:18:00,405 INFO] Step 10900/100000; acc:  70.46; ppl:  2.25; xent: 0.81; lr: 0.00090; 41454/37596 tok/s;   1387 sec\n",
      "[2019-01-07 22:18:00,475 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:18:07,712 INFO] Validation perplexity: 2.167\n",
      "[2019-01-07 22:18:07,713 INFO] Validation accuracy: 71.6277\n",
      "[2019-01-07 22:18:10,448 INFO] Step 10950/100000; acc:  70.94; ppl:  2.21; xent: 0.80; lr: 0.00090; 10638/9738 tok/s;   1397 sec\n",
      "[2019-01-07 22:18:13,177 INFO] Step 11000/100000; acc:  70.06; ppl:  2.27; xent: 0.82; lr: 0.00081; 40832/37174 tok/s;   1400 sec\n",
      "[2019-01-07 22:18:13,250 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:18:20,477 INFO] Validation perplexity: 2.13786\n",
      "[2019-01-07 22:18:20,479 INFO] Validation accuracy: 72.0796\n",
      "[2019-01-07 22:18:20,481 INFO] Saving checkpoint CNN_step_11000.pt\n",
      "[2019-01-07 22:18:23,300 INFO] Step 11050/100000; acc:  70.44; ppl:  2.25; xent: 0.81; lr: 0.00081; 10922/9901 tok/s;   1410 sec\n",
      "[2019-01-07 22:18:26,027 INFO] Step 11100/100000; acc:  70.43; ppl:  2.24; xent: 0.80; lr: 0.00081; 39813/36399 tok/s;   1413 sec\n",
      "[2019-01-07 22:18:26,097 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:18:33,328 INFO] Validation perplexity: 2.07603\n",
      "[2019-01-07 22:18:33,329 INFO] Validation accuracy: 73.4122\n",
      "[2019-01-07 22:18:36,066 INFO] Step 11150/100000; acc:  70.27; ppl:  2.26; xent: 0.81; lr: 0.00081; 11106/10245 tok/s;   1423 sec\n",
      "[2019-01-07 22:18:38,769 INFO] Step 11200/100000; acc:  70.32; ppl:  2.25; xent: 0.81; lr: 0.00081; 40817/37099 tok/s;   1426 sec\n",
      "[2019-01-07 22:18:38,839 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:18:46,067 INFO] Validation perplexity: 2.04635\n",
      "[2019-01-07 22:18:46,068 INFO] Validation accuracy: 73.9592\n",
      "[2019-01-07 22:18:48,754 INFO] Step 11250/100000; acc:  71.26; ppl:  2.19; xent: 0.79; lr: 0.00081; 10424/9386 tok/s;   1436 sec\n",
      "[2019-01-07 22:18:51,489 INFO] Step 11300/100000; acc:  70.66; ppl:  2.23; xent: 0.80; lr: 0.00081; 42064/39185 tok/s;   1438 sec\n",
      "[2019-01-07 22:18:51,560 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:18:58,788 INFO] Validation perplexity: 2.1755\n",
      "[2019-01-07 22:18:58,790 INFO] Validation accuracy: 71.7976\n",
      "[2019-01-07 22:19:01,488 INFO] Step 11350/100000; acc:  70.50; ppl:  2.24; xent: 0.80; lr: 0.00081; 10933/9876 tok/s;   1448 sec\n",
      "[2019-01-07 22:19:04,218 INFO] Step 11400/100000; acc:  70.13; ppl:  2.27; xent: 0.82; lr: 0.00081; 40942/37488 tok/s;   1451 sec\n",
      "[2019-01-07 22:19:04,288 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:19:11,520 INFO] Validation perplexity: 2.22592\n",
      "[2019-01-07 22:19:11,522 INFO] Validation accuracy: 70.4734\n",
      "[2019-01-07 22:19:14,277 INFO] Step 11450/100000; acc:  70.57; ppl:  2.24; xent: 0.81; lr: 0.00081; 11462/10534 tok/s;   1461 sec\n",
      "[2019-01-07 22:19:16,954 INFO] Step 11500/100000; acc:  71.02; ppl:  2.20; xent: 0.79; lr: 0.00081; 39529/35574 tok/s;   1464 sec\n",
      "[2019-01-07 22:19:17,025 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:19:24,255 INFO] Validation perplexity: 2.08554\n",
      "[2019-01-07 22:19:24,256 INFO] Validation accuracy: 73.22\n",
      "[2019-01-07 22:19:26,967 INFO] Step 11550/100000; acc:  70.86; ppl:  2.22; xent: 0.80; lr: 0.00081; 10779/9899 tok/s;   1474 sec\n",
      "[2019-01-07 22:19:29,673 INFO] Step 11600/100000; acc:  70.49; ppl:  2.24; xent: 0.81; lr: 0.00081; 41084/37491 tok/s;   1477 sec\n",
      "[2019-01-07 22:19:29,745 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:19:36,980 INFO] Validation perplexity: 2.16375\n",
      "[2019-01-07 22:19:36,981 INFO] Validation accuracy: 70.9359\n",
      "[2019-01-07 22:19:39,722 INFO] Step 11650/100000; acc:  70.50; ppl:  2.23; xent: 0.80; lr: 0.00081; 11204/10118 tok/s;   1487 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:19:42,418 INFO] Step 11700/100000; acc:  70.83; ppl:  2.22; xent: 0.80; lr: 0.00081; 39638/37004 tok/s;   1489 sec\n",
      "[2019-01-07 22:19:42,488 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:19:49,721 INFO] Validation perplexity: 2.09454\n",
      "[2019-01-07 22:19:49,722 INFO] Validation accuracy: 73.2616\n",
      "[2019-01-07 22:19:52,440 INFO] Step 11750/100000; acc:  70.93; ppl:  2.22; xent: 0.80; lr: 0.00081; 10962/10052 tok/s;   1499 sec\n",
      "[2019-01-07 22:19:55,149 INFO] Step 11800/100000; acc:  70.48; ppl:  2.25; xent: 0.81; lr: 0.00081; 39879/36603 tok/s;   1502 sec\n",
      "[2019-01-07 22:19:55,220 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:20:02,450 INFO] Validation perplexity: 2.10202\n",
      "[2019-01-07 22:20:02,451 INFO] Validation accuracy: 73.0032\n",
      "[2019-01-07 22:20:05,213 INFO] Step 11850/100000; acc:  70.08; ppl:  2.27; xent: 0.82; lr: 0.00081; 11194/10135 tok/s;   1512 sec\n",
      "[2019-01-07 22:20:07,944 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:20:09,658 INFO] Step 11900/100000; acc:  71.54; ppl:  2.19; xent: 0.78; lr: 0.00081; 23622/21920 tok/s;   1517 sec\n",
      "[2019-01-07 22:20:09,729 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:20:16,949 INFO] Validation perplexity: 2.11871\n",
      "[2019-01-07 22:20:16,951 INFO] Validation accuracy: 72.1296\n",
      "[2019-01-07 22:20:19,663 INFO] Step 11950/100000; acc:  70.62; ppl:  2.22; xent: 0.80; lr: 0.00081; 11121/9983 tok/s;   1527 sec\n",
      "[2019-01-07 22:20:22,434 INFO] Step 12000/100000; acc:  70.41; ppl:  2.25; xent: 0.81; lr: 0.00073; 41003/37288 tok/s;   1529 sec\n",
      "[2019-01-07 22:20:22,505 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:20:29,737 INFO] Validation perplexity: 2.23825\n",
      "[2019-01-07 22:20:29,739 INFO] Validation accuracy: 70.3277\n",
      "[2019-01-07 22:20:29,740 INFO] Saving checkpoint CNN_step_12000.pt\n",
      "[2019-01-07 22:20:32,530 INFO] Step 12050/100000; acc:  71.14; ppl:  2.19; xent: 0.78; lr: 0.00073; 10590/9684 tok/s;   1539 sec\n",
      "[2019-01-07 22:20:35,250 INFO] Step 12100/100000; acc:  71.13; ppl:  2.20; xent: 0.79; lr: 0.00073; 40283/37282 tok/s;   1542 sec\n",
      "[2019-01-07 22:20:35,321 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:20:42,559 INFO] Validation perplexity: 2.0757\n",
      "[2019-01-07 22:20:42,561 INFO] Validation accuracy: 73.1983\n",
      "[2019-01-07 22:20:45,299 INFO] Step 12150/100000; acc:  70.35; ppl:  2.24; xent: 0.81; lr: 0.00073; 11179/10218 tok/s;   1552 sec\n",
      "[2019-01-07 22:20:48,017 INFO] Step 12200/100000; acc:  71.43; ppl:  2.20; xent: 0.79; lr: 0.00073; 40824/37555 tok/s;   1555 sec\n",
      "[2019-01-07 22:20:48,088 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:20:55,311 INFO] Validation perplexity: 2.17441\n",
      "[2019-01-07 22:20:55,313 INFO] Validation accuracy: 71.4616\n",
      "[2019-01-07 22:20:58,014 INFO] Step 12250/100000; acc:  71.56; ppl:  2.18; xent: 0.78; lr: 0.00073; 10812/9935 tok/s;   1565 sec\n",
      "[2019-01-07 22:21:00,752 INFO] Step 12300/100000; acc:  70.98; ppl:  2.21; xent: 0.79; lr: 0.00073; 40209/36289 tok/s;   1568 sec\n",
      "[2019-01-07 22:21:00,825 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:21:08,062 INFO] Validation perplexity: 2.13363\n",
      "[2019-01-07 22:21:08,064 INFO] Validation accuracy: 72.2927\n",
      "[2019-01-07 22:21:10,771 INFO] Step 12350/100000; acc:  71.37; ppl:  2.18; xent: 0.78; lr: 0.00073; 10940/10091 tok/s;   1578 sec\n",
      "[2019-01-07 22:21:13,495 INFO] Step 12400/100000; acc:  70.83; ppl:  2.22; xent: 0.80; lr: 0.00073; 40150/36243 tok/s;   1580 sec\n",
      "[2019-01-07 22:21:13,565 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:21:20,802 INFO] Validation perplexity: 2.10241\n",
      "[2019-01-07 22:21:20,803 INFO] Validation accuracy: 72.3998\n",
      "[2019-01-07 22:21:23,543 INFO] Step 12450/100000; acc:  71.32; ppl:  2.20; xent: 0.79; lr: 0.00073; 10914/10018 tok/s;   1590 sec\n",
      "[2019-01-07 22:21:26,254 INFO] Step 12500/100000; acc:  70.99; ppl:  2.21; xent: 0.79; lr: 0.00073; 40495/36462 tok/s;   1593 sec\n",
      "[2019-01-07 22:21:26,327 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:21:33,557 INFO] Validation perplexity: 2.13067\n",
      "[2019-01-07 22:21:33,558 INFO] Validation accuracy: 72.5115\n",
      "[2019-01-07 22:21:36,291 INFO] Step 12550/100000; acc:  71.38; ppl:  2.19; xent: 0.78; lr: 0.00073; 11228/10376 tok/s;   1603 sec\n",
      "[2019-01-07 22:21:39,053 INFO] Step 12600/100000; acc:  70.77; ppl:  2.22; xent: 0.80; lr: 0.00073; 41163/37604 tok/s;   1606 sec\n",
      "[2019-01-07 22:21:39,123 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:21:46,362 INFO] Validation perplexity: 2.04192\n",
      "[2019-01-07 22:21:46,364 INFO] Validation accuracy: 73.9396\n",
      "[2019-01-07 22:21:49,088 INFO] Step 12650/100000; acc:  71.25; ppl:  2.19; xent: 0.79; lr: 0.00073; 10498/9587 tok/s;   1616 sec\n",
      "[2019-01-07 22:21:51,809 INFO] Step 12700/100000; acc:  71.49; ppl:  2.18; xent: 0.78; lr: 0.00073; 39247/36927 tok/s;   1619 sec\n",
      "[2019-01-07 22:21:51,880 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:21:59,110 INFO] Validation perplexity: 2.08248\n",
      "[2019-01-07 22:21:59,111 INFO] Validation accuracy: 73.2366\n",
      "[2019-01-07 22:22:01,839 INFO] Step 12750/100000; acc:  71.03; ppl:  2.21; xent: 0.79; lr: 0.00073; 11341/10233 tok/s;   1629 sec\n",
      "[2019-01-07 22:22:04,564 INFO] Step 12800/100000; acc:  70.79; ppl:  2.22; xent: 0.80; lr: 0.00073; 39263/35690 tok/s;   1631 sec\n",
      "[2019-01-07 22:22:04,635 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:22:11,866 INFO] Validation perplexity: 2.19013\n",
      "[2019-01-07 22:22:11,867 INFO] Validation accuracy: 71.5265\n",
      "[2019-01-07 22:22:14,593 INFO] Step 12850/100000; acc:  71.08; ppl:  2.20; xent: 0.79; lr: 0.00073; 11389/10286 tok/s;   1641 sec\n",
      "[2019-01-07 22:22:17,326 INFO] Step 12900/100000; acc:  71.00; ppl:  2.21; xent: 0.79; lr: 0.00073; 40362/37259 tok/s;   1644 sec\n",
      "[2019-01-07 22:22:17,399 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:22:24,634 INFO] Validation perplexity: 2.1215\n",
      "[2019-01-07 22:22:24,635 INFO] Validation accuracy: 72.2824\n",
      "[2019-01-07 22:22:27,365 INFO] Step 12950/100000; acc:  71.34; ppl:  2.20; xent: 0.79; lr: 0.00073; 10902/9896 tok/s;   1654 sec\n",
      "[2019-01-07 22:22:30,098 INFO] Step 13000/100000; acc:  71.45; ppl:  2.17; xent: 0.77; lr: 0.00066; 39676/36357 tok/s;   1657 sec\n",
      "[2019-01-07 22:22:30,171 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:22:37,401 INFO] Validation perplexity: 2.21993\n",
      "[2019-01-07 22:22:37,402 INFO] Validation accuracy: 71.0403\n",
      "[2019-01-07 22:22:37,403 INFO] Saving checkpoint CNN_step_13000.pt\n",
      "[2019-01-07 22:22:40,221 INFO] Step 13050/100000; acc:  70.94; ppl:  2.21; xent: 0.80; lr: 0.00066; 11029/10057 tok/s;   1667 sec\n",
      "[2019-01-07 22:22:42,936 INFO] Step 13100/100000; acc:  71.02; ppl:  2.21; xent: 0.79; lr: 0.00066; 42419/38587 tok/s;   1670 sec\n",
      "[2019-01-07 22:22:43,007 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:22:50,236 INFO] Validation perplexity: 2.08003\n",
      "[2019-01-07 22:22:50,238 INFO] Validation accuracy: 72.9888\n",
      "[2019-01-07 22:22:52,957 INFO] Step 13150/100000; acc:  71.54; ppl:  2.18; xent: 0.78; lr: 0.00066; 10665/9857 tok/s;   1680 sec\n",
      "[2019-01-07 22:22:55,635 INFO] Step 13200/100000; acc:  71.24; ppl:  2.19; xent: 0.78; lr: 0.00066; 40961/36961 tok/s;   1683 sec\n",
      "[2019-01-07 22:22:55,706 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:23:02,940 INFO] Validation perplexity: 2.11698\n",
      "[2019-01-07 22:23:02,941 INFO] Validation accuracy: 72.5272\n",
      "[2019-01-07 22:23:05,637 INFO] Step 13250/100000; acc:  71.43; ppl:  2.17; xent: 0.77; lr: 0.00066; 10542/9806 tok/s;   1693 sec\n",
      "[2019-01-07 22:23:08,396 INFO] Step 13300/100000; acc:  71.16; ppl:  2.20; xent: 0.79; lr: 0.00066; 42487/38318 tok/s;   1695 sec\n",
      "[2019-01-07 22:23:08,466 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:23:15,690 INFO] Validation perplexity: 2.19505\n",
      "[2019-01-07 22:23:15,692 INFO] Validation accuracy: 71.3631\n",
      "[2019-01-07 22:23:18,412 INFO] Step 13350/100000; acc:  71.61; ppl:  2.17; xent: 0.77; lr: 0.00066; 11294/10294 tok/s;   1705 sec\n",
      "[2019-01-07 22:23:21,100 INFO] Step 13400/100000; acc:  71.50; ppl:  2.18; xent: 0.78; lr: 0.00066; 39376/36197 tok/s;   1708 sec\n",
      "[2019-01-07 22:23:21,170 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:23:28,396 INFO] Validation perplexity: 2.17601\n",
      "[2019-01-07 22:23:28,398 INFO] Validation accuracy: 71.9175\n",
      "[2019-01-07 22:23:31,104 INFO] Step 13450/100000; acc:  71.41; ppl:  2.17; xent: 0.78; lr: 0.00066; 11031/10054 tok/s;   1718 sec\n",
      "[2019-01-07 22:23:33,815 INFO] Step 13500/100000; acc:  71.54; ppl:  2.17; xent: 0.77; lr: 0.00066; 39250/36523 tok/s;   1721 sec\n",
      "[2019-01-07 22:23:33,885 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:23:41,115 INFO] Validation perplexity: 2.09965\n",
      "[2019-01-07 22:23:41,117 INFO] Validation accuracy: 72.9691\n",
      "[2019-01-07 22:23:43,845 INFO] Step 13550/100000; acc:  71.16; ppl:  2.20; xent: 0.79; lr: 0.00066; 11396/10386 tok/s;   1731 sec\n",
      "[2019-01-07 22:23:46,569 INFO] Step 13600/100000; acc:  71.43; ppl:  2.17; xent: 0.78; lr: 0.00066; 38381/34395 tok/s;   1733 sec\n",
      "[2019-01-07 22:23:46,640 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:23:53,879 INFO] Validation perplexity: 2.06434\n",
      "[2019-01-07 22:23:53,881 INFO] Validation accuracy: 73.6277\n",
      "[2019-01-07 22:23:56,598 INFO] Step 13650/100000; acc:  71.48; ppl:  2.16; xent: 0.77; lr: 0.00066; 11058/10216 tok/s;   1743 sec\n",
      "[2019-01-07 22:23:59,329 INFO] Step 13700/100000; acc:  71.34; ppl:  2.18; xent: 0.78; lr: 0.00066; 40858/37018 tok/s;   1746 sec\n",
      "[2019-01-07 22:23:59,399 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:24:06,631 INFO] Validation perplexity: 2.12826\n",
      "[2019-01-07 22:24:06,633 INFO] Validation accuracy: 72.6182\n",
      "[2019-01-07 22:24:09,340 INFO] Step 13750/100000; acc:  71.68; ppl:  2.18; xent: 0.78; lr: 0.00066; 10957/9980 tok/s;   1756 sec\n",
      "[2019-01-07 22:24:12,051 INFO] Step 13800/100000; acc:  71.03; ppl:  2.20; xent: 0.79; lr: 0.00066; 40917/37795 tok/s;   1759 sec\n",
      "[2019-01-07 22:24:12,122 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:24:19,357 INFO] Validation perplexity: 2.14606\n",
      "[2019-01-07 22:24:19,358 INFO] Validation accuracy: 72.1598\n",
      "[2019-01-07 22:24:22,088 INFO] Step 13850/100000; acc:  71.42; ppl:  2.18; xent: 0.78; lr: 0.00066; 11257/10311 tok/s;   1769 sec\n",
      "[2019-01-07 22:24:24,808 INFO] Step 13900/100000; acc:  71.93; ppl:  2.15; xent: 0.76; lr: 0.00066; 40287/36902 tok/s;   1772 sec\n",
      "[2019-01-07 22:24:24,878 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:24:32,112 INFO] Validation perplexity: 2.07895\n",
      "[2019-01-07 22:24:32,113 INFO] Validation accuracy: 73.1128\n",
      "[2019-01-07 22:24:34,865 INFO] Step 13950/100000; acc:  71.31; ppl:  2.19; xent: 0.78; lr: 0.00066; 10977/10043 tok/s;   1782 sec\n",
      "[2019-01-07 22:24:37,547 INFO] Step 14000/100000; acc:  71.87; ppl:  2.16; xent: 0.77; lr: 0.00059; 38803/35307 tok/s;   1784 sec\n",
      "[2019-01-07 22:24:37,617 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:24:44,861 INFO] Validation perplexity: 2.10882\n",
      "[2019-01-07 22:24:44,862 INFO] Validation accuracy: 72.0866\n",
      "[2019-01-07 22:24:44,864 INFO] Saving checkpoint CNN_step_14000.pt\n",
      "[2019-01-07 22:24:47,736 INFO] Step 14050/100000; acc:  70.75; ppl:  2.22; xent: 0.80; lr: 0.00059; 11366/10262 tok/s;   1795 sec\n",
      "[2019-01-07 22:24:50,438 INFO] Step 14100/100000; acc:  71.97; ppl:  2.14; xent: 0.76; lr: 0.00059; 40082/37160 tok/s;   1797 sec\n",
      "[2019-01-07 22:24:50,509 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:24:57,742 INFO] Validation perplexity: 2.13263\n",
      "[2019-01-07 22:24:57,743 INFO] Validation accuracy: 72.5399\n",
      "[2019-01-07 22:25:00,454 INFO] Step 14150/100000; acc:  71.72; ppl:  2.16; xent: 0.77; lr: 0.00059; 10897/9929 tok/s;   1807 sec\n",
      "[2019-01-07 22:25:03,179 INFO] Step 14200/100000; acc:  71.66; ppl:  2.17; xent: 0.77; lr: 0.00059; 39505/35951 tok/s;   1810 sec\n",
      "[2019-01-07 22:25:03,250 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:25:10,488 INFO] Validation perplexity: 2.07824\n",
      "[2019-01-07 22:25:10,490 INFO] Validation accuracy: 73.4076\n",
      "[2019-01-07 22:25:13,242 INFO] Step 14250/100000; acc:  72.08; ppl:  2.14; xent: 0.76; lr: 0.00059; 11125/10114 tok/s;   1820 sec\n",
      "[2019-01-07 22:25:15,959 INFO] Step 14300/100000; acc:  71.62; ppl:  2.16; xent: 0.77; lr: 0.00059; 39948/37135 tok/s;   1823 sec\n",
      "[2019-01-07 22:25:16,030 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:25:23,271 INFO] Validation perplexity: 2.09554\n",
      "[2019-01-07 22:25:23,272 INFO] Validation accuracy: 73.3149\n",
      "[2019-01-07 22:25:26,005 INFO] Step 14350/100000; acc:  71.49; ppl:  2.18; xent: 0.78; lr: 0.00059; 11492/10359 tok/s;   1833 sec\n",
      "[2019-01-07 22:25:28,740 INFO] Step 14400/100000; acc:  71.40; ppl:  2.18; xent: 0.78; lr: 0.00059; 40373/36901 tok/s;   1836 sec\n",
      "[2019-01-07 22:25:28,810 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:25:36,047 INFO] Validation perplexity: 2.17919\n",
      "[2019-01-07 22:25:36,048 INFO] Validation accuracy: 71.8405\n",
      "[2019-01-07 22:25:38,758 INFO] Step 14450/100000; acc:  71.99; ppl:  2.14; xent: 0.76; lr: 0.00059; 10980/10102 tok/s;   1846 sec\n",
      "[2019-01-07 22:25:41,457 INFO] Step 14500/100000; acc:  71.58; ppl:  2.16; xent: 0.77; lr: 0.00059; 39974/36098 tok/s;   1848 sec\n",
      "[2019-01-07 22:25:41,528 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:25:48,769 INFO] Validation perplexity: 2.02769\n",
      "[2019-01-07 22:25:48,770 INFO] Validation accuracy: 74.3159\n",
      "[2019-01-07 22:25:51,495 INFO] Step 14550/100000; acc:  71.89; ppl:  2.16; xent: 0.77; lr: 0.00059; 10900/10029 tok/s;   1858 sec\n",
      "[2019-01-07 22:25:54,232 INFO] Step 14600/100000; acc:  71.34; ppl:  2.18; xent: 0.78; lr: 0.00059; 40647/36993 tok/s;   1861 sec\n",
      "[2019-01-07 22:25:54,303 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:26:01,540 INFO] Validation perplexity: 2.12141\n",
      "[2019-01-07 22:26:01,541 INFO] Validation accuracy: 72.8404\n",
      "[2019-01-07 22:26:04,240 INFO] Step 14650/100000; acc:  71.66; ppl:  2.15; xent: 0.77; lr: 0.00059; 10676/9850 tok/s;   1871 sec\n",
      "[2019-01-07 22:26:06,992 INFO] Step 14700/100000; acc:  71.94; ppl:  2.16; xent: 0.77; lr: 0.00059; 40918/37294 tok/s;   1874 sec\n",
      "[2019-01-07 22:26:07,064 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:26:14,301 INFO] Validation perplexity: 2.12247\n",
      "[2019-01-07 22:26:14,303 INFO] Validation accuracy: 72.5121\n",
      "[2019-01-07 22:26:17,010 INFO] Step 14750/100000; acc:  71.98; ppl:  2.15; xent: 0.76; lr: 0.00059; 10717/9993 tok/s;   1884 sec\n",
      "[2019-01-07 22:26:19,766 INFO] Step 14800/100000; acc:  71.37; ppl:  2.19; xent: 0.78; lr: 0.00059; 39464/35464 tok/s;   1887 sec\n",
      "[2019-01-07 22:26:19,837 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:26:27,108 INFO] Validation perplexity: 2.13324\n",
      "[2019-01-07 22:26:27,109 INFO] Validation accuracy: 72.1299\n",
      "[2019-01-07 22:26:30,851 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:26:31,587 INFO] Step 14850/100000; acc:  72.19; ppl:  2.13; xent: 0.76; lr: 0.00059; 9316/8647 tok/s;   1898 sec\n",
      "[2019-01-07 22:26:34,275 INFO] Step 14900/100000; acc:  71.68; ppl:  2.16; xent: 0.77; lr: 0.00059; 40330/36474 tok/s;   1901 sec\n",
      "[2019-01-07 22:26:34,346 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:26:41,585 INFO] Validation perplexity: 2.01838\n",
      "[2019-01-07 22:26:41,587 INFO] Validation accuracy: 74.4328\n",
      "[2019-01-07 22:26:44,323 INFO] Step 14950/100000; acc:  71.67; ppl:  2.16; xent: 0.77; lr: 0.00059; 11190/10181 tok/s;   1911 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:26:47,052 INFO] Step 15000/100000; acc:  71.87; ppl:  2.15; xent: 0.76; lr: 0.00053; 40185/36761 tok/s;   1914 sec\n",
      "[2019-01-07 22:26:47,121 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:26:54,355 INFO] Validation perplexity: 2.16129\n",
      "[2019-01-07 22:26:54,357 INFO] Validation accuracy: 71.6689\n",
      "[2019-01-07 22:26:54,358 INFO] Saving checkpoint CNN_step_15000.pt\n",
      "[2019-01-07 22:26:57,213 INFO] Step 15050/100000; acc:  72.03; ppl:  2.14; xent: 0.76; lr: 0.00053; 10928/9958 tok/s;   1924 sec\n",
      "[2019-01-07 22:26:59,959 INFO] Step 15100/100000; acc:  71.33; ppl:  2.18; xent: 0.78; lr: 0.00053; 41034/37676 tok/s;   1927 sec\n",
      "[2019-01-07 22:27:00,031 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:27:07,264 INFO] Validation perplexity: 2.23706\n",
      "[2019-01-07 22:27:07,265 INFO] Validation accuracy: 71.0098\n",
      "[2019-01-07 22:27:09,991 INFO] Step 15150/100000; acc:  72.23; ppl:  2.13; xent: 0.76; lr: 0.00053; 10801/9887 tok/s;   1937 sec\n",
      "[2019-01-07 22:27:12,729 INFO] Step 15200/100000; acc:  72.32; ppl:  2.14; xent: 0.76; lr: 0.00053; 40380/36995 tok/s;   1940 sec\n",
      "[2019-01-07 22:27:12,801 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:27:20,037 INFO] Validation perplexity: 2.14827\n",
      "[2019-01-07 22:27:20,039 INFO] Validation accuracy: 72.113\n",
      "[2019-01-07 22:27:22,777 INFO] Step 15250/100000; acc:  72.08; ppl:  2.15; xent: 0.76; lr: 0.00053; 10848/9866 tok/s;   1950 sec\n",
      "[2019-01-07 22:27:25,463 INFO] Step 15300/100000; acc:  72.47; ppl:  2.11; xent: 0.75; lr: 0.00053; 40215/37051 tok/s;   1952 sec\n",
      "[2019-01-07 22:27:25,534 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:27:32,774 INFO] Validation perplexity: 2.06483\n",
      "[2019-01-07 22:27:32,775 INFO] Validation accuracy: 73.5728\n",
      "[2019-01-07 22:27:35,495 INFO] Step 15350/100000; acc:  72.15; ppl:  2.14; xent: 0.76; lr: 0.00053; 10987/9849 tok/s;   1962 sec\n",
      "[2019-01-07 22:27:38,208 INFO] Step 15400/100000; acc:  72.06; ppl:  2.15; xent: 0.76; lr: 0.00053; 40007/36861 tok/s;   1965 sec\n",
      "[2019-01-07 22:27:38,278 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:27:45,510 INFO] Validation perplexity: 2.08671\n",
      "[2019-01-07 22:27:45,511 INFO] Validation accuracy: 73.1699\n",
      "[2019-01-07 22:27:48,250 INFO] Step 15450/100000; acc:  72.19; ppl:  2.14; xent: 0.76; lr: 0.00053; 11223/10157 tok/s;   1975 sec\n",
      "[2019-01-07 22:27:50,979 INFO] Step 15500/100000; acc:  72.35; ppl:  2.13; xent: 0.75; lr: 0.00053; 41344/38063 tok/s;   1978 sec\n",
      "[2019-01-07 22:27:51,050 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:27:58,283 INFO] Validation perplexity: 2.09899\n",
      "[2019-01-07 22:27:58,284 INFO] Validation accuracy: 72.8663\n",
      "[2019-01-07 22:28:01,045 INFO] Step 15550/100000; acc:  71.66; ppl:  2.17; xent: 0.77; lr: 0.00053; 11330/10409 tok/s;   1988 sec\n",
      "[2019-01-07 22:28:03,727 INFO] Step 15600/100000; acc:  72.18; ppl:  2.13; xent: 0.75; lr: 0.00053; 39267/35542 tok/s;   1991 sec\n",
      "[2019-01-07 22:28:03,798 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:28:11,029 INFO] Validation perplexity: 2.08289\n",
      "[2019-01-07 22:28:11,031 INFO] Validation accuracy: 73.4436\n",
      "[2019-01-07 22:28:13,742 INFO] Step 15650/100000; acc:  72.24; ppl:  2.13; xent: 0.75; lr: 0.00053; 10952/10233 tok/s;   2001 sec\n",
      "[2019-01-07 22:28:16,440 INFO] Step 15700/100000; acc:  72.21; ppl:  2.14; xent: 0.76; lr: 0.00053; 40155/36306 tok/s;   2003 sec\n",
      "[2019-01-07 22:28:16,511 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:28:23,746 INFO] Validation perplexity: 2.02535\n",
      "[2019-01-07 22:28:23,748 INFO] Validation accuracy: 74.1562\n",
      "[2019-01-07 22:28:26,475 INFO] Step 15750/100000; acc:  72.01; ppl:  2.14; xent: 0.76; lr: 0.00053; 10746/9850 tok/s;   2013 sec\n",
      "[2019-01-07 22:28:29,198 INFO] Step 15800/100000; acc:  72.26; ppl:  2.13; xent: 0.76; lr: 0.00053; 40665/37151 tok/s;   2016 sec\n",
      "[2019-01-07 22:28:29,269 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:28:36,496 INFO] Validation perplexity: 2.12658\n",
      "[2019-01-07 22:28:36,497 INFO] Validation accuracy: 72.6572\n",
      "[2019-01-07 22:28:39,213 INFO] Step 15850/100000; acc:  71.92; ppl:  2.15; xent: 0.76; lr: 0.00053; 11102/10148 tok/s;   2026 sec\n",
      "[2019-01-07 22:28:41,934 INFO] Step 15900/100000; acc:  71.93; ppl:  2.15; xent: 0.77; lr: 0.00053; 41097/37620 tok/s;   2029 sec\n",
      "[2019-01-07 22:28:42,004 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:28:49,238 INFO] Validation perplexity: 2.09828\n",
      "[2019-01-07 22:28:49,239 INFO] Validation accuracy: 73.1616\n",
      "[2019-01-07 22:28:51,958 INFO] Step 15950/100000; acc:  72.06; ppl:  2.14; xent: 0.76; lr: 0.00053; 11299/10162 tok/s;   2039 sec\n",
      "[2019-01-07 22:28:54,665 INFO] Step 16000/100000; acc:  72.05; ppl:  2.14; xent: 0.76; lr: 0.00048; 39365/36195 tok/s;   2042 sec\n",
      "[2019-01-07 22:28:54,736 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:29:01,971 INFO] Validation perplexity: 2.09625\n",
      "[2019-01-07 22:29:01,972 INFO] Validation accuracy: 72.4853\n",
      "[2019-01-07 22:29:01,974 INFO] Saving checkpoint CNN_step_16000.pt\n",
      "[2019-01-07 22:29:04,843 INFO] Step 16050/100000; acc:  72.06; ppl:  2.13; xent: 0.76; lr: 0.00048; 11363/10413 tok/s;   2052 sec\n",
      "[2019-01-07 22:29:07,533 INFO] Step 16100/100000; acc:  72.24; ppl:  2.13; xent: 0.76; lr: 0.00048; 39648/35992 tok/s;   2054 sec\n",
      "[2019-01-07 22:29:07,604 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:29:14,837 INFO] Validation perplexity: 2.1173\n",
      "[2019-01-07 22:29:14,838 INFO] Validation accuracy: 72.647\n",
      "[2019-01-07 22:29:17,589 INFO] Step 16150/100000; acc:  72.11; ppl:  2.13; xent: 0.76; lr: 0.00048; 10975/10065 tok/s;   2064 sec\n",
      "[2019-01-07 22:29:20,282 INFO] Step 16200/100000; acc:  72.40; ppl:  2.12; xent: 0.75; lr: 0.00048; 39221/36010 tok/s;   2067 sec\n",
      "[2019-01-07 22:29:20,355 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:29:27,588 INFO] Validation perplexity: 2.09882\n",
      "[2019-01-07 22:29:27,590 INFO] Validation accuracy: 73.0412\n",
      "[2019-01-07 22:29:30,346 INFO] Step 16250/100000; acc:  72.38; ppl:  2.12; xent: 0.75; lr: 0.00048; 11383/10372 tok/s;   2077 sec\n",
      "[2019-01-07 22:29:33,109 INFO] Step 16300/100000; acc:  72.19; ppl:  2.13; xent: 0.76; lr: 0.00048; 41679/37815 tok/s;   2080 sec\n",
      "[2019-01-07 22:29:33,179 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:29:40,403 INFO] Validation perplexity: 2.13584\n",
      "[2019-01-07 22:29:40,405 INFO] Validation accuracy: 72.0092\n",
      "[2019-01-07 22:29:43,075 INFO] Step 16350/100000; acc:  72.76; ppl:  2.10; xent: 0.74; lr: 0.00048; 10236/9508 tok/s;   2090 sec\n",
      "[2019-01-07 22:29:45,776 INFO] Step 16400/100000; acc:  72.17; ppl:  2.13; xent: 0.75; lr: 0.00048; 42105/38339 tok/s;   2093 sec\n",
      "[2019-01-07 22:29:45,847 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:29:53,078 INFO] Validation perplexity: 2.16661\n",
      "[2019-01-07 22:29:53,079 INFO] Validation accuracy: 71.5294\n",
      "[2019-01-07 22:29:55,765 INFO] Step 16450/100000; acc:  72.28; ppl:  2.12; xent: 0.75; lr: 0.00048; 10678/9728 tok/s;   2103 sec\n",
      "[2019-01-07 22:29:58,494 INFO] Step 16500/100000; acc:  72.17; ppl:  2.13; xent: 0.76; lr: 0.00048; 40505/37515 tok/s;   2105 sec\n",
      "[2019-01-07 22:29:58,564 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:30:05,795 INFO] Validation perplexity: 2.14781\n",
      "[2019-01-07 22:30:05,796 INFO] Validation accuracy: 72.2617\n",
      "[2019-01-07 22:30:08,540 INFO] Step 16550/100000; acc:  72.12; ppl:  2.13; xent: 0.76; lr: 0.00048; 11097/9974 tok/s;   2115 sec\n",
      "[2019-01-07 22:30:11,251 INFO] Step 16600/100000; acc:  72.32; ppl:  2.12; xent: 0.75; lr: 0.00048; 40595/37021 tok/s;   2118 sec\n",
      "[2019-01-07 22:30:11,322 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:30:18,559 INFO] Validation perplexity: 2.03317\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:30:18,560 INFO] Validation accuracy: 74.0322\n",
      "[2019-01-07 22:30:21,259 INFO] Step 16650/100000; acc:  72.43; ppl:  2.11; xent: 0.75; lr: 0.00048; 11124/10197 tok/s;   2128 sec\n",
      "[2019-01-07 22:30:23,937 INFO] Step 16700/100000; acc:  72.35; ppl:  2.11; xent: 0.75; lr: 0.00048; 39362/35945 tok/s;   2131 sec\n",
      "[2019-01-07 22:30:24,008 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:30:31,241 INFO] Validation perplexity: 2.12126\n",
      "[2019-01-07 22:30:31,243 INFO] Validation accuracy: 72.73\n",
      "[2019-01-07 22:30:33,972 INFO] Step 16750/100000; acc:  72.07; ppl:  2.14; xent: 0.76; lr: 0.00048; 11265/10355 tok/s;   2141 sec\n",
      "[2019-01-07 22:30:36,669 INFO] Step 16800/100000; acc:  72.37; ppl:  2.12; xent: 0.75; lr: 0.00048; 40997/37721 tok/s;   2144 sec\n",
      "[2019-01-07 22:30:36,741 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:30:43,978 INFO] Validation perplexity: 2.15792\n",
      "[2019-01-07 22:30:43,980 INFO] Validation accuracy: 72.1353\n",
      "[2019-01-07 22:30:46,694 INFO] Step 16850/100000; acc:  72.36; ppl:  2.12; xent: 0.75; lr: 0.00048; 11203/10210 tok/s;   2154 sec\n",
      "[2019-01-07 22:30:49,396 INFO] Step 16900/100000; acc:  72.77; ppl:  2.09; xent: 0.74; lr: 0.00048; 38671/35492 tok/s;   2156 sec\n",
      "[2019-01-07 22:30:49,467 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:30:56,698 INFO] Validation perplexity: 2.07548\n",
      "[2019-01-07 22:30:56,699 INFO] Validation accuracy: 73.2236\n",
      "[2019-01-07 22:30:59,444 INFO] Step 16950/100000; acc:  71.87; ppl:  2.16; xent: 0.77; lr: 0.00048; 11298/10256 tok/s;   2166 sec\n",
      "[2019-01-07 22:31:02,148 INFO] Step 17000/100000; acc:  72.33; ppl:  2.13; xent: 0.75; lr: 0.00043; 40703/36886 tok/s;   2169 sec\n",
      "[2019-01-07 22:31:02,220 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:31:09,453 INFO] Validation perplexity: 2.1097\n",
      "[2019-01-07 22:31:09,455 INFO] Validation accuracy: 72.468\n",
      "[2019-01-07 22:31:09,456 INFO] Saving checkpoint CNN_step_17000.pt\n",
      "[2019-01-07 22:31:12,271 INFO] Step 17050/100000; acc:  72.80; ppl:  2.10; xent: 0.74; lr: 0.00043; 10720/9855 tok/s;   2179 sec\n",
      "[2019-01-07 22:31:15,000 INFO] Step 17100/100000; acc:  71.96; ppl:  2.14; xent: 0.76; lr: 0.00043; 41367/37697 tok/s;   2182 sec\n",
      "[2019-01-07 22:31:15,071 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:31:22,299 INFO] Validation perplexity: 2.11233\n",
      "[2019-01-07 22:31:22,301 INFO] Validation accuracy: 72.1491\n",
      "[2019-01-07 22:31:25,061 INFO] Step 17150/100000; acc:  72.41; ppl:  2.12; xent: 0.75; lr: 0.00043; 10925/10052 tok/s;   2192 sec\n",
      "[2019-01-07 22:31:27,753 INFO] Step 17200/100000; acc:  72.98; ppl:  2.08; xent: 0.73; lr: 0.00043; 39444/35058 tok/s;   2195 sec\n",
      "[2019-01-07 22:31:27,826 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:31:35,055 INFO] Validation perplexity: 2.15553\n",
      "[2019-01-07 22:31:35,056 INFO] Validation accuracy: 72.1865\n",
      "[2019-01-07 22:31:37,782 INFO] Step 17250/100000; acc:  72.56; ppl:  2.12; xent: 0.75; lr: 0.00043; 10925/10216 tok/s;   2205 sec\n",
      "[2019-01-07 22:31:40,487 INFO] Step 17300/100000; acc:  72.51; ppl:  2.11; xent: 0.75; lr: 0.00043; 42681/38780 tok/s;   2207 sec\n",
      "[2019-01-07 22:31:40,558 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:31:47,794 INFO] Validation perplexity: 2.10914\n",
      "[2019-01-07 22:31:47,795 INFO] Validation accuracy: 72.7066\n",
      "[2019-01-07 22:31:50,562 INFO] Step 17350/100000; acc:  72.11; ppl:  2.14; xent: 0.76; lr: 0.00043; 10981/10007 tok/s;   2217 sec\n",
      "[2019-01-07 22:31:53,342 INFO] Step 17400/100000; acc:  72.58; ppl:  2.11; xent: 0.74; lr: 0.00043; 40601/37609 tok/s;   2220 sec\n",
      "[2019-01-07 22:31:53,413 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:32:00,651 INFO] Validation perplexity: 2.04179\n",
      "[2019-01-07 22:32:00,653 INFO] Validation accuracy: 73.9234\n",
      "[2019-01-07 22:32:03,367 INFO] Step 17450/100000; acc:  72.57; ppl:  2.10; xent: 0.74; lr: 0.00043; 10504/9430 tok/s;   2230 sec\n",
      "[2019-01-07 22:32:06,102 INFO] Step 17500/100000; acc:  72.83; ppl:  2.10; xent: 0.74; lr: 0.00043; 41148/37751 tok/s;   2233 sec\n",
      "[2019-01-07 22:32:06,173 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:32:13,408 INFO] Validation perplexity: 2.20209\n",
      "[2019-01-07 22:32:13,410 INFO] Validation accuracy: 71.35\n",
      "[2019-01-07 22:32:16,118 INFO] Step 17550/100000; acc:  72.35; ppl:  2.12; xent: 0.75; lr: 0.00043; 10717/9880 tok/s;   2243 sec\n",
      "[2019-01-07 22:32:18,846 INFO] Step 17600/100000; acc:  72.61; ppl:  2.10; xent: 0.74; lr: 0.00043; 40413/36732 tok/s;   2246 sec\n",
      "[2019-01-07 22:32:18,917 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:32:26,151 INFO] Validation perplexity: 2.09806\n",
      "[2019-01-07 22:32:26,153 INFO] Validation accuracy: 73.0944\n",
      "[2019-01-07 22:32:28,876 INFO] Step 17650/100000; acc:  72.86; ppl:  2.09; xent: 0.74; lr: 0.00043; 10873/10102 tok/s;   2256 sec\n",
      "[2019-01-07 22:32:31,586 INFO] Step 17700/100000; acc:  72.74; ppl:  2.10; xent: 0.74; lr: 0.00043; 39970/36667 tok/s;   2258 sec\n",
      "[2019-01-07 22:32:31,659 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:32:38,910 INFO] Validation perplexity: 2.08503\n",
      "[2019-01-07 22:32:38,912 INFO] Validation accuracy: 73.2589\n",
      "[2019-01-07 22:32:41,647 INFO] Step 17750/100000; acc:  72.02; ppl:  2.15; xent: 0.76; lr: 0.00043; 10871/9819 tok/s;   2269 sec\n",
      "[2019-01-07 22:32:44,397 INFO] Step 17800/100000; acc:  72.32; ppl:  2.13; xent: 0.76; lr: 0.00043; 40558/37457 tok/s;   2271 sec\n",
      "[2019-01-07 22:32:44,467 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:32:51,726 INFO] Validation perplexity: 2.13474\n",
      "[2019-01-07 22:32:51,728 INFO] Validation accuracy: 72.3056\n",
      "[2019-01-07 22:32:53,722 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:32:56,199 INFO] Step 17850/100000; acc:  72.96; ppl:  2.08; xent: 0.73; lr: 0.00043; 9095/8316 tok/s;   2283 sec\n",
      "[2019-01-07 22:32:58,920 INFO] Step 17900/100000; acc:  72.44; ppl:  2.11; xent: 0.75; lr: 0.00043; 41340/37447 tok/s;   2286 sec\n",
      "[2019-01-07 22:32:58,992 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:33:06,218 INFO] Validation perplexity: 2.18333\n",
      "[2019-01-07 22:33:06,220 INFO] Validation accuracy: 71.7889\n",
      "[2019-01-07 22:33:08,961 INFO] Step 17950/100000; acc:  72.29; ppl:  2.12; xent: 0.75; lr: 0.00043; 11012/9945 tok/s;   2296 sec\n",
      "[2019-01-07 22:33:11,620 INFO] Step 18000/100000; acc:  72.88; ppl:  2.09; xent: 0.74; lr: 0.00039; 41536/38238 tok/s;   2299 sec\n",
      "[2019-01-07 22:33:11,691 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:33:18,927 INFO] Validation perplexity: 2.18424\n",
      "[2019-01-07 22:33:18,928 INFO] Validation accuracy: 71.8302\n",
      "[2019-01-07 22:33:18,930 INFO] Saving checkpoint CNN_step_18000.pt\n",
      "[2019-01-07 22:33:21,773 INFO] Step 18050/100000; acc:  72.32; ppl:  2.13; xent: 0.75; lr: 0.00039; 10910/10045 tok/s;   2309 sec\n",
      "[2019-01-07 22:33:24,459 INFO] Step 18100/100000; acc:  73.02; ppl:  2.09; xent: 0.74; lr: 0.00039; 40621/37233 tok/s;   2311 sec\n",
      "[2019-01-07 22:33:24,530 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:33:31,765 INFO] Validation perplexity: 2.11252\n",
      "[2019-01-07 22:33:31,767 INFO] Validation accuracy: 72.8829\n",
      "[2019-01-07 22:33:34,534 INFO] Step 18150/100000; acc:  72.56; ppl:  2.12; xent: 0.75; lr: 0.00039; 11039/10011 tok/s;   2321 sec\n",
      "[2019-01-07 22:33:37,240 INFO] Step 18200/100000; acc:  73.15; ppl:  2.08; xent: 0.73; lr: 0.00039; 40017/36344 tok/s;   2324 sec\n",
      "[2019-01-07 22:33:37,311 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:33:44,549 INFO] Validation perplexity: 2.09627\n",
      "[2019-01-07 22:33:44,550 INFO] Validation accuracy: 73.0523\n",
      "[2019-01-07 22:33:47,257 INFO] Step 18250/100000; acc:  73.05; ppl:  2.08; xent: 0.73; lr: 0.00039; 10476/9681 tok/s;   2334 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:33:49,969 INFO] Step 18300/100000; acc:  72.89; ppl:  2.09; xent: 0.74; lr: 0.00039; 42279/38955 tok/s;   2337 sec\n",
      "[2019-01-07 22:33:50,041 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:33:57,283 INFO] Validation perplexity: 2.14408\n",
      "[2019-01-07 22:33:57,284 INFO] Validation accuracy: 72.5707\n",
      "[2019-01-07 22:33:59,990 INFO] Step 18350/100000; acc:  72.95; ppl:  2.09; xent: 0.74; lr: 0.00039; 10424/9379 tok/s;   2347 sec\n",
      "[2019-01-07 22:34:02,719 INFO] Step 18400/100000; acc:  72.77; ppl:  2.10; xent: 0.74; lr: 0.00039; 42055/38700 tok/s;   2350 sec\n",
      "[2019-01-07 22:34:02,790 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:34:10,034 INFO] Validation perplexity: 2.10189\n",
      "[2019-01-07 22:34:10,036 INFO] Validation accuracy: 73.1857\n",
      "[2019-01-07 22:34:12,764 INFO] Step 18450/100000; acc:  72.48; ppl:  2.11; xent: 0.75; lr: 0.00039; 11310/10164 tok/s;   2360 sec\n",
      "[2019-01-07 22:34:15,489 INFO] Step 18500/100000; acc:  72.96; ppl:  2.09; xent: 0.74; lr: 0.00039; 39951/36787 tok/s;   2362 sec\n",
      "[2019-01-07 22:34:15,560 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:34:22,801 INFO] Validation perplexity: 2.21543\n",
      "[2019-01-07 22:34:22,803 INFO] Validation accuracy: 71.4545\n",
      "[2019-01-07 22:34:25,557 INFO] Step 18550/100000; acc:  72.59; ppl:  2.10; xent: 0.74; lr: 0.00039; 11288/10343 tok/s;   2372 sec\n",
      "[2019-01-07 22:34:28,235 INFO] Step 18600/100000; acc:  73.00; ppl:  2.08; xent: 0.73; lr: 0.00039; 39147/36170 tok/s;   2375 sec\n",
      "[2019-01-07 22:34:28,305 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:34:35,544 INFO] Validation perplexity: 2.11059\n",
      "[2019-01-07 22:34:35,546 INFO] Validation accuracy: 73.0244\n",
      "[2019-01-07 22:34:38,245 INFO] Step 18650/100000; acc:  72.91; ppl:  2.09; xent: 0.74; lr: 0.00039; 10661/9815 tok/s;   2385 sec\n",
      "[2019-01-07 22:34:41,015 INFO] Step 18700/100000; acc:  72.43; ppl:  2.12; xent: 0.75; lr: 0.00039; 41655/37571 tok/s;   2388 sec\n",
      "[2019-01-07 22:34:41,089 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:34:48,327 INFO] Validation perplexity: 2.10948\n",
      "[2019-01-07 22:34:48,329 INFO] Validation accuracy: 72.7199\n",
      "[2019-01-07 22:34:51,033 INFO] Step 18750/100000; acc:  72.89; ppl:  2.09; xent: 0.74; lr: 0.00039; 10519/9697 tok/s;   2398 sec\n",
      "[2019-01-07 22:34:53,795 INFO] Step 18800/100000; acc:  72.49; ppl:  2.12; xent: 0.75; lr: 0.00039; 41955/37609 tok/s;   2401 sec\n",
      "[2019-01-07 22:34:53,868 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:35:01,107 INFO] Validation perplexity: 2.06227\n",
      "[2019-01-07 22:35:01,108 INFO] Validation accuracy: 73.3554\n",
      "[2019-01-07 22:35:03,824 INFO] Step 18850/100000; acc:  73.00; ppl:  2.09; xent: 0.74; lr: 0.00039; 10657/9885 tok/s;   2411 sec\n",
      "[2019-01-07 22:35:06,526 INFO] Step 18900/100000; acc:  72.80; ppl:  2.10; xent: 0.74; lr: 0.00039; 41267/37465 tok/s;   2413 sec\n",
      "[2019-01-07 22:35:06,597 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:35:13,850 INFO] Validation perplexity: 2.14331\n",
      "[2019-01-07 22:35:13,852 INFO] Validation accuracy: 71.9603\n",
      "[2019-01-07 22:35:16,602 INFO] Step 18950/100000; acc:  72.86; ppl:  2.08; xent: 0.73; lr: 0.00039; 10649/9768 tok/s;   2423 sec\n",
      "[2019-01-07 22:35:19,343 INFO] Step 19000/100000; acc:  72.55; ppl:  2.10; xent: 0.74; lr: 0.00035; 42271/38617 tok/s;   2426 sec\n",
      "[2019-01-07 22:35:19,415 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:35:26,651 INFO] Validation perplexity: 2.12622\n",
      "[2019-01-07 22:35:26,653 INFO] Validation accuracy: 72.415\n",
      "[2019-01-07 22:35:26,655 INFO] Saving checkpoint CNN_step_19000.pt\n",
      "[2019-01-07 22:35:29,509 INFO] Step 19050/100000; acc:  72.69; ppl:  2.10; xent: 0.74; lr: 0.00035; 11093/10107 tok/s;   2436 sec\n",
      "[2019-01-07 22:35:32,184 INFO] Step 19100/100000; acc:  73.13; ppl:  2.08; xent: 0.73; lr: 0.00035; 39567/36480 tok/s;   2439 sec\n",
      "[2019-01-07 22:35:32,255 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:35:39,497 INFO] Validation perplexity: 2.21553\n",
      "[2019-01-07 22:35:39,499 INFO] Validation accuracy: 71.4342\n",
      "[2019-01-07 22:35:42,234 INFO] Step 19150/100000; acc:  72.87; ppl:  2.08; xent: 0.73; lr: 0.00035; 10862/9796 tok/s;   2449 sec\n",
      "[2019-01-07 22:35:44,943 INFO] Step 19200/100000; acc:  72.83; ppl:  2.08; xent: 0.73; lr: 0.00035; 40102/37070 tok/s;   2452 sec\n",
      "[2019-01-07 22:35:45,014 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:35:52,252 INFO] Validation perplexity: 2.10205\n",
      "[2019-01-07 22:35:52,254 INFO] Validation accuracy: 72.8259\n",
      "[2019-01-07 22:35:55,029 INFO] Step 19250/100000; acc:  72.72; ppl:  2.09; xent: 0.74; lr: 0.00035; 11445/10421 tok/s;   2462 sec\n",
      "[2019-01-07 22:35:57,722 INFO] Step 19300/100000; acc:  73.43; ppl:  2.06; xent: 0.72; lr: 0.00035; 39956/35990 tok/s;   2465 sec\n",
      "[2019-01-07 22:35:57,793 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:36:05,033 INFO] Validation perplexity: 2.15177\n",
      "[2019-01-07 22:36:05,035 INFO] Validation accuracy: 72.019\n",
      "[2019-01-07 22:36:07,780 INFO] Step 19350/100000; acc:  72.78; ppl:  2.10; xent: 0.74; lr: 0.00035; 11130/10270 tok/s;   2475 sec\n",
      "[2019-01-07 22:36:10,446 INFO] Step 19400/100000; acc:  72.97; ppl:  2.08; xent: 0.73; lr: 0.00035; 41137/37960 tok/s;   2477 sec\n",
      "[2019-01-07 22:36:10,518 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:36:17,753 INFO] Validation perplexity: 2.1833\n",
      "[2019-01-07 22:36:17,754 INFO] Validation accuracy: 71.8397\n",
      "[2019-01-07 22:36:20,431 INFO] Step 19450/100000; acc:  72.81; ppl:  2.09; xent: 0.74; lr: 0.00035; 10519/9689 tok/s;   2487 sec\n",
      "[2019-01-07 22:36:23,161 INFO] Step 19500/100000; acc:  73.04; ppl:  2.08; xent: 0.73; lr: 0.00035; 40726/37125 tok/s;   2490 sec\n",
      "[2019-01-07 22:36:23,232 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:36:30,466 INFO] Validation perplexity: 2.10357\n",
      "[2019-01-07 22:36:30,468 INFO] Validation accuracy: 72.5075\n",
      "[2019-01-07 22:36:33,182 INFO] Step 19550/100000; acc:  72.72; ppl:  2.09; xent: 0.74; lr: 0.00035; 10734/9654 tok/s;   2500 sec\n",
      "[2019-01-07 22:36:35,910 INFO] Step 19600/100000; acc:  73.12; ppl:  2.07; xent: 0.73; lr: 0.00035; 41821/38405 tok/s;   2503 sec\n",
      "[2019-01-07 22:36:35,984 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:36:43,224 INFO] Validation perplexity: 2.10667\n",
      "[2019-01-07 22:36:43,225 INFO] Validation accuracy: 72.8129\n",
      "[2019-01-07 22:36:45,925 INFO] Step 19650/100000; acc:  72.94; ppl:  2.08; xent: 0.73; lr: 0.00035; 10708/9727 tok/s;   2513 sec\n",
      "[2019-01-07 22:36:48,634 INFO] Step 19700/100000; acc:  73.19; ppl:  2.08; xent: 0.73; lr: 0.00035; 40568/37182 tok/s;   2516 sec\n",
      "[2019-01-07 22:36:48,704 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:36:55,933 INFO] Validation perplexity: 2.10825\n",
      "[2019-01-07 22:36:55,934 INFO] Validation accuracy: 72.7799\n",
      "[2019-01-07 22:36:58,676 INFO] Step 19750/100000; acc:  72.44; ppl:  2.11; xent: 0.75; lr: 0.00035; 11011/10164 tok/s;   2526 sec\n",
      "[2019-01-07 22:37:01,420 INFO] Step 19800/100000; acc:  72.78; ppl:  2.10; xent: 0.74; lr: 0.00035; 41864/38135 tok/s;   2528 sec\n",
      "[2019-01-07 22:37:01,491 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:37:08,729 INFO] Validation perplexity: 2.13185\n",
      "[2019-01-07 22:37:08,731 INFO] Validation accuracy: 72.6294\n",
      "[2019-01-07 22:37:11,431 INFO] Step 19850/100000; acc:  73.53; ppl:  2.05; xent: 0.72; lr: 0.00035; 10722/9820 tok/s;   2538 sec\n",
      "[2019-01-07 22:37:14,161 INFO] Step 19900/100000; acc:  72.63; ppl:  2.10; xent: 0.74; lr: 0.00035; 40293/36872 tok/s;   2541 sec\n",
      "[2019-01-07 22:37:14,234 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:37:21,472 INFO] Validation perplexity: 2.1142\n",
      "[2019-01-07 22:37:21,473 INFO] Validation accuracy: 72.6768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:37:24,160 INFO] Step 19950/100000; acc:  73.30; ppl:  2.08; xent: 0.73; lr: 0.00035; 10968/10038 tok/s;   2551 sec\n",
      "[2019-01-07 22:37:26,876 INFO] Step 20000/100000; acc:  72.54; ppl:  2.11; xent: 0.75; lr: 0.00031; 40049/36090 tok/s;   2554 sec\n",
      "[2019-01-07 22:37:26,949 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:37:34,190 INFO] Validation perplexity: 2.08223\n",
      "[2019-01-07 22:37:34,191 INFO] Validation accuracy: 73.2569\n",
      "[2019-01-07 22:37:34,193 INFO] Saving checkpoint CNN_step_20000.pt\n",
      "[2019-01-07 22:37:37,033 INFO] Step 20050/100000; acc:  72.92; ppl:  2.09; xent: 0.74; lr: 0.00031; 11243/10405 tok/s;   2564 sec\n",
      "[2019-01-07 22:37:39,723 INFO] Step 20100/100000; acc:  73.22; ppl:  2.07; xent: 0.73; lr: 0.00031; 40620/36997 tok/s;   2567 sec\n",
      "[2019-01-07 22:37:39,796 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:37:47,023 INFO] Validation perplexity: 2.08283\n",
      "[2019-01-07 22:37:47,025 INFO] Validation accuracy: 73.1209\n",
      "[2019-01-07 22:37:49,703 INFO] Step 20150/100000; acc:  73.58; ppl:  2.04; xent: 0.71; lr: 0.00031; 10346/9313 tok/s;   2577 sec\n",
      "[2019-01-07 22:37:52,461 INFO] Step 20200/100000; acc:  72.80; ppl:  2.09; xent: 0.74; lr: 0.00031; 41541/38431 tok/s;   2579 sec\n",
      "[2019-01-07 22:37:52,535 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:37:59,766 INFO] Validation perplexity: 2.14713\n",
      "[2019-01-07 22:37:59,767 INFO] Validation accuracy: 71.8266\n",
      "[2019-01-07 22:38:02,491 INFO] Step 20250/100000; acc:  73.02; ppl:  2.07; xent: 0.73; lr: 0.00031; 10948/10010 tok/s;   2589 sec\n",
      "[2019-01-07 22:38:05,217 INFO] Step 20300/100000; acc:  72.59; ppl:  2.11; xent: 0.75; lr: 0.00031; 41591/37769 tok/s;   2592 sec\n",
      "[2019-01-07 22:38:05,287 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:38:12,517 INFO] Validation perplexity: 2.12782\n",
      "[2019-01-07 22:38:12,518 INFO] Validation accuracy: 72.1412\n",
      "[2019-01-07 22:38:15,267 INFO] Step 20350/100000; acc:  72.98; ppl:  2.08; xent: 0.73; lr: 0.00031; 11225/10332 tok/s;   2602 sec\n",
      "[2019-01-07 22:38:17,963 INFO] Step 20400/100000; acc:  73.46; ppl:  2.06; xent: 0.72; lr: 0.00031; 39845/35950 tok/s;   2605 sec\n",
      "[2019-01-07 22:38:18,035 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:38:25,271 INFO] Validation perplexity: 2.09506\n",
      "[2019-01-07 22:38:25,273 INFO] Validation accuracy: 73.2011\n",
      "[2019-01-07 22:38:27,959 INFO] Step 20450/100000; acc:  73.29; ppl:  2.06; xent: 0.72; lr: 0.00031; 10610/9740 tok/s;   2615 sec\n",
      "[2019-01-07 22:38:30,698 INFO] Step 20500/100000; acc:  73.03; ppl:  2.09; xent: 0.74; lr: 0.00031; 41195/37610 tok/s;   2618 sec\n",
      "[2019-01-07 22:38:30,769 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:38:37,997 INFO] Validation perplexity: 2.14475\n",
      "[2019-01-07 22:38:37,998 INFO] Validation accuracy: 72.5009\n",
      "[2019-01-07 22:38:40,709 INFO] Step 20550/100000; acc:  72.86; ppl:  2.08; xent: 0.73; lr: 0.00031; 11217/10201 tok/s;   2628 sec\n",
      "[2019-01-07 22:38:43,402 INFO] Step 20600/100000; acc:  73.20; ppl:  2.07; xent: 0.73; lr: 0.00031; 39395/36543 tok/s;   2630 sec\n",
      "[2019-01-07 22:38:43,476 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:38:50,712 INFO] Validation perplexity: 2.08733\n",
      "[2019-01-07 22:38:50,714 INFO] Validation accuracy: 73.2133\n",
      "[2019-01-07 22:38:53,436 INFO] Step 20650/100000; acc:  73.17; ppl:  2.07; xent: 0.73; lr: 0.00031; 10965/10018 tok/s;   2640 sec\n",
      "[2019-01-07 22:38:56,140 INFO] Step 20700/100000; acc:  72.80; ppl:  2.09; xent: 0.74; lr: 0.00031; 40125/36822 tok/s;   2643 sec\n",
      "[2019-01-07 22:38:56,211 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:39:03,440 INFO] Validation perplexity: 2.08137\n",
      "[2019-01-07 22:39:03,441 INFO] Validation accuracy: 73.4059\n",
      "[2019-01-07 22:39:06,198 INFO] Step 20750/100000; acc:  72.39; ppl:  2.11; xent: 0.75; lr: 0.00031; 11279/10212 tok/s;   2653 sec\n",
      "[2019-01-07 22:39:09,154 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:39:10,653 INFO] Step 20800/100000; acc:  73.81; ppl:  2.03; xent: 0.71; lr: 0.00031; 23833/22137 tok/s;   2658 sec\n",
      "[2019-01-07 22:39:10,725 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:39:17,943 INFO] Validation perplexity: 2.11601\n",
      "[2019-01-07 22:39:17,945 INFO] Validation accuracy: 72.7236\n",
      "[2019-01-07 22:39:20,653 INFO] Step 20850/100000; acc:  73.17; ppl:  2.08; xent: 0.73; lr: 0.00031; 10858/9712 tok/s;   2668 sec\n",
      "[2019-01-07 22:39:23,401 INFO] Step 20900/100000; acc:  72.81; ppl:  2.09; xent: 0.74; lr: 0.00031; 40703/37378 tok/s;   2670 sec\n",
      "[2019-01-07 22:39:23,472 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:39:30,723 INFO] Validation perplexity: 2.11124\n",
      "[2019-01-07 22:39:30,725 INFO] Validation accuracy: 72.9288\n",
      "[2019-01-07 22:39:33,427 INFO] Step 20950/100000; acc:  73.12; ppl:  2.06; xent: 0.73; lr: 0.00031; 10967/9963 tok/s;   2680 sec\n",
      "[2019-01-07 22:39:36,142 INFO] Step 21000/100000; acc:  73.15; ppl:  2.07; xent: 0.73; lr: 0.00028; 40473/37605 tok/s;   2683 sec\n",
      "[2019-01-07 22:39:36,215 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:39:43,468 INFO] Validation perplexity: 2.18249\n",
      "[2019-01-07 22:39:43,470 INFO] Validation accuracy: 71.547\n",
      "[2019-01-07 22:39:43,471 INFO] Saving checkpoint CNN_step_21000.pt\n",
      "[2019-01-07 22:39:46,341 INFO] Step 21050/100000; acc:  72.66; ppl:  2.10; xent: 0.74; lr: 0.00028; 11241/10163 tok/s;   2693 sec\n",
      "[2019-01-07 22:39:49,072 INFO] Step 21100/100000; acc:  73.59; ppl:  2.05; xent: 0.72; lr: 0.00028; 39588/36717 tok/s;   2696 sec\n",
      "[2019-01-07 22:39:49,143 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:39:56,385 INFO] Validation perplexity: 2.14779\n",
      "[2019-01-07 22:39:56,387 INFO] Validation accuracy: 72.3074\n",
      "[2019-01-07 22:39:59,087 INFO] Step 21150/100000; acc:  73.59; ppl:  2.05; xent: 0.72; lr: 0.00028; 10679/9770 tok/s;   2706 sec\n",
      "[2019-01-07 22:40:01,847 INFO] Step 21200/100000; acc:  73.25; ppl:  2.07; xent: 0.73; lr: 0.00028; 40387/36481 tok/s;   2709 sec\n",
      "[2019-01-07 22:40:01,918 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:40:09,165 INFO] Validation perplexity: 2.1176\n",
      "[2019-01-07 22:40:09,167 INFO] Validation accuracy: 72.7676\n",
      "[2019-01-07 22:40:11,854 INFO] Step 21250/100000; acc:  73.75; ppl:  2.04; xent: 0.71; lr: 0.00028; 10861/10074 tok/s;   2719 sec\n",
      "[2019-01-07 22:40:14,560 INFO] Step 21300/100000; acc:  73.21; ppl:  2.07; xent: 0.73; lr: 0.00028; 40257/36301 tok/s;   2721 sec\n",
      "[2019-01-07 22:40:14,631 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:40:21,883 INFO] Validation perplexity: 2.07408\n",
      "[2019-01-07 22:40:21,885 INFO] Validation accuracy: 73.1938\n",
      "[2019-01-07 22:40:24,593 INFO] Step 21350/100000; acc:  73.30; ppl:  2.06; xent: 0.72; lr: 0.00028; 10919/10018 tok/s;   2731 sec\n",
      "[2019-01-07 22:40:27,322 INFO] Step 21400/100000; acc:  73.16; ppl:  2.07; xent: 0.73; lr: 0.00028; 40160/36417 tok/s;   2734 sec\n",
      "[2019-01-07 22:40:27,393 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:40:34,649 INFO] Validation perplexity: 2.07969\n",
      "[2019-01-07 22:40:34,651 INFO] Validation accuracy: 73.3064\n",
      "[2019-01-07 22:40:37,363 INFO] Step 21450/100000; acc:  73.32; ppl:  2.06; xent: 0.72; lr: 0.00028; 11152/10259 tok/s;   2744 sec\n",
      "[2019-01-07 22:40:40,129 INFO] Step 21500/100000; acc:  73.01; ppl:  2.09; xent: 0.74; lr: 0.00028; 42330/38341 tok/s;   2747 sec\n",
      "[2019-01-07 22:40:40,200 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:40:47,448 INFO] Validation perplexity: 2.05905\n",
      "[2019-01-07 22:40:47,450 INFO] Validation accuracy: 73.8187\n",
      "[2019-01-07 22:40:50,160 INFO] Step 21550/100000; acc:  73.28; ppl:  2.06; xent: 0.72; lr: 0.00028; 10573/9644 tok/s;   2757 sec\n",
      "[2019-01-07 22:40:52,859 INFO] Step 21600/100000; acc:  73.67; ppl:  2.05; xent: 0.72; lr: 0.00028; 39537/37332 tok/s;   2760 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:40:52,930 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:41:00,176 INFO] Validation perplexity: 2.13479\n",
      "[2019-01-07 22:41:00,177 INFO] Validation accuracy: 72.0403\n",
      "[2019-01-07 22:41:02,896 INFO] Step 21650/100000; acc:  73.39; ppl:  2.06; xent: 0.72; lr: 0.00028; 10959/9904 tok/s;   2770 sec\n",
      "[2019-01-07 22:41:05,638 INFO] Step 21700/100000; acc:  72.94; ppl:  2.09; xent: 0.74; lr: 0.00028; 39826/36278 tok/s;   2773 sec\n",
      "[2019-01-07 22:41:05,710 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:41:12,962 INFO] Validation perplexity: 2.16763\n",
      "[2019-01-07 22:41:12,964 INFO] Validation accuracy: 72.0544\n",
      "[2019-01-07 22:41:15,707 INFO] Step 21750/100000; acc:  73.15; ppl:  2.08; xent: 0.73; lr: 0.00028; 11479/10356 tok/s;   2783 sec\n",
      "[2019-01-07 22:41:18,434 INFO] Step 21800/100000; acc:  73.21; ppl:  2.07; xent: 0.73; lr: 0.00028; 39663/36536 tok/s;   2785 sec\n",
      "[2019-01-07 22:41:18,507 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:41:25,756 INFO] Validation perplexity: 2.09859\n",
      "[2019-01-07 22:41:25,757 INFO] Validation accuracy: 73.174\n",
      "[2019-01-07 22:41:28,479 INFO] Step 21850/100000; acc:  73.03; ppl:  2.08; xent: 0.73; lr: 0.00028; 10984/9997 tok/s;   2795 sec\n",
      "[2019-01-07 22:41:31,204 INFO] Step 21900/100000; acc:  73.89; ppl:  2.03; xent: 0.71; lr: 0.00028; 39630/36128 tok/s;   2798 sec\n",
      "[2019-01-07 22:41:31,279 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:41:38,539 INFO] Validation perplexity: 2.1174\n",
      "[2019-01-07 22:41:38,541 INFO] Validation accuracy: 72.766\n",
      "[2019-01-07 22:41:41,266 INFO] Step 21950/100000; acc:  72.96; ppl:  2.08; xent: 0.73; lr: 0.00028; 11072/10107 tok/s;   2808 sec\n",
      "[2019-01-07 22:41:44,033 INFO] Step 22000/100000; acc:  73.15; ppl:  2.07; xent: 0.73; lr: 0.00025; 41976/38261 tok/s;   2811 sec\n",
      "[2019-01-07 22:41:44,104 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:41:51,346 INFO] Validation perplexity: 2.13244\n",
      "[2019-01-07 22:41:51,348 INFO] Validation accuracy: 72.1076\n",
      "[2019-01-07 22:41:51,349 INFO] Saving checkpoint CNN_step_22000.pt\n",
      "[2019-01-07 22:41:54,154 INFO] Step 22050/100000; acc:  73.41; ppl:  2.05; xent: 0.72; lr: 0.00025; 10315/9547 tok/s;   2821 sec\n",
      "[2019-01-07 22:41:56,880 INFO] Step 22100/100000; acc:  73.34; ppl:  2.06; xent: 0.72; lr: 0.00025; 40943/37136 tok/s;   2824 sec\n",
      "[2019-01-07 22:41:56,953 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:42:04,194 INFO] Validation perplexity: 2.09517\n",
      "[2019-01-07 22:42:04,196 INFO] Validation accuracy: 73.0352\n",
      "[2019-01-07 22:42:06,907 INFO] Step 22150/100000; acc:  73.53; ppl:  2.05; xent: 0.72; lr: 0.00025; 10605/9760 tok/s;   2834 sec\n",
      "[2019-01-07 22:42:09,667 INFO] Step 22200/100000; acc:  73.13; ppl:  2.06; xent: 0.72; lr: 0.00025; 41365/37370 tok/s;   2837 sec\n",
      "[2019-01-07 22:42:09,738 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:42:16,990 INFO] Validation perplexity: 2.12567\n",
      "[2019-01-07 22:42:16,992 INFO] Validation accuracy: 72.5806\n",
      "[2019-01-07 22:42:19,717 INFO] Step 22250/100000; acc:  73.70; ppl:  2.05; xent: 0.72; lr: 0.00025; 11465/10503 tok/s;   2847 sec\n",
      "[2019-01-07 22:42:22,397 INFO] Step 22300/100000; acc:  73.39; ppl:  2.07; xent: 0.73; lr: 0.00025; 39281/36207 tok/s;   2849 sec\n",
      "[2019-01-07 22:42:22,468 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:42:29,715 INFO] Validation perplexity: 2.12595\n",
      "[2019-01-07 22:42:29,717 INFO] Validation accuracy: 72.739\n",
      "[2019-01-07 22:42:32,418 INFO] Step 22350/100000; acc:  73.18; ppl:  2.06; xent: 0.72; lr: 0.00025; 11160/10100 tok/s;   2859 sec\n",
      "[2019-01-07 22:42:35,097 INFO] Step 22400/100000; acc:  73.82; ppl:  2.03; xent: 0.71; lr: 0.00025; 39093/36438 tok/s;   2862 sec\n",
      "[2019-01-07 22:42:35,169 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:42:42,431 INFO] Validation perplexity: 2.09219\n",
      "[2019-01-07 22:42:42,433 INFO] Validation accuracy: 73.0805\n",
      "[2019-01-07 22:42:45,156 INFO] Step 22450/100000; acc:  73.07; ppl:  2.07; xent: 0.73; lr: 0.00025; 11469/10446 tok/s;   2872 sec\n",
      "[2019-01-07 22:42:47,847 INFO] Step 22500/100000; acc:  73.23; ppl:  2.06; xent: 0.72; lr: 0.00025; 38789/34911 tok/s;   2875 sec\n",
      "[2019-01-07 22:42:47,918 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:42:55,168 INFO] Validation perplexity: 2.04523\n",
      "[2019-01-07 22:42:55,169 INFO] Validation accuracy: 73.8886\n",
      "[2019-01-07 22:42:57,881 INFO] Step 22550/100000; acc:  73.74; ppl:  2.03; xent: 0.71; lr: 0.00025; 11115/10252 tok/s;   2885 sec\n",
      "[2019-01-07 22:43:00,594 INFO] Step 22600/100000; acc:  73.40; ppl:  2.05; xent: 0.72; lr: 0.00025; 40429/36669 tok/s;   2887 sec\n",
      "[2019-01-07 22:43:00,667 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:43:07,916 INFO] Validation perplexity: 2.0703\n",
      "[2019-01-07 22:43:07,917 INFO] Validation accuracy: 73.4085\n",
      "[2019-01-07 22:43:10,635 INFO] Step 22650/100000; acc:  73.58; ppl:  2.05; xent: 0.72; lr: 0.00025; 10965/9964 tok/s;   2898 sec\n",
      "[2019-01-07 22:43:13,378 INFO] Step 22700/100000; acc:  73.08; ppl:  2.08; xent: 0.73; lr: 0.00025; 41130/38101 tok/s;   2900 sec\n",
      "[2019-01-07 22:43:13,450 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:43:20,700 INFO] Validation perplexity: 2.15234\n",
      "[2019-01-07 22:43:20,702 INFO] Validation accuracy: 72.2344\n",
      "[2019-01-07 22:43:23,439 INFO] Step 22750/100000; acc:  73.32; ppl:  2.07; xent: 0.73; lr: 0.00025; 11140/10173 tok/s;   2910 sec\n",
      "[2019-01-07 22:43:26,139 INFO] Step 22800/100000; acc:  73.75; ppl:  2.04; xent: 0.71; lr: 0.00025; 40461/37053 tok/s;   2913 sec\n",
      "[2019-01-07 22:43:26,211 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:43:33,450 INFO] Validation perplexity: 2.06613\n",
      "[2019-01-07 22:43:33,451 INFO] Validation accuracy: 73.4826\n",
      "[2019-01-07 22:43:36,166 INFO] Step 22850/100000; acc:  73.75; ppl:  2.05; xent: 0.72; lr: 0.00025; 10707/9774 tok/s;   2923 sec\n",
      "[2019-01-07 22:43:38,885 INFO] Step 22900/100000; acc:  73.26; ppl:  2.07; xent: 0.73; lr: 0.00025; 40079/36511 tok/s;   2926 sec\n",
      "[2019-01-07 22:43:38,957 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:43:46,204 INFO] Validation perplexity: 2.1476\n",
      "[2019-01-07 22:43:46,206 INFO] Validation accuracy: 72.4818\n",
      "[2019-01-07 22:43:48,948 INFO] Step 22950/100000; acc:  73.11; ppl:  2.08; xent: 0.73; lr: 0.00025; 11284/10199 tok/s;   2936 sec\n",
      "[2019-01-07 22:43:51,625 INFO] Step 23000/100000; acc:  73.95; ppl:  2.03; xent: 0.71; lr: 0.00023; 39897/37029 tok/s;   2939 sec\n",
      "[2019-01-07 22:43:51,696 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:43:58,933 INFO] Validation perplexity: 2.14898\n",
      "[2019-01-07 22:43:58,934 INFO] Validation accuracy: 72.3897\n",
      "[2019-01-07 22:43:58,936 INFO] Saving checkpoint CNN_step_23000.pt\n",
      "[2019-01-07 22:44:01,767 INFO] Step 23050/100000; acc:  73.10; ppl:  2.07; xent: 0.73; lr: 0.00023; 10978/10031 tok/s;   2949 sec\n",
      "[2019-01-07 22:44:04,483 INFO] Step 23100/100000; acc:  73.56; ppl:  2.04; xent: 0.72; lr: 0.00023; 39417/35594 tok/s;   2951 sec\n",
      "[2019-01-07 22:44:04,554 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:44:11,791 INFO] Validation perplexity: 2.06193\n",
      "[2019-01-07 22:44:11,792 INFO] Validation accuracy: 73.3967\n",
      "[2019-01-07 22:44:14,526 INFO] Step 23150/100000; acc:  73.89; ppl:  2.03; xent: 0.71; lr: 0.00023; 11185/10158 tok/s;   2961 sec\n",
      "[2019-01-07 22:44:17,231 INFO] Step 23200/100000; acc:  73.19; ppl:  2.07; xent: 0.73; lr: 0.00023; 40477/37706 tok/s;   2964 sec\n",
      "[2019-01-07 22:44:17,304 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:44:24,532 INFO] Validation perplexity: 2.03137\n",
      "[2019-01-07 22:44:24,534 INFO] Validation accuracy: 73.8881\n",
      "[2019-01-07 22:44:27,250 INFO] Step 23250/100000; acc:  73.22; ppl:  2.07; xent: 0.73; lr: 0.00023; 11403/10347 tok/s;   2974 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:44:30,001 INFO] Step 23300/100000; acc:  73.11; ppl:  2.07; xent: 0.73; lr: 0.00023; 40301/36605 tok/s;   2977 sec\n",
      "[2019-01-07 22:44:30,073 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:44:37,311 INFO] Validation perplexity: 2.06681\n",
      "[2019-01-07 22:44:37,313 INFO] Validation accuracy: 73.6026\n",
      "[2019-01-07 22:44:40,031 INFO] Step 23350/100000; acc:  73.81; ppl:  2.04; xent: 0.71; lr: 0.00023; 10895/10067 tok/s;   2987 sec\n",
      "[2019-01-07 22:44:42,762 INFO] Step 23400/100000; acc:  73.62; ppl:  2.05; xent: 0.72; lr: 0.00023; 39988/36118 tok/s;   2990 sec\n",
      "[2019-01-07 22:44:42,833 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:44:50,068 INFO] Validation perplexity: 2.11099\n",
      "[2019-01-07 22:44:50,070 INFO] Validation accuracy: 72.8685\n",
      "[2019-01-07 22:44:52,787 INFO] Step 23450/100000; acc:  73.59; ppl:  2.05; xent: 0.72; lr: 0.00023; 10690/9821 tok/s;   3000 sec\n",
      "[2019-01-07 22:44:55,531 INFO] Step 23500/100000; acc:  73.22; ppl:  2.07; xent: 0.73; lr: 0.00023; 40808/37166 tok/s;   3002 sec\n",
      "[2019-01-07 22:44:55,602 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:45:02,841 INFO] Validation perplexity: 2.12231\n",
      "[2019-01-07 22:45:02,843 INFO] Validation accuracy: 72.5401\n",
      "[2019-01-07 22:45:05,553 INFO] Step 23550/100000; acc:  73.60; ppl:  2.04; xent: 0.71; lr: 0.00023; 10869/9950 tok/s;   3012 sec\n",
      "[2019-01-07 22:45:08,261 INFO] Step 23600/100000; acc:  73.65; ppl:  2.05; xent: 0.72; lr: 0.00023; 40934/37431 tok/s;   3015 sec\n",
      "[2019-01-07 22:45:08,335 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:45:15,569 INFO] Validation perplexity: 2.05916\n",
      "[2019-01-07 22:45:15,571 INFO] Validation accuracy: 73.8067\n",
      "[2019-01-07 22:45:18,257 INFO] Step 23650/100000; acc:  73.56; ppl:  2.05; xent: 0.72; lr: 0.00023; 10819/10113 tok/s;   3025 sec\n",
      "[2019-01-07 22:45:20,976 INFO] Step 23700/100000; acc:  73.15; ppl:  2.07; xent: 0.73; lr: 0.00023; 39482/35353 tok/s;   3028 sec\n",
      "[2019-01-07 22:45:21,049 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:45:28,315 INFO] Validation perplexity: 2.10363\n",
      "[2019-01-07 22:45:28,317 INFO] Validation accuracy: 72.8485\n",
      "[2019-01-07 22:45:32,295 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:45:32,811 INFO] Step 23750/100000; acc:  73.66; ppl:  2.04; xent: 0.71; lr: 0.00023; 9425/8734 tok/s;   3040 sec\n",
      "[2019-01-07 22:45:35,513 INFO] Step 23800/100000; acc:  73.73; ppl:  2.04; xent: 0.71; lr: 0.00023; 40056/36532 tok/s;   3042 sec\n",
      "[2019-01-07 22:45:35,585 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:45:42,813 INFO] Validation perplexity: 2.11969\n",
      "[2019-01-07 22:45:42,814 INFO] Validation accuracy: 72.6955\n",
      "[2019-01-07 22:45:45,545 INFO] Step 23850/100000; acc:  73.39; ppl:  2.06; xent: 0.72; lr: 0.00023; 11165/10103 tok/s;   3052 sec\n",
      "[2019-01-07 22:45:48,238 INFO] Step 23900/100000; acc:  73.45; ppl:  2.05; xent: 0.72; lr: 0.00023; 40882/37357 tok/s;   3055 sec\n",
      "[2019-01-07 22:45:48,310 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:45:55,550 INFO] Validation perplexity: 2.1445\n",
      "[2019-01-07 22:45:55,552 INFO] Validation accuracy: 72.012\n",
      "[2019-01-07 22:45:58,293 INFO] Step 23950/100000; acc:  73.72; ppl:  2.03; xent: 0.71; lr: 0.00023; 11012/10110 tok/s;   3065 sec\n",
      "[2019-01-07 22:46:01,002 INFO] Step 24000/100000; acc:  73.19; ppl:  2.07; xent: 0.73; lr: 0.00021; 40715/37285 tok/s;   3068 sec\n",
      "[2019-01-07 22:46:01,073 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:46:08,305 INFO] Validation perplexity: 2.06403\n",
      "[2019-01-07 22:46:08,306 INFO] Validation accuracy: 73.6771\n",
      "[2019-01-07 22:46:08,308 INFO] Saving checkpoint CNN_step_24000.pt\n",
      "[2019-01-07 22:46:11,131 INFO] Step 24050/100000; acc:  73.72; ppl:  2.04; xent: 0.71; lr: 0.00021; 10993/9985 tok/s;   3078 sec\n",
      "[2019-01-07 22:46:13,853 INFO] Step 24100/100000; acc:  73.54; ppl:  2.05; xent: 0.72; lr: 0.00021; 40682/37317 tok/s;   3081 sec\n",
      "[2019-01-07 22:46:13,924 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:46:21,161 INFO] Validation perplexity: 2.15705\n",
      "[2019-01-07 22:46:21,162 INFO] Validation accuracy: 72.0717\n",
      "[2019-01-07 22:46:23,874 INFO] Step 24150/100000; acc:  73.78; ppl:  2.03; xent: 0.71; lr: 0.00021; 10687/9783 tok/s;   3091 sec\n",
      "[2019-01-07 22:46:26,569 INFO] Step 24200/100000; acc:  73.95; ppl:  2.03; xent: 0.71; lr: 0.00021; 40377/37320 tok/s;   3093 sec\n",
      "[2019-01-07 22:46:26,640 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:46:33,877 INFO] Validation perplexity: 2.22456\n",
      "[2019-01-07 22:46:33,879 INFO] Validation accuracy: 71.1934\n",
      "[2019-01-07 22:46:36,617 INFO] Step 24250/100000; acc:  73.50; ppl:  2.05; xent: 0.72; lr: 0.00021; 11048/9897 tok/s;   3104 sec\n",
      "[2019-01-07 22:46:39,306 INFO] Step 24300/100000; acc:  73.99; ppl:  2.03; xent: 0.71; lr: 0.00021; 39993/36747 tok/s;   3106 sec\n",
      "[2019-01-07 22:46:39,377 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:46:46,611 INFO] Validation perplexity: 2.12868\n",
      "[2019-01-07 22:46:46,613 INFO] Validation accuracy: 72.2387\n",
      "[2019-01-07 22:46:49,337 INFO] Step 24350/100000; acc:  73.32; ppl:  2.06; xent: 0.72; lr: 0.00021; 11201/10222 tok/s;   3116 sec\n",
      "[2019-01-07 22:46:52,087 INFO] Step 24400/100000; acc:  73.66; ppl:  2.05; xent: 0.72; lr: 0.00021; 41627/38060 tok/s;   3119 sec\n",
      "[2019-01-07 22:46:52,158 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:46:59,390 INFO] Validation perplexity: 2.13252\n",
      "[2019-01-07 22:46:59,392 INFO] Validation accuracy: 72.4334\n",
      "[2019-01-07 22:47:02,154 INFO] Step 24450/100000; acc:  73.50; ppl:  2.06; xent: 0.72; lr: 0.00021; 11107/10158 tok/s;   3129 sec\n",
      "[2019-01-07 22:47:04,849 INFO] Step 24500/100000; acc:  73.75; ppl:  2.03; xent: 0.71; lr: 0.00021; 39465/35993 tok/s;   3132 sec\n",
      "[2019-01-07 22:47:04,920 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:47:12,158 INFO] Validation perplexity: 2.05323\n",
      "[2019-01-07 22:47:12,159 INFO] Validation accuracy: 73.4892\n",
      "[2019-01-07 22:47:14,886 INFO] Step 24550/100000; acc:  73.93; ppl:  2.03; xent: 0.71; lr: 0.00021; 10961/10168 tok/s;   3142 sec\n",
      "[2019-01-07 22:47:17,573 INFO] Step 24600/100000; acc:  73.75; ppl:  2.03; xent: 0.71; lr: 0.00021; 39994/36327 tok/s;   3144 sec\n",
      "[2019-01-07 22:47:17,643 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:47:24,878 INFO] Validation perplexity: 2.13086\n",
      "[2019-01-07 22:47:24,879 INFO] Validation accuracy: 72.2729\n",
      "[2019-01-07 22:47:27,618 INFO] Step 24650/100000; acc:  73.42; ppl:  2.06; xent: 0.72; lr: 0.00021; 10865/9971 tok/s;   3155 sec\n",
      "[2019-01-07 22:47:30,337 INFO] Step 24700/100000; acc:  73.65; ppl:  2.04; xent: 0.71; lr: 0.00021; 40671/37221 tok/s;   3157 sec\n",
      "[2019-01-07 22:47:30,408 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:47:37,648 INFO] Validation perplexity: 2.10596\n",
      "[2019-01-07 22:47:37,650 INFO] Validation accuracy: 72.6129\n",
      "[2019-01-07 22:47:40,377 INFO] Step 24750/100000; acc:  73.61; ppl:  2.05; xent: 0.72; lr: 0.00021; 11111/10055 tok/s;   3167 sec\n",
      "[2019-01-07 22:47:43,074 INFO] Step 24800/100000; acc:  73.53; ppl:  2.06; xent: 0.72; lr: 0.00021; 40844/37584 tok/s;   3170 sec\n",
      "[2019-01-07 22:47:43,145 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:47:50,380 INFO] Validation perplexity: 2.11283\n",
      "[2019-01-07 22:47:50,381 INFO] Validation accuracy: 72.8216\n",
      "[2019-01-07 22:47:53,150 INFO] Step 24850/100000; acc:  73.44; ppl:  2.05; xent: 0.72; lr: 0.00021; 11318/10181 tok/s;   3180 sec\n",
      "[2019-01-07 22:47:55,824 INFO] Step 24900/100000; acc:  73.65; ppl:  2.04; xent: 0.71; lr: 0.00021; 39927/36741 tok/s;   3183 sec\n",
      "[2019-01-07 22:47:55,895 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:48:03,137 INFO] Validation perplexity: 2.07415\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:48:03,138 INFO] Validation accuracy: 73.4798\n",
      "[2019-01-07 22:48:05,835 INFO] Step 24950/100000; acc:  73.58; ppl:  2.04; xent: 0.71; lr: 0.00021; 11448/10509 tok/s;   3193 sec\n",
      "[2019-01-07 22:48:08,511 INFO] Step 25000/100000; acc:  73.39; ppl:  2.06; xent: 0.72; lr: 0.00019; 40768/36832 tok/s;   3195 sec\n",
      "[2019-01-07 22:48:08,582 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:48:15,817 INFO] Validation perplexity: 2.07031\n",
      "[2019-01-07 22:48:15,819 INFO] Validation accuracy: 73.4041\n",
      "[2019-01-07 22:48:15,821 INFO] Saving checkpoint CNN_step_25000.pt\n",
      "[2019-01-07 22:48:18,641 INFO] Step 25050/100000; acc:  73.76; ppl:  2.04; xent: 0.71; lr: 0.00019; 10624/9782 tok/s;   3206 sec\n",
      "[2019-01-07 22:48:21,343 INFO] Step 25100/100000; acc:  73.93; ppl:  2.02; xent: 0.70; lr: 0.00019; 39648/36440 tok/s;   3208 sec\n",
      "[2019-01-07 22:48:21,414 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:48:28,652 INFO] Validation perplexity: 2.13515\n",
      "[2019-01-07 22:48:28,653 INFO] Validation accuracy: 72.5466\n",
      "[2019-01-07 22:48:31,402 INFO] Step 25150/100000; acc:  73.54; ppl:  2.04; xent: 0.71; lr: 0.00019; 11376/10309 tok/s;   3218 sec\n",
      "[2019-01-07 22:48:34,132 INFO] Step 25200/100000; acc:  73.83; ppl:  2.04; xent: 0.71; lr: 0.00019; 41806/38328 tok/s;   3221 sec\n",
      "[2019-01-07 22:48:34,204 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:48:41,447 INFO] Validation perplexity: 2.06667\n",
      "[2019-01-07 22:48:41,448 INFO] Validation accuracy: 73.6766\n",
      "[2019-01-07 22:48:44,147 INFO] Step 25250/100000; acc:  73.90; ppl:  2.03; xent: 0.71; lr: 0.00019; 10392/9578 tok/s;   3231 sec\n",
      "[2019-01-07 22:48:46,850 INFO] Step 25300/100000; acc:  73.75; ppl:  2.04; xent: 0.71; lr: 0.00019; 41783/37997 tok/s;   3234 sec\n",
      "[2019-01-07 22:48:46,921 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:48:54,154 INFO] Validation perplexity: 2.09682\n",
      "[2019-01-07 22:48:54,156 INFO] Validation accuracy: 73.3044\n",
      "[2019-01-07 22:48:56,886 INFO] Step 25350/100000; acc:  73.60; ppl:  2.04; xent: 0.71; lr: 0.00019; 10686/9771 tok/s;   3244 sec\n",
      "[2019-01-07 22:48:59,575 INFO] Step 25400/100000; acc:  74.05; ppl:  2.03; xent: 0.71; lr: 0.00019; 40623/37659 tok/s;   3246 sec\n",
      "[2019-01-07 22:48:59,646 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:49:06,881 INFO] Validation perplexity: 2.08928\n",
      "[2019-01-07 22:49:06,883 INFO] Validation accuracy: 73.1104\n",
      "[2019-01-07 22:49:09,606 INFO] Step 25450/100000; acc:  73.57; ppl:  2.05; xent: 0.72; lr: 0.00019; 11199/10082 tok/s;   3256 sec\n",
      "[2019-01-07 22:49:12,313 INFO] Step 25500/100000; acc:  73.81; ppl:  2.03; xent: 0.71; lr: 0.00019; 40582/37056 tok/s;   3259 sec\n",
      "[2019-01-07 22:49:12,386 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:49:19,623 INFO] Validation perplexity: 2.09945\n",
      "[2019-01-07 22:49:19,624 INFO] Validation accuracy: 72.9169\n",
      "[2019-01-07 22:49:22,329 INFO] Step 25550/100000; acc:  73.84; ppl:  2.03; xent: 0.71; lr: 0.00019; 10963/10016 tok/s;   3269 sec\n",
      "[2019-01-07 22:49:25,027 INFO] Step 25600/100000; acc:  73.82; ppl:  2.03; xent: 0.71; lr: 0.00019; 40012/36329 tok/s;   3272 sec\n",
      "[2019-01-07 22:49:25,100 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:49:32,333 INFO] Validation perplexity: 2.10227\n",
      "[2019-01-07 22:49:32,335 INFO] Validation accuracy: 72.8863\n",
      "[2019-01-07 22:49:35,087 INFO] Step 25650/100000; acc:  73.42; ppl:  2.05; xent: 0.72; lr: 0.00019; 11106/10266 tok/s;   3282 sec\n",
      "[2019-01-07 22:49:37,800 INFO] Step 25700/100000; acc:  73.89; ppl:  2.03; xent: 0.71; lr: 0.00019; 40439/37270 tok/s;   3285 sec\n",
      "[2019-01-07 22:49:37,870 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:49:45,110 INFO] Validation perplexity: 2.09938\n",
      "[2019-01-07 22:49:45,111 INFO] Validation accuracy: 72.9828\n",
      "[2019-01-07 22:49:47,850 INFO] Step 25750/100000; acc:  73.76; ppl:  2.03; xent: 0.71; lr: 0.00019; 11177/10148 tok/s;   3295 sec\n",
      "[2019-01-07 22:49:50,552 INFO] Step 25800/100000; acc:  73.98; ppl:  2.01; xent: 0.70; lr: 0.00019; 40371/36915 tok/s;   3297 sec\n",
      "[2019-01-07 22:49:50,622 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:49:57,847 INFO] Validation perplexity: 2.13982\n",
      "[2019-01-07 22:49:57,849 INFO] Validation accuracy: 72.542\n",
      "[2019-01-07 22:50:00,581 INFO] Step 25850/100000; acc:  73.22; ppl:  2.07; xent: 0.73; lr: 0.00019; 10882/9913 tok/s;   3307 sec\n",
      "[2019-01-07 22:50:03,297 INFO] Step 25900/100000; acc:  73.83; ppl:  2.04; xent: 0.71; lr: 0.00019; 41444/37705 tok/s;   3310 sec\n",
      "[2019-01-07 22:50:03,368 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:50:10,594 INFO] Validation perplexity: 2.1762\n",
      "[2019-01-07 22:50:10,596 INFO] Validation accuracy: 71.8705\n",
      "[2019-01-07 22:50:13,299 INFO] Step 25950/100000; acc:  73.99; ppl:  2.02; xent: 0.70; lr: 0.00019; 10596/9718 tok/s;   3320 sec\n",
      "[2019-01-07 22:50:16,024 INFO] Step 26000/100000; acc:  73.66; ppl:  2.04; xent: 0.71; lr: 0.00017; 41360/37765 tok/s;   3323 sec\n",
      "[2019-01-07 22:50:16,095 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:50:23,320 INFO] Validation perplexity: 2.11463\n",
      "[2019-01-07 22:50:23,321 INFO] Validation accuracy: 72.8927\n",
      "[2019-01-07 22:50:23,324 INFO] Saving checkpoint CNN_step_26000.pt\n",
      "[2019-01-07 22:50:26,175 INFO] Step 26050/100000; acc:  73.65; ppl:  2.05; xent: 0.72; lr: 0.00017; 11119/10118 tok/s;   3333 sec\n",
      "[2019-01-07 22:50:28,843 INFO] Step 26100/100000; acc:  74.29; ppl:  2.00; xent: 0.69; lr: 0.00017; 38617/34576 tok/s;   3336 sec\n",
      "[2019-01-07 22:50:28,914 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:50:36,138 INFO] Validation perplexity: 2.106\n",
      "[2019-01-07 22:50:36,139 INFO] Validation accuracy: 72.9173\n",
      "[2019-01-07 22:50:38,868 INFO] Step 26150/100000; acc:  73.88; ppl:  2.04; xent: 0.71; lr: 0.00017; 10965/10320 tok/s;   3346 sec\n",
      "[2019-01-07 22:50:41,581 INFO] Step 26200/100000; acc:  73.89; ppl:  2.02; xent: 0.70; lr: 0.00017; 42361/38276 tok/s;   3348 sec\n",
      "[2019-01-07 22:50:41,653 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:50:48,870 INFO] Validation perplexity: 2.11758\n",
      "[2019-01-07 22:50:48,872 INFO] Validation accuracy: 72.8315\n",
      "[2019-01-07 22:50:51,599 INFO] Step 26250/100000; acc:  73.43; ppl:  2.07; xent: 0.73; lr: 0.00017; 11008/10060 tok/s;   3358 sec\n",
      "[2019-01-07 22:50:54,348 INFO] Step 26300/100000; acc:  73.61; ppl:  2.04; xent: 0.71; lr: 0.00017; 42262/39067 tok/s;   3361 sec\n",
      "[2019-01-07 22:50:54,419 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:51:01,644 INFO] Validation perplexity: 2.11364\n",
      "[2019-01-07 22:51:01,645 INFO] Validation accuracy: 72.8561\n",
      "[2019-01-07 22:51:04,352 INFO] Step 26350/100000; acc:  73.97; ppl:  2.03; xent: 0.71; lr: 0.00017; 10508/9412 tok/s;   3371 sec\n",
      "[2019-01-07 22:51:07,064 INFO] Step 26400/100000; acc:  74.08; ppl:  2.02; xent: 0.70; lr: 0.00017; 40198/36818 tok/s;   3374 sec\n",
      "[2019-01-07 22:51:07,136 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:51:14,368 INFO] Validation perplexity: 2.13705\n",
      "[2019-01-07 22:51:14,370 INFO] Validation accuracy: 72.5243\n",
      "[2019-01-07 22:51:17,070 INFO] Step 26450/100000; acc:  73.61; ppl:  2.04; xent: 0.71; lr: 0.00017; 10889/10012 tok/s;   3384 sec\n",
      "[2019-01-07 22:51:19,777 INFO] Step 26500/100000; acc:  73.54; ppl:  2.04; xent: 0.71; lr: 0.00017; 40796/36966 tok/s;   3387 sec\n",
      "[2019-01-07 22:51:19,847 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:51:27,074 INFO] Validation perplexity: 2.1067\n",
      "[2019-01-07 22:51:27,076 INFO] Validation accuracy: 72.7905\n",
      "[2019-01-07 22:51:29,799 INFO] Step 26550/100000; acc:  74.02; ppl:  2.03; xent: 0.71; lr: 0.00017; 11071/10335 tok/s;   3397 sec\n",
      "[2019-01-07 22:51:32,479 INFO] Step 26600/100000; acc:  73.93; ppl:  2.02; xent: 0.71; lr: 0.00017; 40227/36712 tok/s;   3399 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2019-01-07 22:51:32,552 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:51:39,777 INFO] Validation perplexity: 2.14841\n",
      "[2019-01-07 22:51:39,779 INFO] Validation accuracy: 72.3328\n",
      "[2019-01-07 22:51:42,518 INFO] Step 26650/100000; acc:  73.45; ppl:  2.05; xent: 0.72; lr: 0.00017; 10883/9906 tok/s;   3409 sec\n",
      "[2019-01-07 22:51:45,279 INFO] Step 26700/100000; acc:  73.58; ppl:  2.05; xent: 0.72; lr: 0.00017; 40984/37635 tok/s;   3412 sec\n",
      "[2019-01-07 22:51:45,350 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n",
      "[2019-01-07 22:51:52,602 INFO] Validation perplexity: 2.12674\n",
      "[2019-01-07 22:51:52,603 INFO] Validation accuracy: 72.5092\n",
      "[2019-01-07 22:51:54,784 INFO] Loading train dataset from data/USP.train.0.pt, number of examples: 189902\n",
      "[2019-01-07 22:51:57,034 INFO] Step 26750/100000; acc:  74.44; ppl:  2.00; xent: 0.69; lr: 0.00017; 8892/8212 tok/s;   3424 sec\n",
      "[2019-01-07 22:51:59,730 INFO] Step 26800/100000; acc:  73.72; ppl:  2.03; xent: 0.71; lr: 0.00017; 41110/36860 tok/s;   3427 sec\n",
      "[2019-01-07 22:51:59,802 INFO] Loading valid dataset from data/USP.valid.0.pt, number of examples: 29551\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-fdf8e3204b56>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0msave_checkpoint_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m           )\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0msingle_main\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/train_single.py\u001b[0m in \u001b[0;36mmain\u001b[0;34m(opt, device_id)\u001b[0m\n\u001b[1;32m    143\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Starting training on CPU, could be very slow'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m     trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,\n\u001b[0;32m--> 145\u001b[0;31m                   opt.valid_steps)\n\u001b[0m\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensorboard\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, train_iter_fct, valid_iter_fct, train_steps, valid_steps)\u001b[0m\n\u001b[1;32m    190\u001b[0m                                             % (self.gpu_rank, step))\n\u001b[1;32m    191\u001b[0m                             \u001b[0mvalid_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalid_iter_fct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 192\u001b[0;31m                             \u001b[0mvalid_stats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_iter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    193\u001b[0m                             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgpu_verbose_level\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m                                 logger.info('GpuRank %d: gather valid stat \\\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/trainer.py\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(self, valid_iter)\u001b[0m\n\u001b[1;32m    236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m             \u001b[0;31m# F-prop through the model.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 238\u001b[0;31m             \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_lengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    240\u001b[0m             \u001b[0;31m# Compute loss.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/models/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, src, tgt, lengths)\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0mtgt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtgt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m  \u001b[0;31m# exclude last target from inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m         \u001b[0menc_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory_bank\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory_bank\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menc_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         dec_out, attns = self.decoder(tgt, memory_bank,\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/encoders/transformer.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, src, lengths)\u001b[0m\n\u001b[1;32m    110\u001b[0m         \u001b[0;31m# Run the forward pass of every layer of the tranformer.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/encoders/transformer.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs, mask)\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0minput_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         context, _ = self.self_attn(input_norm, input_norm, input_norm,\n\u001b[0;32m---> 50\u001b[0;31m                                     mask=mask)\n\u001b[0m\u001b[1;32m     51\u001b[0m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeed_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/OpenNMT_py-0.5.0-py3.6.egg/onmt/modules/multi_headed_attn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, key, value, query, mask, layer_cache, type)\u001b[0m\n\u001b[1;32m    186\u001b[0m         \u001b[0mcontext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdrop_attn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinal_linear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m         \u001b[0;31m# CHECK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;31m# batch_, q_len_, d_ = output.size()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/sci/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1352\u001b[0m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unwrap_optional\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1354\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1355\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1356\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unwrap_optional\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "from onmt.train_single import main as single_main\n",
    "from onmt.new_opts import Opts, PreprocessorOpts\n",
    "from onmt.preprocess import build_save_dataset, build_save_vocab\n",
    "import onmt.inputters as inputters\n",
    "\n",
    "opt = Opts(\n",
    "        encoder_type=\"transformer\", \n",
    "        decoder_type=\"transformer\",\n",
    "        layers=4,\n",
    "        norma\n",
    "        heads=4, \n",
    "        rnn_size=256, \n",
    "        word_vec_size=256, \n",
    "        transformer_ff=256,\n",
    "        data=\"data/USP\", \n",
    "        save_model=\"CNN\", \n",
    "        epochs=10, \n",
    "        batch_size=64, \n",
    "        gpu_ranks=[0], \n",
    "        valid_steps=100, \n",
    "        valid_batch_size=64, \n",
    "        decay_steps=1000, \n",
    "        start_decay_steps=10000, \n",
    "        learning_rate_decay=0.9, \n",
    "        optim=\"adam\",\n",
    "        learning_rate=0.001,\n",
    "        save_checkpoint_steps=1000,\n",
    "        \n",
    "          )\n",
    "single_main(opt, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/phillnik/miniconda3/envs/sci/lib/python3.6/site-packages/torchtext/data/field.py:323: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  var = torch.tensor(arr, dtype=self.dtype, device=device)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRED AVG SCORE: -0.0198, PRED PPL: 1.0200\n",
      "GOLD AVG SCORE: -0.3363, GOLD PPL: 1.3997\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([[tensor(-0.8770)],\n",
       "  [tensor(-0.0124)],\n",
       "  [tensor(-0.2436)],\n",
       "  [tensor(-1.5064)],\n",
       "  [tensor(-0.1061)],\n",
       "  [tensor(-0.3343)],\n",
       "  [tensor(-0.0057)],\n",
       "  [tensor(-2.0630)],\n",
       "  [tensor(-0.9585)],\n",
       "  [tensor(-0.3555)],\n",
       "  [tensor(-0.0467)],\n",
       "  [tensor(-0.9754)],\n",
       "  [tensor(-0.5869)],\n",
       "  [tensor(-1.2384)],\n",
       "  [tensor(-0.3307)],\n",
       "  [tensor(-0.1737)],\n",
       "  [tensor(-0.0034)],\n",
       "  [tensor(-0.1133)],\n",
       "  [tensor(-0.3574)],\n",
       "  [tensor(-1.6835)],\n",
       "  [tensor(-0.5869)],\n",
       "  [tensor(-0.1965)],\n",
       "  [tensor(-0.0340)],\n",
       "  [tensor(-0.0652)],\n",
       "  [tensor(-1.0951)],\n",
       "  [tensor(-0.7087)],\n",
       "  [tensor(-0.0974)],\n",
       "  [tensor(-0.0437)],\n",
       "  [tensor(-0.2391)],\n",
       "  [tensor(-0.1346)],\n",
       "  [tensor(-0.0152)],\n",
       "  [tensor(-0.0012)],\n",
       "  [tensor(-1.9566)],\n",
       "  [tensor(-1.9566)],\n",
       "  [tensor(-0.5214)],\n",
       "  [tensor(-0.9882)],\n",
       "  [tensor(-0.9221)],\n",
       "  [tensor(-0.5694)],\n",
       "  [tensor(-0.5874)],\n",
       "  [tensor(-0.7630)],\n",
       "  [tensor(-0.5233)],\n",
       "  [tensor(-0.5294)],\n",
       "  [tensor(-0.4749)],\n",
       "  [tensor(-0.0228)],\n",
       "  [tensor(-0.5563)],\n",
       "  [tensor(-0.1259)],\n",
       "  [tensor(-0.0519)],\n",
       "  [tensor(-1.5326)],\n",
       "  [tensor(-0.0353)],\n",
       "  [tensor(-0.2980)],\n",
       "  [tensor(-0.2326)],\n",
       "  [tensor(-0.6305)],\n",
       "  [tensor(-0.0445)],\n",
       "  [tensor(-1.2102)],\n",
       "  [tensor(-0.8238)],\n",
       "  [tensor(-0.5933)],\n",
       "  [tensor(-0.5933)],\n",
       "  [tensor(-0.7847)],\n",
       "  [tensor(-0.8440)],\n",
       "  [tensor(-0.0533)],\n",
       "  [tensor(-0.7734)],\n",
       "  [tensor(-0.2096)],\n",
       "  [tensor(-1.2527)],\n",
       "  [tensor(-2.3400)],\n",
       "  [tensor(-0.8839)],\n",
       "  [tensor(-0.6162)],\n",
       "  [tensor(-1.2497)],\n",
       "  [tensor(-0.2536)],\n",
       "  [tensor(-0.0119)],\n",
       "  [tensor(-0.0447)],\n",
       "  [tensor(-0.8695)],\n",
       "  [tensor(-0.2274)],\n",
       "  [tensor(-0.8480)],\n",
       "  [tensor(-1.0896)],\n",
       "  [tensor(-0.1367)],\n",
       "  [tensor(-0.1961)],\n",
       "  [tensor(-1.5336)],\n",
       "  [tensor(-0.4139)],\n",
       "  [tensor(-0.0033)],\n",
       "  [tensor(-0.4159)],\n",
       "  [tensor(-0.9424)],\n",
       "  [tensor(-0.9630)],\n",
       "  [tensor(-0.5679)],\n",
       "  [tensor(-0.8479)],\n",
       "  [tensor(-0.5690)],\n",
       "  [tensor(-0.1736)],\n",
       "  [tensor(-0.2563)],\n",
       "  [tensor(-0.3292)],\n",
       "  [tensor(-0.0104)],\n",
       "  [tensor(-1.6606)],\n",
       "  [tensor(-0.1095)],\n",
       "  [tensor(-0.4749)],\n",
       "  [tensor(-0.0086)],\n",
       "  [tensor(-0.1419)],\n",
       "  [tensor(-0.5784)],\n",
       "  [tensor(-0.0113)],\n",
       "  [tensor(-0.9117)],\n",
       "  [tensor(-0.6308)],\n",
       "  [tensor(-0.4106)],\n",
       "  [tensor(-0.1518)],\n",
       "  [tensor(-0.8151)],\n",
       "  [tensor(-0.4100)],\n",
       "  [tensor(-1.3021)],\n",
       "  [tensor(-0.2937)],\n",
       "  [tensor(-1.2687)],\n",
       "  [tensor(-0.3633)],\n",
       "  [tensor(-0.5854)],\n",
       "  [tensor(-0.0182)],\n",
       "  [tensor(-0.1173)],\n",
       "  [tensor(-0.3453)],\n",
       "  [tensor(-0.3670)],\n",
       "  [tensor(-0.6202)],\n",
       "  [tensor(-0.0418)],\n",
       "  [tensor(-0.0094)],\n",
       "  [tensor(-0.9065)],\n",
       "  [tensor(-0.8625)],\n",
       "  [tensor(-0.0218)],\n",
       "  [tensor(-0.4943)],\n",
       "  [tensor(-0.7424)],\n",
       "  [tensor(-1.1972)],\n",
       "  [tensor(-0.0007)],\n",
       "  [tensor(-0.0303)],\n",
       "  [tensor(-0.0102)],\n",
       "  [tensor(-0.0755)],\n",
       "  [tensor(-0.5524)],\n",
       "  [tensor(-1.2595)],\n",
       "  [tensor(-1.3196)],\n",
       "  [tensor(-0.5690)],\n",
       "  [tensor(-0.5231)],\n",
       "  [tensor(-0.1687)],\n",
       "  [tensor(-1.4273)],\n",
       "  [tensor(-0.3784)],\n",
       "  [tensor(-1.1196)],\n",
       "  [tensor(-0.0040)],\n",
       "  [tensor(-0.0027)],\n",
       "  [tensor(-0.0199)],\n",
       "  [tensor(-0.2953)],\n",
       "  [tensor(-0.8647)],\n",
       "  [tensor(-0.0024)],\n",
       "  [tensor(-0.0013)],\n",
       "  [tensor(-1.9910)],\n",
       "  [tensor(-0.5886)],\n",
       "  [tensor(-0.8602)],\n",
       "  [tensor(-2.7079)],\n",
       "  [tensor(-0.1589)],\n",
       "  [tensor(-0.0348)],\n",
       "  [tensor(-0.1397)],\n",
       "  [tensor(-0.6136)],\n",
       "  [tensor(-0.1663)],\n",
       "  [tensor(-0.4030)],\n",
       "  [tensor(-0.7147)],\n",
       "  [tensor(-0.4577)],\n",
       "  [tensor(-0.0325)],\n",
       "  [tensor(-0.0004)],\n",
       "  [tensor(-0.0094)],\n",
       "  [tensor(-1.0292)],\n",
       "  [tensor(-0.0391)],\n",
       "  [tensor(-0.0391)],\n",
       "  [tensor(-0.5421)],\n",
       "  [tensor(-0.5855)],\n",
       "  [tensor(-0.0279)],\n",
       "  [tensor(-0.1495)],\n",
       "  [tensor(-0.0925)],\n",
       "  [tensor(-1.3288)],\n",
       "  [tensor(-1.2538)],\n",
       "  [tensor(-1.8597)],\n",
       "  [tensor(-0.0928)],\n",
       "  [tensor(-0.2013)],\n",
       "  [tensor(-0.2686)],\n",
       "  [tensor(-0.5193)],\n",
       "  [tensor(-0.6252)],\n",
       "  [tensor(-0.4946)],\n",
       "  [tensor(-0.0005)],\n",
       "  [tensor(-0.1837)],\n",
       "  [tensor(-0.2526)],\n",
       "  [tensor(-1.4405)],\n",
       "  [tensor(-0.0015)],\n",
       "  [tensor(-0.1837)],\n",
       "  [tensor(-1.3270)],\n",
       "  [tensor(-0.6444)],\n",
       "  [tensor(-0.2344)],\n",
       "  [tensor(-0.9049)],\n",
       "  [tensor(-0.6495)],\n",
       "  [tensor(-0.8297)],\n",
       "  [tensor(-0.7018)],\n",
       "  [tensor(-0.8120)],\n",
       "  [tensor(-0.0243)],\n",
       "  [tensor(-0.5723)],\n",
       "  [tensor(-0.5723)],\n",
       "  [tensor(-0.1088)],\n",
       "  [tensor(-0.5202)],\n",
       "  [tensor(-0.5202)],\n",
       "  [tensor(-0.1642)],\n",
       "  [tensor(-0.0459)],\n",
       "  [tensor(-0.1656)],\n",
       "  [tensor(-0.9419)],\n",
       "  [tensor(-2.3525)],\n",
       "  [tensor(-1.2820)],\n",
       "  [tensor(-0.0982)],\n",
       "  [tensor(-1.6380)],\n",
       "  [tensor(-0.0452)],\n",
       "  [tensor(-0.0331)],\n",
       "  [tensor(-0.6572)],\n",
       "  [tensor(-0.7686)],\n",
       "  [tensor(-0.0548)],\n",
       "  [tensor(-0.0266)],\n",
       "  [tensor(-0.0118)],\n",
       "  [tensor(-0.7887)],\n",
       "  [tensor(-0.0941)],\n",
       "  [tensor(-0.9431)],\n",
       "  [tensor(-0.2166)],\n",
       "  [tensor(-0.9328)],\n",
       "  [tensor(-2.0528)],\n",
       "  [tensor(-2.8309)],\n",
       "  [tensor(-1.1820)],\n",
       "  [tensor(-1.4363)],\n",
       "  [tensor(-0.4330)],\n",
       "  [tensor(-0.3975)],\n",
       "  [tensor(-0.4388)],\n",
       "  [tensor(-0.0038)],\n",
       "  [tensor(-1.5374)],\n",
       "  [tensor(-0.6290)],\n",
       "  [tensor(-0.0556)],\n",
       "  [tensor(-0.9037)],\n",
       "  [tensor(-0.6532)],\n",
       "  [tensor(-0.1445)],\n",
       "  [tensor(-0.3536)],\n",
       "  [tensor(-0.9982)],\n",
       "  [tensor(-0.9982)],\n",
       "  [tensor(-1.4327)],\n",
       "  [tensor(-0.5410)],\n",
       "  [tensor(-0.0428)],\n",
       "  [tensor(-0.3466)],\n",
       "  [tensor(-0.0434)],\n",
       "  [tensor(-0.0048)],\n",
       "  [tensor(-0.0111)],\n",
       "  [tensor(-0.0223)],\n",
       "  [tensor(-0.0651)],\n",
       "  [tensor(-0.6093)],\n",
       "  [tensor(-1.3188)],\n",
       "  [tensor(-0.0070)],\n",
       "  [tensor(-2.1813)],\n",
       "  [tensor(-3.0988)],\n",
       "  [tensor(-1.8800)],\n",
       "  [tensor(-1.3327)],\n",
       "  [tensor(-0.7434)],\n",
       "  [tensor(-0.1594)],\n",
       "  [tensor(-0.0120)],\n",
       "  [tensor(-0.4100)],\n",
       "  [tensor(-1.0570)],\n",
       "  [tensor(-0.0789)],\n",
       "  [tensor(-0.0052)],\n",
       "  [tensor(-0.9762)],\n",
       "  [tensor(-0.0110)],\n",
       "  [tensor(-0.6718)],\n",
       "  [tensor(-1.6071)],\n",
       "  [tensor(-0.0546)],\n",
       "  [tensor(-0.5473)],\n",
       "  [tensor(-0.4371)],\n",
       "  [tensor(-0.4371)],\n",
       "  [tensor(-0.3093)],\n",
       "  [tensor(-0.5091)],\n",
       "  [tensor(-1.0139)],\n",
       "  [tensor(-0.0713)],\n",
       "  [tensor(-0.6237)],\n",
       "  [tensor(-0.0030)],\n",
       "  [tensor(-0.0979)],\n",
       "  [tensor(-0.4100)],\n",
       "  [tensor(-0.0364)],\n",
       "  [tensor(-0.9205)],\n",
       "  [tensor(-0.8833)],\n",
       "  [tensor(-0.1113)],\n",
       "  [tensor(-0.2865)],\n",
       "  [tensor(-0.0031)],\n",
       "  [tensor(-1.0039)],\n",
       "  [tensor(-0.0503)],\n",
       "  [tensor(-1.4489)],\n",
       "  [tensor(-1.1564)],\n",
       "  [tensor(-0.0041)],\n",
       "  [tensor(-0.0231)],\n",
       "  [tensor(-0.1162)],\n",
       "  [tensor(-0.0142)],\n",
       "  [tensor(-0.0732)],\n",
       "  [tensor(-0.5094)],\n",
       "  [tensor(-0.0042)],\n",
       "  [tensor(-1.5215)],\n",
       "  [tensor(-0.0302)],\n",
       "  [tensor(-0.1469)],\n",
       "  [tensor(-0.0090)],\n",
       "  [tensor(-0.2608)],\n",
       "  [tensor(-0.3078)],\n",
       "  [tensor(-0.0616)],\n",
       "  [tensor(-0.0017)],\n",
       "  [tensor(-1.0614)],\n",
       "  [tensor(-1.2097)],\n",
       "  [tensor(-1.8016)],\n",
       "  [tensor(-0.7106)],\n",
       "  [tensor(-1.6253)],\n",
       "  [tensor(-1.7875)],\n",
       "  [tensor(-1.4346)],\n",
       "  [tensor(-0.4098)],\n",
       "  [tensor(-1.0584)],\n",
       "  [tensor(-0.6469)],\n",
       "  [tensor(-1.0269)],\n",
       "  [tensor(-1.2794)],\n",
       "  [tensor(-1.2794)],\n",
       "  [tensor(-0.1190)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-2.5614)],\n",
       "  [tensor(-0.1003)],\n",
       "  [tensor(-0.0373)],\n",
       "  [tensor(-0.3973)],\n",
       "  [tensor(-0.0139)],\n",
       "  [tensor(-2.7079)],\n",
       "  [tensor(-0.1857)],\n",
       "  [tensor(-1.1995)],\n",
       "  [tensor(-1.1288)],\n",
       "  [tensor(-0.4787)],\n",
       "  [tensor(-3.1224)],\n",
       "  [tensor(-0.0704)],\n",
       "  [tensor(-0.7782)],\n",
       "  [tensor(-0.3362)],\n",
       "  [tensor(-0.8667)],\n",
       "  [tensor(-0.2189)],\n",
       "  [tensor(-0.6652)],\n",
       "  [tensor(-0.0355)],\n",
       "  [tensor(-0.0956)],\n",
       "  [tensor(-0.3578)],\n",
       "  [tensor(-0.2806)],\n",
       "  [tensor(-0.2806)],\n",
       "  [tensor(-0.8293)],\n",
       "  [tensor(-1.1890)],\n",
       "  [tensor(-0.0393)],\n",
       "  [tensor(-1.7009)],\n",
       "  [tensor(-0.0087)],\n",
       "  [tensor(-1.8543)],\n",
       "  [tensor(-2.2457)],\n",
       "  [tensor(-2.5102)],\n",
       "  [tensor(-0.0610)],\n",
       "  [tensor(-0.7204)],\n",
       "  [tensor(-1.0853)],\n",
       "  [tensor(-0.2086)],\n",
       "  [tensor(-0.0201)],\n",
       "  [tensor(-0.4319)],\n",
       "  [tensor(-0.6836)],\n",
       "  [tensor(-0.1270)],\n",
       "  [tensor(-1.8840)],\n",
       "  [tensor(-0.4101)],\n",
       "  [tensor(-0.0029)],\n",
       "  [tensor(-0.0029)],\n",
       "  [tensor(-0.0613)],\n",
       "  [tensor(-0.2685)],\n",
       "  [tensor(-1.6728)],\n",
       "  [tensor(-0.5760)],\n",
       "  [tensor(-0.7619)],\n",
       "  [tensor(-1.6254)],\n",
       "  [tensor(-1.1640)],\n",
       "  [tensor(-0.2541)],\n",
       "  [tensor(-0.5280)],\n",
       "  [tensor(-0.7705)],\n",
       "  [tensor(-1.3748)],\n",
       "  [tensor(-0.0055)],\n",
       "  [tensor(-0.8625)],\n",
       "  [tensor(-1.1821)],\n",
       "  [tensor(-0.9947)],\n",
       "  [tensor(-0.0015)],\n",
       "  [tensor(-2.1507)],\n",
       "  [tensor(-0.5958)],\n",
       "  [tensor(-0.8238)],\n",
       "  [tensor(-0.2511)],\n",
       "  [tensor(-0.7453)],\n",
       "  [tensor(-0.2132)],\n",
       "  [tensor(-0.5299)],\n",
       "  [tensor(-0.0051)],\n",
       "  [tensor(-1.2703)],\n",
       "  [tensor(-0.0719)],\n",
       "  [tensor(-0.0259)],\n",
       "  [tensor(-1.2207)],\n",
       "  [tensor(-0.8236)],\n",
       "  [tensor(-0.1290)],\n",
       "  [tensor(-0.1397)],\n",
       "  [tensor(-0.5505)],\n",
       "  [tensor(-0.0262)],\n",
       "  [tensor(-0.0179)],\n",
       "  [tensor(-0.5938)],\n",
       "  [tensor(-0.4358)],\n",
       "  [tensor(-0.9237)],\n",
       "  [tensor(-0.9237)],\n",
       "  [tensor(-0.8594)],\n",
       "  [tensor(-1.6631)],\n",
       "  [tensor(-3.6518)],\n",
       "  [tensor(-0.6388)],\n",
       "  [tensor(-0.1255)],\n",
       "  [tensor(-0.0208)],\n",
       "  [tensor(-0.3480)],\n",
       "  [tensor(-1.7202)],\n",
       "  [tensor(-1.2799)],\n",
       "  [tensor(-0.0304)],\n",
       "  [tensor(-1.4378)],\n",
       "  [tensor(-0.5906)],\n",
       "  [tensor(-0.9406)],\n",
       "  [tensor(-0.2065)],\n",
       "  [tensor(-0.2061)],\n",
       "  [tensor(-2.7534)],\n",
       "  [tensor(-0.0217)],\n",
       "  [tensor(-1.6044)],\n",
       "  [tensor(-1.0028)],\n",
       "  [tensor(-0.7851)],\n",
       "  [tensor(-0.4834)],\n",
       "  [tensor(-0.7632)],\n",
       "  [tensor(-0.0635)],\n",
       "  [tensor(-0.0595)],\n",
       "  [tensor(-0.0320)],\n",
       "  [tensor(-0.6699)],\n",
       "  [tensor(-0.9798)],\n",
       "  [tensor(-1.1972)],\n",
       "  [tensor(-0.6909)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.0800)],\n",
       "  [tensor(-0.2138)],\n",
       "  [tensor(-0.1705)],\n",
       "  [tensor(-0.0877)],\n",
       "  [tensor(-0.5971)],\n",
       "  [tensor(-0.0418)],\n",
       "  [tensor(-2.1949)],\n",
       "  [tensor(-0.0393)],\n",
       "  [tensor(-1.3372)],\n",
       "  [tensor(-0.0233)],\n",
       "  [tensor(-1.0382)],\n",
       "  [tensor(-1.0382)],\n",
       "  [tensor(-0.0968)],\n",
       "  [tensor(-0.9356)],\n",
       "  [tensor(-0.5076)],\n",
       "  [tensor(-0.1410)],\n",
       "  [tensor(-1.5557)],\n",
       "  [tensor(-1.7359)],\n",
       "  [tensor(-1.1561)],\n",
       "  [tensor(-2.1774)],\n",
       "  [tensor(-0.2188)],\n",
       "  [tensor(-0.8488)],\n",
       "  [tensor(-0.8488)],\n",
       "  [tensor(-0.8488)],\n",
       "  [tensor(-0.8488)],\n",
       "  [tensor(-0.8488)],\n",
       "  [tensor(-0.0013)],\n",
       "  [tensor(-1.5444)],\n",
       "  [tensor(-1.0654)],\n",
       "  [tensor(-0.5840)],\n",
       "  [tensor(-0.8493)],\n",
       "  [tensor(-0.2132)],\n",
       "  [tensor(-0.0998)],\n",
       "  [tensor(-0.5151)],\n",
       "  [tensor(-1.0691)],\n",
       "  [tensor(-0.7416)],\n",
       "  [tensor(-0.8370)],\n",
       "  [tensor(-0.1396)],\n",
       "  [tensor(-0.3576)],\n",
       "  [tensor(-0.2044)],\n",
       "  [tensor(-0.0247)],\n",
       "  [tensor(-1.1169)],\n",
       "  [tensor(-0.2098)],\n",
       "  [tensor(-0.2377)],\n",
       "  [tensor(-0.4573)],\n",
       "  [tensor(-0.2813)],\n",
       "  [tensor(-0.2158)],\n",
       "  [tensor(-0.0251)],\n",
       "  [tensor(-0.9499)],\n",
       "  [tensor(-0.0082)],\n",
       "  [tensor(-0.0371)],\n",
       "  [tensor(-0.3649)],\n",
       "  [tensor(-0.2891)],\n",
       "  [tensor(-0.2697)],\n",
       "  [tensor(-0.3290)],\n",
       "  [tensor(-0.1027)],\n",
       "  [tensor(-0.3478)],\n",
       "  [tensor(-0.2932)],\n",
       "  [tensor(-0.5088)],\n",
       "  [tensor(-0.0040)],\n",
       "  [tensor(-0.6715)],\n",
       "  [tensor(-0.0257)],\n",
       "  [tensor(-0.0083)],\n",
       "  [tensor(-0.1151)],\n",
       "  [tensor(-0.0850)],\n",
       "  [tensor(-0.0074)],\n",
       "  [tensor(-1.8065)],\n",
       "  [tensor(-0.0368)],\n",
       "  [tensor(-2.8234)],\n",
       "  [tensor(-1.3672)],\n",
       "  [tensor(-0.0556)],\n",
       "  [tensor(-0.1410)],\n",
       "  [tensor(-1.9071)],\n",
       "  [tensor(-0.5870)],\n",
       "  [tensor(-0.0036)],\n",
       "  [tensor(-0.0025)],\n",
       "  [tensor(-1.6029)],\n",
       "  [tensor(-0.3935)],\n",
       "  [tensor(-0.0106)],\n",
       "  [tensor(-0.0098)],\n",
       "  [tensor(-0.2654)],\n",
       "  [tensor(-0.9905)],\n",
       "  [tensor(-0.7267)],\n",
       "  [tensor(-1.7804)],\n",
       "  [tensor(-1.0945)],\n",
       "  [tensor(-0.8237)],\n",
       "  [tensor(-0.0075)],\n",
       "  [tensor(-0.9664)],\n",
       "  [tensor(-0.0034)],\n",
       "  [tensor(-2.1397)],\n",
       "  [tensor(-1.0936)],\n",
       "  [tensor(-0.0150)],\n",
       "  [tensor(-0.8057)],\n",
       "  [tensor(-0.9186)],\n",
       "  [tensor(-0.4518)],\n",
       "  [tensor(-0.1669)],\n",
       "  [tensor(-0.1669)],\n",
       "  [tensor(-1.3645)],\n",
       "  [tensor(-0.0007)],\n",
       "  [tensor(-0.0055)],\n",
       "  [tensor(-2.3255)],\n",
       "  [tensor(-0.3160)],\n",
       "  [tensor(-0.0917)],\n",
       "  [tensor(-0.0580)],\n",
       "  [tensor(-0.0092)],\n",
       "  [tensor(-0.8849)],\n",
       "  [tensor(-0.0132)],\n",
       "  [tensor(-0.1587)],\n",
       "  [tensor(-0.2519)],\n",
       "  [tensor(-0.9935)],\n",
       "  [tensor(-1.3452)],\n",
       "  [tensor(-0.0189)],\n",
       "  [tensor(-0.4987)],\n",
       "  [tensor(-0.1453)],\n",
       "  [tensor(-0.1399)],\n",
       "  [tensor(-1.0971)],\n",
       "  [tensor(-2.1947)],\n",
       "  [tensor(-0.5885)],\n",
       "  [tensor(-1.1435)],\n",
       "  [tensor(-0.0328)],\n",
       "  [tensor(-0.0035)],\n",
       "  [tensor(-0.1619)],\n",
       "  [tensor(-0.2750)],\n",
       "  [tensor(-0.9004)],\n",
       "  [tensor(-0.5587)],\n",
       "  [tensor(-1.2240)],\n",
       "  [tensor(-0.0029)],\n",
       "  [tensor(-0.0356)],\n",
       "  [tensor(-0.0011)],\n",
       "  [tensor(-0.2094)],\n",
       "  [tensor(-1.6189)],\n",
       "  [tensor(-1.7902)],\n",
       "  [tensor(-1.7902)],\n",
       "  [tensor(-1.6708)],\n",
       "  [tensor(-1.1095)],\n",
       "  [tensor(-0.4783)],\n",
       "  [tensor(-1.4619)],\n",
       "  [tensor(-0.3615)],\n",
       "  [tensor(-1.5857)],\n",
       "  [tensor(-0.9207)],\n",
       "  [tensor(-0.2755)],\n",
       "  [tensor(-0.2506)],\n",
       "  [tensor(-0.1807)],\n",
       "  [tensor(-2.0932)],\n",
       "  [tensor(-1.2638)],\n",
       "  [tensor(-1.2671)],\n",
       "  [tensor(-0.3091)],\n",
       "  [tensor(-0.2576)],\n",
       "  [tensor(-0.0630)],\n",
       "  [tensor(-0.0212)],\n",
       "  [tensor(-1.7385)],\n",
       "  [tensor(-0.1546)],\n",
       "  [tensor(-0.9834)],\n",
       "  [tensor(-1.1323)],\n",
       "  [tensor(-0.6357)],\n",
       "  [tensor(-0.7486)],\n",
       "  [tensor(-0.0079)],\n",
       "  [tensor(-0.1775)],\n",
       "  [tensor(-0.0033)],\n",
       "  [tensor(-0.7170)],\n",
       "  [tensor(-0.0020)],\n",
       "  [tensor(-0.0013)],\n",
       "  [tensor(-1.1380)],\n",
       "  [tensor(-1.9595)],\n",
       "  [tensor(-2.4257)],\n",
       "  [tensor(-2.2194)],\n",
       "  [tensor(-1.0951)],\n",
       "  [tensor(-1.9470)],\n",
       "  [tensor(-0.0877)],\n",
       "  [tensor(-0.0055)],\n",
       "  [tensor(-0.0150)],\n",
       "  [tensor(-0.0596)],\n",
       "  [tensor(-0.1451)],\n",
       "  [tensor(-1.9182)],\n",
       "  [tensor(-0.0028)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-2.5614)],\n",
       "  [tensor(-2.5614)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-2.5614)],\n",
       "  [tensor(-2.5614)],\n",
       "  [tensor(-2.5614)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-0.7065)],\n",
       "  [tensor(-3.1199)],\n",
       "  [tensor(-0.0038)],\n",
       "  [tensor(-0.2331)],\n",
       "  [tensor(-0.5801)],\n",
       "  [tensor(-0.8759)],\n",
       "  [tensor(-0.4708)],\n",
       "  [tensor(-0.1746)],\n",
       "  [tensor(-0.0258)],\n",
       "  [tensor(-1.3524)],\n",
       "  [tensor(-1.0824)],\n",
       "  [tensor(-0.0185)],\n",
       "  [tensor(-0.3112)],\n",
       "  [tensor(-1.1186)],\n",
       "  [tensor(-0.4760)],\n",
       "  [tensor(-1.6788)],\n",
       "  [tensor(-1.6977)],\n",
       "  [tensor(-0.4371)],\n",
       "  [tensor(-0.0079)],\n",
       "  [tensor(-0.1523)],\n",
       "  [tensor(-1.2850)],\n",
       "  [tensor(-0.1750)],\n",
       "  [tensor(-0.9733)],\n",
       "  [tensor(-1.0297)],\n",
       "  [tensor(-1.2851)],\n",
       "  [tensor(-0.0268)],\n",
       "  [tensor(-0.0036)],\n",
       "  [tensor(-0.0539)],\n",
       "  [tensor(-0.3218)],\n",
       "  [tensor(-0.0105)],\n",
       "  [tensor(-0.0008)],\n",
       "  [tensor(-0.7430)],\n",
       "  [tensor(-0.0241)],\n",
       "  [tensor(-0.1862)],\n",
       "  [tensor(-0.2341)],\n",
       "  [tensor(-0.6612)],\n",
       "  [tensor(-0.0205)],\n",
       "  [tensor(-2.6643)],\n",
       "  [tensor(-0.5443)],\n",
       "  [tensor(-0.0030)],\n",
       "  [tensor(-0.0283)],\n",
       "  [tensor(-0.0997)],\n",
       "  [tensor(-0.0027)],\n",
       "  [tensor(-0.0065)],\n",
       "  [tensor(-0.0028)],\n",
       "  [tensor(-0.1875)],\n",
       "  [tensor(-0.1499)],\n",
       "  [tensor(-0.0239)],\n",
       "  [tensor(-0.2370)],\n",
       "  [tensor(-1.4135)],\n",
       "  [tensor(-0.0377)],\n",
       "  [tensor(-0.2684)],\n",
       "  [tensor(-0.9298)],\n",
       "  [tensor(-0.7777)],\n",
       "  [tensor(-0.8288)],\n",
       "  [tensor(-0.9516)],\n",
       "  [tensor(-0.4865)],\n",
       "  [tensor(-1.9287)],\n",
       "  [tensor(-0.0024)],\n",
       "  [tensor(-0.0258)],\n",
       "  [tensor(-0.8006)],\n",
       "  [tensor(-0.6598)],\n",
       "  [tensor(-0.0332)],\n",
       "  [tensor(-1.3371)],\n",
       "  [tensor(-0.2349)],\n",
       "  [tensor(-0.0223)],\n",
       "  [tensor(-0.0093)],\n",
       "  [tensor(-0.2420)],\n",
       "  [tensor(-0.6346)],\n",
       "  [tensor(-0.0190)],\n",
       "  [tensor(-0.5275)],\n",
       "  [tensor(-0.3855)],\n",
       "  [tensor(-0.4291)],\n",
       "  [tensor(-0.3156)],\n",
       "  [tensor(-0.3107)],\n",
       "  [tensor(-0.5976)],\n",
       "  [tensor(-0.2360)],\n",
       "  [tensor(-0.1349)],\n",
       "  [tensor(-1.2076)],\n",
       "  [tensor(-0.0656)],\n",
       "  [tensor(-0.5877)],\n",
       "  [tensor(-1.4243)],\n",
       "  [tensor(-2.9101)],\n",
       "  [tensor(-0.6776)],\n",
       "  [tensor(-0.7425)],\n",
       "  [tensor(-0.5831)],\n",
       "  [tensor(-0.5893)],\n",
       "  [tensor(-0.0010)],\n",
       "  [tensor(-0.2943)],\n",
       "  [tensor(-0.8167)],\n",
       "  [tensor(-1.3917)],\n",
       "  [tensor(-0.7341)],\n",
       "  [tensor(-0.0271)],\n",
       "  [tensor(-0.0110)],\n",
       "  [tensor(-0.0699)],\n",
       "  [tensor(-0.1674)],\n",
       "  [tensor(-0.3548)],\n",
       "  [tensor(-1.0475)],\n",
       "  [tensor(-0.3466)],\n",
       "  [tensor(-0.3466)],\n",
       "  [tensor(-1.8310)],\n",
       "  [tensor(-1.5155)],\n",
       "  [tensor(-0.7031)],\n",
       "  [tensor(-0.7371)],\n",
       "  [tensor(-0.8327)],\n",
       "  [tensor(-0.3051)],\n",
       "  [tensor(-0.0047)],\n",
       "  [tensor(-0.0152)],\n",
       "  [tensor(-0.6284)],\n",
       "  [tensor(-1.6218)],\n",
       "  [tensor(-1.2686)],\n",
       "  [tensor(-0.6120)],\n",
       "  [tensor(-0.6110)],\n",
       "  [tensor(-0.0178)],\n",
       "  [tensor(-0.0111)],\n",
       "  [tensor(-0.0051)],\n",
       "  [tensor(-0.0029)],\n",
       "  [tensor(-0.0411)],\n",
       "  [tensor(-0.8712)],\n",
       "  [tensor(-0.2854)],\n",
       "  [tensor(-1.3136)],\n",
       "  [tensor(-0.2213)],\n",
       "  [tensor(-0.0274)],\n",
       "  [tensor(-0.0160)],\n",
       "  [tensor(-0.7329)],\n",
       "  [tensor(-0.4492)],\n",
       "  [tensor(-0.0185)],\n",
       "  [tensor(-0.2671)],\n",
       "  [tensor(-0.5347)],\n",
       "  [tensor(-0.3907)],\n",
       "  [tensor(-0.0439)],\n",
       "  [tensor(-0.1255)],\n",
       "  [tensor(-1.1650)],\n",
       "  [tensor(-0.1023)],\n",
       "  [tensor(-0.2472)],\n",
       "  [tensor(-1.7838)],\n",
       "  [tensor(-2.6586)],\n",
       "  [tensor(-0.1289)],\n",
       "  [tensor(-0.1289)],\n",
       "  [tensor(-2.6586)],\n",
       "  [tensor(-0.0094)],\n",
       "  [tensor(-0.6103)],\n",
       "  [tensor(-0.4040)],\n",
       "  [tensor(-0.0058)],\n",
       "  [tensor(-0.1022)],\n",
       "  [tensor(-1.8099)],\n",
       "  [tensor(-0.0182)],\n",
       "  [tensor(-0.0710)],\n",
       "  [tensor(-0.0002)],\n",
       "  [tensor(-0.0014)],\n",
       "  [tensor(-0.3828)],\n",
       "  [tensor(-0.0150)],\n",
       "  [tensor(-0.5464)],\n",
       "  [tensor(-0.3649)],\n",
       "  [tensor(-0.1047)],\n",
       "  [tensor(-0.0232)],\n",
       "  [tensor(-0.0183)],\n",
       "  [tensor(-0.1976)],\n",
       "  [tensor(-0.0925)],\n",
       "  [tensor(-0.8251)],\n",
       "  [tensor(-0.0907)],\n",
       "  [tensor(-0.0338)],\n",
       "  [tensor(-1.8106)],\n",
       "  [tensor(-0.1054)],\n",
       "  [tensor(-0.1484)],\n",
       "  [tensor(-0.0973)],\n",
       "  [tensor(-1.8496)],\n",
       "  [tensor(-0.3556)],\n",
       "  [tensor(-0.1406)],\n",
       "  [tensor(-0.0415)],\n",
       "  [tensor(-0.1479)],\n",
       "  [tensor(-0.0014)],\n",
       "  [tensor(-0.1683)],\n",
       "  [tensor(-1.7152)],\n",
       "  [tensor(-0.4479)],\n",
       "  [tensor(-0.4175)],\n",
       "  [tensor(-0.0098)],\n",
       "  [tensor(-0.0009)],\n",
       "  [tensor(-0.6020)],\n",
       "  [tensor(-0.2679)],\n",
       "  [tensor(-0.4655)],\n",
       "  [tensor(-0.0807)],\n",
       "  [tensor(-0.0940)],\n",
       "  [tensor(-0.0008)],\n",
       "  [tensor(-0.0119)],\n",
       "  [tensor(-1.0272)],\n",
       "  [tensor(-0.0501)],\n",
       "  [tensor(-0.3409)],\n",
       "  [tensor(-0.1495)],\n",
       "  [tensor(-0.3490)],\n",
       "  [tensor(-2.1489)],\n",
       "  [tensor(-0.0613)],\n",
       "  [tensor(-0.0274)],\n",
       "  [tensor(-0.0048)],\n",
       "  [tensor(-0.0610)],\n",
       "  [tensor(-0.6870)],\n",
       "  [tensor(-0.8825)],\n",
       "  [tensor(-1.4072)],\n",
       "  [tensor(-1.3885)],\n",
       "  [tensor(-1.3793)],\n",
       "  [tensor(-0.1134)],\n",
       "  [tensor(-1.8377)],\n",
       "  [tensor(-0.8605)],\n",
       "  [tensor(-0.1663)],\n",
       "  [tensor(-0.1107)],\n",
       "  [tensor(-0.9239)],\n",
       "  [tensor(-0.3081)],\n",
       "  [tensor(-0.0201)],\n",
       "  [tensor(-1.4362)],\n",
       "  [tensor(-0.2005)],\n",
       "  [tensor(-1.8144)],\n",
       "  [tensor(-0.1117)],\n",
       "  [tensor(-1.9411)],\n",
       "  [tensor(-0.0060)],\n",
       "  [tensor(-0.0050)],\n",
       "  [tensor(-1.2102)],\n",
       "  [tensor(-0.0043)],\n",
       "  [tensor(-0.0147)],\n",
       "  [tensor(-0.2665)],\n",
       "  [tensor(-0.5595)],\n",
       "  [tensor(-0.4849)],\n",
       "  [tensor(-0.4884)],\n",
       "  [tensor(-1.8469)],\n",
       "  [tensor(-0.0779)],\n",
       "  [tensor(-0.0107)],\n",
       "  [tensor(-0.0226)],\n",
       "  [tensor(-1.1325)],\n",
       "  [tensor(-0.3920)],\n",
       "  [tensor(-0.3304)],\n",
       "  [tensor(-0.1408)],\n",
       "  [tensor(-0.0060)],\n",
       "  [tensor(-0.1225)],\n",
       "  [tensor(-0.0038)],\n",
       "  [tensor(-0.0041)],\n",
       "  [tensor(-0.0104)],\n",
       "  [tensor(-0.0062)],\n",
       "  [tensor(-0.0607)],\n",
       "  [tensor(-0.2331)],\n",
       "  [tensor(-0.0091)],\n",
       "  [tensor(-0.8251)],\n",
       "  [tensor(-1.1459)],\n",
       "  [tensor(-0.5326)],\n",
       "  [tensor(-2.6380)],\n",
       "  [tensor(-0.8251)],\n",
       "  [tensor(-0.5449)],\n",
       "  [tensor(-2.1995)],\n",
       "  [tensor(-0.0279)],\n",
       "  [tensor(-0.2094)],\n",
       "  [tensor(-0.0417)],\n",
       "  [tensor(-0.0189)],\n",
       "  [tensor(-0.9969)],\n",
       "  [tensor(-1.3011)],\n",
       "  [tensor(-0.6427)],\n",
       "  [tensor(-0.2124)],\n",
       "  [tensor(-0.8795)],\n",
       "  [tensor(-0.3841)],\n",
       "  [tensor(-0.0008)],\n",
       "  [tensor(-1.5445)],\n",
       "  [tensor(-0.0171)],\n",
       "  [tensor(-1.5080)],\n",
       "  [tensor(-0.0407)],\n",
       "  [tensor(-0.0013)],\n",
       "  [tensor(-0.0263)],\n",
       "  [tensor(-0.0051)],\n",
       "  [tensor(-0.0172)],\n",
       "  [tensor(-0.0585)],\n",
       "  [tensor(-0.0116)],\n",
       "  [tensor(-0.9971)],\n",
       "  [tensor(-0.0079)],\n",
       "  [tensor(-0.1528)],\n",
       "  [tensor(-0.2651)],\n",
       "  [tensor(-3.7389)],\n",
       "  [tensor(-1.6220)],\n",
       "  [tensor(-0.2811)],\n",
       "  [tensor(-0.0742)],\n",
       "  [tensor(-0.3746)],\n",
       "  [tensor(-0.0051)],\n",
       "  [tensor(-0.5799)],\n",
       "  [tensor(-0.0366)],\n",
       "  [tensor(-0.7705)],\n",
       "  [tensor(-0.0039)],\n",
       "  [tensor(-0.0915)],\n",
       "  [tensor(-0.0054)],\n",
       "  [tensor(-0.0399)],\n",
       "  [tensor(-1.2212)],\n",
       "  [tensor(-0.8569)],\n",
       "  [tensor(-0.8648)],\n",
       "  [tensor(-0.1733)],\n",
       "  [tensor(-1.9787)],\n",
       "  [tensor(-2.5101)],\n",
       "  [tensor(-0.0481)],\n",
       "  [tensor(-1.7565)],\n",
       "  [tensor(-0.3475)],\n",
       "  [tensor(-0.6694)],\n",
       "  [tensor(-0.6089)],\n",
       "  [tensor(-0.6511)],\n",
       "  [tensor(-0.3253)],\n",
       "  [tensor(-0.2611)],\n",
       "  [tensor(-0.5761)],\n",
       "  [tensor(-3.9650)],\n",
       "  [tensor(-0.0394)],\n",
       "  [tensor(-1.5271)],\n",
       "  [tensor(-0.6322)],\n",
       "  [tensor(-0.3023)],\n",
       "  [tensor(-0.1971)],\n",
       "  [tensor(-1.4649)],\n",
       "  [tensor(-0.0270)],\n",
       "  [tensor(-0.3144)],\n",
       "  [tensor(-1.8303)],\n",
       "  [tensor(-0.0287)],\n",
       "  [tensor(-3.1926)],\n",
       "  [tensor(-0.0246)],\n",
       "  [tensor(-1.1805)],\n",
       "  [tensor(-0.6873)],\n",
       "  [tensor(-0.5035)],\n",
       "  [tensor(-1.8026)],\n",
       "  [tensor(-1.4947)],\n",
       "  [tensor(-0.5787)],\n",
       "  [tensor(-1.2060)],\n",
       "  [tensor(-0.3959)],\n",
       "  [tensor(-0.0244)],\n",
       "  [tensor(-1.9623)],\n",
       "  [tensor(-0.5503)],\n",
       "  [tensor(-0.0062)],\n",
       "  [tensor(-1.8324)],\n",
       "  [tensor(-0.1477)],\n",
       "  [tensor(-0.1477)],\n",
       "  [tensor(-0.6790)],\n",
       "  [tensor(-1.4625)],\n",
       "  [tensor(-0.3969)],\n",
       "  [tensor(-0.2218)],\n",
       "  [tensor(-0.5817)],\n",
       "  [tensor(-0.0616)],\n",
       "  [tensor(-0.0616)],\n",
       "  [tensor(-0.4620)],\n",
       "  [tensor(-0.0082)],\n",
       "  [tensor(-1.3829)],\n",
       "  [tensor(-3.0264)],\n",
       "  [tensor(-0.3614)],\n",
       "  [tensor(-0.0272)],\n",
       "  [tensor(-0.3993)],\n",
       "  [tensor(-0.0131)],\n",
       "  [tensor(-1.3594)],\n",
       "  [tensor(-0.5027)],\n",
       "  [tensor(-0.4237)],\n",
       "  [tensor(-0.3878)],\n",
       "  [tensor(-0.0075)],\n",
       "  [tensor(-0.1021)],\n",
       "  [tensor(-0.0257)],\n",
       "  [tensor(-0.6289)],\n",
       "  [tensor(-2.2399)],\n",
       "  [tensor(-0.4222)],\n",
       "  [tensor(-0.0538)],\n",
       "  [tensor(-0.0069)],\n",
       "  [tensor(-0.4580)],\n",
       "  [tensor(-0.2921)],\n",
       "  [tensor(-1.6710)],\n",
       "  [tensor(-1.0879)],\n",
       "  [tensor(-1.3822)],\n",
       "  [tensor(-1.1816)],\n",
       "  [tensor(-0.0979)],\n",
       "  [tensor(-0.3178)],\n",
       "  [tensor(-0.0097)],\n",
       "  [tensor(-0.0415)],\n",
       "  [tensor(-0.1749)],\n",
       "  [tensor(-0.5435)],\n",
       "  [tensor(-1.3594)],\n",
       "  [tensor(-0.5906)],\n",
       "  [tensor(-0.2958)],\n",
       "  [tensor(-0.7501)],\n",
       "  [tensor(-0.0028)],\n",
       "  [tensor(-0.2190)],\n",
       "  [tensor(-0.1218)],\n",
       "  [tensor(-2.4149)],\n",
       "  [tensor(-0.4752)],\n",
       "  [tensor(-2.1200)],\n",
       "  [tensor(-1.8358)],\n",
       "  [tensor(-0.0647)],\n",
       "  [tensor(-0.2977)],\n",
       "  [tensor(-0.0048)],\n",
       "  [tensor(-0.6483)],\n",
       "  [tensor(-0.0826)],\n",
       "  [tensor(-0.0826)],\n",
       "  [tensor(-0.7606)],\n",
       "  [tensor(-0.0307)],\n",
       "  [tensor(-0.1160)],\n",
       "  [tensor(-0.0012)],\n",
       "  [tensor(-0.0386)],\n",
       "  [tensor(-1.1157)],\n",
       "  [tensor(-0.7292)],\n",
       "  [tensor(-0.4004)],\n",
       "  ...],\n",
       " [['C N 1 c 2 c c c c c 2 C C ( = O ) c 2 c ( Br ) c c c c 2 1'],\n",
       "  ['C C ( = O ) N c 1 c c c 2 n c 3 n ( c 2 c 1 ) C C C 3'],\n",
       "  ['C C ( C ) ( C ) N = N C ( C ) ( C C C ( = O ) O ) C ( N ) = O'],\n",
       "  ['C = C C ( C ( = S ) N C C O ) c 1 c c c c ( C ) n 1'],\n",
       "  ['C C ( C ) C ( C ( = S ) N C N 1 C C C C C 1 ) c 1 c c c c n 1'],\n",
       "  ['C C C c 1 n c ( C ) c ( C C C ) n c 1 C'],\n",
       "  ['C O c 1 c c ( O ) n c ( S C ) n 1'],\n",
       "  ['C C C ( O ) C N 1 N = C 2 c 3 c c c ( Cl ) c c 3 C ( = O ) C 2 C 1'],\n",
       "  ['C C ( C C # N ) C ( = O ) c 1 c c c c c 1 O'],\n",
       "  ['C O c 1 c c c c ( O C ) c 1 C C C C ( = O ) O'],\n",
       "  ['C N 1 C C C C C 1 C C c 1 c c c c c 1 N C ( = O ) c 1 c c c ( Br ) c c 1'],\n",
       "  ['O = C ( N c 1 c c c c c 1 C c 1 c c c c n 1 ) c 1 c c c ( O ) c c 1'],\n",
       "  ['O = c 1 c 2 c c ( C Br ) c c c 2 o c 2 n c c c c 1 2'],\n",
       "  ['Cl C c 1 c c 2 c ( c n 1 ) O C c 1 c c c n c 1 - 2'],\n",
       "  ['O = C ( N O ) c 1 c c c 2 o c 3 n c c c c 3 c ( = O ) c 2 c 1'],\n",
       "  ['C C O C ( = O ) c 1 c c c ( O c 2 n c c c c 2 C = O ) c c 1'],\n",
       "  ['O = C = N c 1 c c c ( O C c 2 n c ( Cl ) c c c 2 Cl ) c c 1'],\n",
       "  ['C C O c 1 n c ( Cl ) n c 2 c s c c 1 2'],\n",
       "  ['C [C@] 1 2 C C [C@H] 3 [C@@H] ( C = C ( N = [N+] = [N-] ) C 4 = C C ( = O ) C C [C@@] 4 3 C ) [C@@H] 1 C C C 2 = O'],\n",
       "  ['C O C 1 = C C 2 = C C [C@H] 3 [C@@H] 4 C C C ( = O ) [C@@] 4 ( C ) C C [C@@H] 3 [C@] 2 ( C ) C C 1'],\n",
       "  ['O = c 1 c 2 c c ( C Br ) c c c 2 o c 2 n c c c c 1 2'],\n",
       "  ['N # C C c 1 c c c 2 o c 3 n c c c c 3 c ( = O ) c 2 c 1'],\n",
       "  ['C C C N 1 N C N ( c 2 n n c ( O C ) s 2 ) C 1 = O'],\n",
       "  ['C N ( N ) C ( = O ) N c 1 n n c ( C 2 C C C C C 2 ) s 1'],\n",
       "  ['O = C 1 N C ( = O ) c 2 c c c c c 2 1'],\n",
       "  ['O = C ( C C 1 C C C N 1 ) c 1 c c c 2 c ( c 1 ) C c 1 c c c c c 1 - 2'],\n",
       "  ['O = C ( O ) c 1 c c c c c 1 C ( = O ) c 1 c [nH] c 2 c c c c c 1 2'],\n",
       "  ['C C C C C C ( C ) C ( C ) c 1 c c ( O ) c 2 c ( c 1 ) O C 1 = C ( C C C ( C ) C 1 ) C 2 C'],\n",
       "  ['C c 1 c c ( O ) c ( S ) c ( = O ) o 1'],\n",
       "  ['C C C C 1 C c 2 c 3 c ( c ( C ) c ( C ) c 2 C 1 O ) O C ( C ( = O ) O ) C 3'],\n",
       "  ['C C ( = O ) S C c 1 c c c o 1'],\n",
       "  ['C N ( C ( = O ) O C c 1 c c c c c 1 ) N ( C c 1 c c c ( C ( = O ) N C c 2 c c c o 2 ) c c 1 ) C ( = O ) O C c 1 c c c c c 1'],\n",
       "  ['C C 1 C C C C ( c 2 c c ( C ) n o 2 ) = C 1 c 1 c c ( C ) n o 1'],\n",
       "  ['C C 1 C C C C ( c 2 c c ( C ) n o 2 ) = C 1 c 1 c c ( C ) n o 1'],\n",
       "  ['C C ( C ( = O ) O ) C ( = O ) C c 1 c c c c ( C ( = O ) c 2 c c c c c 2 ) c 1'],\n",
       "  ['C O c 1 c ( C ) c c c c 1 C ( = O ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C = C 1 C = C C C ( C ) ( C ) C 1 C ( = O ) C C ( C ) C C ( = O ) C 1 = C ( C ) C = C C C 1 ( C ) C'],\n",
       "  ['C O c 1 c c c 2 c ( c 1 ) C 1 ( C ) C C C C C ( C 2 ) C 1 = N'],\n",
       "  ['C C C ( = O ) c 1 c c c ( C = O ) c c 1'],\n",
       "  ['C P ( C ) ( = O ) C O C ( = O ) c 1 c c c ( N = C 2 c 3 c c c c c 3 C C C 2 C C ( = O ) c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C ( = O ) N c 1 c c c ( - n 2 c ( - c 3 c c c c c 3 ) c c 3 c 2 C C C C 3 ) c c 1 C ( = O ) O'],\n",
       "  ['O = C ( O ) c 1 c c ( - n 2 c ( - c 3 c c c ( Cl ) c c 3 ) c c 3 c 2 C C C C 3 ) c c c 1 O'],\n",
       "  ['O C c 1 c n c 2 c c 3 c ( c c 2 c 1 O ) O C O 3'],\n",
       "  ['C c 1 c c c ( C ( = O ) C O c 2 c c c 3 o c 4 c c c c c 4 c 3 c 2 ) c c 1'],\n",
       "  ['C C O C ( = O ) C C N C c 1 c c c c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C C C N 1 C C C ( O S ( = O ) ( = O ) c 2 c c c c c 2 C ) N ( c 2 n n c ( S C ) s 2 ) C 1 = O'],\n",
       "  ['C # C C ( C ) ( C ) N ( C C C = O ) C ( = O ) N c 1 n n c ( C ( F ) ( F ) F ) s 1'],\n",
       "  ['C # C C ( C ) ( C ) N ( C C C ( = O ) N c 1 n n c ( C ( F ) ( F ) F ) s 1 ) C ( = O ) N c 1 n n c ( C ( F ) ( F ) F ) s 1'],\n",
       "  ['O = C 1 C C C c 2 c ( O C C C Br ) c c c c 2 1'],\n",
       "  ['S c 1 c c c ( - n 2 c n n n 2 ) c c 1'],\n",
       "  ['C C N ( C C ) c 1 c c c 2 c c ( C ( N ) = O ) o c 2 c 1'],\n",
       "  ['C C ( C ) = C C O C ( C ) ( C = C ( C ) C ) O C ( C ) = O'],\n",
       "  ['C C ( = O ) O c 1 c c c ( Cl ) c c 1 C ( = O ) Cl'],\n",
       "  ['C O c 1 c c c ( O C ( C ) = O ) c ( C ( = O ) C # N ) c 1'],\n",
       "  ['O C C C C C C C 1 = C C C C 1 = O'],\n",
       "  ['C C C C C C ( N ) C C [C@H] 1 C C C ( = O ) [C@@H] 1 C C C C C C C ( = O ) O'],\n",
       "  ['C C C C C C ( N ) C C [C@H] 1 C C C ( = O ) [C@@H] 1 C C C C C C C ( = O ) O'],\n",
       "  ['O = C C 1 ( c 2 c c c c c 2 ) C C C ( O ) C C 1'],\n",
       "  ['C C ( C ) ( C ) c 1 c c c ( O ) c ( C ( C ) ( C ) C ) c 1'],\n",
       "  ['C c 1 n c ( N C C S C c 2 [nH] c n c 2 C ) [nH] c ( = O ) c 1 C c 1 c c c c c 1'],\n",
       "  ['C N ( C ) C ( = O ) / C = C / C ( = O ) c 1 c c c ( - c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C c 1 c c c ( C = C 2 C ( = O ) C C ( C ) ( C ) N C ( C ) ( C ) C 2 ) c c 1'],\n",
       "  ['C C ( = O ) O C C ( = O ) [C@@] 1 ( O ) [C@@H] ( C ) C [C@H] 2 [C@@H] 3 C C C 4 = C C ( = O ) C = C [C@] 4 ( C ) [C@H] 3 [C@@H] ( Cl ) C [C@@] 2 1 C'],\n",
       "  ['C C C ( = O ) O C C 1 = C C 2 = C C [C@H] 3 [C@@H] 4 C C C ( = O ) [C@@] 4 ( C ) C C [C@@H] 3 [C@@] 2 ( C ) C C 1'],\n",
       "  ['C C ( = O ) O [C@] 1 ( C ( = O ) C O C ( = O ) C ( C ) C ) [C@@H] ( C ) C [C@H] 2 [C@@H] 3 C C C 4 = C C ( = O ) C = C [C@] 4 ( C ) [C@H] 3 [C@@H] ( Cl ) C [C@@] 2 1 C'],\n",
       "  ['C c 1 [nH] c 2 c c c ( F ) c c 2 c 1 C C C N 1 C C N ( c 2 c c c c c 2 ) C C 1'],\n",
       "  ['O = C ( N C 1 C C C C C 1 ) N C 1 C C C C C 1'],\n",
       "  ['C C O C ( = O ) C c 1 s c ( N C ( C ) = O ) n c 1 - c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C C O C ( = O ) C c 1 s c ( N C ( = O ) N c 2 c c c c c 2 ) n c 1 - c 1 c c c ( Cl ) c c 1'],\n",
       "  ['O C c 1 c n c 2 n 1 - c 1 c c c ( Cl ) c c 1 C ( c 1 c c c c c 1 ) = N C 2'],\n",
       "  ['C C O C ( = O ) C = C ( C ) N c 1 c c c ( N ) c ( [N+] ( = O ) [O-] ) c 1'],\n",
       "  ['C C C C C C ( O ) C C C 1 C ( = O ) C ( O ) C C 1 C # N'],\n",
       "  ['C O C ( = O ) C C [C@H] ( N ) C ( = O ) O C'],\n",
       "  ['C C ( C ) ( C ) O C ( = O ) [C@@H] 1 C C C N 1'],\n",
       "  ['C [C@H] ( N C ( = O ) C N C ( = O ) [C@@H] ( N C ( = O ) O C c 1 c c c c c 1 ) C ( C ) C ) C ( = O ) N 1 C C C [C@H] 1 C ( = O ) O C ( C ) ( C ) C'],\n",
       "  ['C N C ( = O ) c 1 c c c ( C C C ( Br ) C Br ) c n 1'],\n",
       "  ['O C 1 ( c 2 c c c ( C ( F ) ( F ) F ) c c 2 ) c 2 c c c c c 2 C 2 = N C C C N 2 1'],\n",
       "  ['C C O C ( = O ) C 1 = C ( C ) N 2 C C N C 2 = C ( C ( = O ) O C C ) C 1 c 1 c c c c c 1'],\n",
       "  ['C C O C ( = O ) C 1 = C ( N ) N C 2 = C ( C ( = O ) C C C 2 ) C 1 c 1 c c c c ( Cl ) c 1'],\n",
       "  ['C = C 1 C C ( = O ) C ( C ( C ) C ) C C 2 = C ( C ) C C C 1 2'],\n",
       "  ['C [C@@H] 1 C [C@H] 2 [C@@H] 3 C [C@H] ( F ) C 4 = C C ( = O ) C = C [C@] 4 ( C ) [C@H] 3 [C@@H] ( O ) C [C@] 2 ( C ) [C@H] 1 C ( = O ) C C C ( = O ) O'],\n",
       "  ['C C ( O ) C O c 1 c c c c 2 c 1 C C C O 2'],\n",
       "  ['N # C C 1 C C C ( = O ) C 1 C C C C C C C O'],\n",
       "  ['C C C C C C ( = O ) C = C C 1 C C C ( O ) C 1 C C C C C C C O'],\n",
       "  ['C C C C C C ( O ) C = C C 1 C C C ( O ) C 1 C C C C C C ( = O ) O'],\n",
       "  ['C O C ( = O ) O c 1 c c ( C O ) c n c 1 C'],\n",
       "  ['N = C ( N ) N c 1 n c 2 c c c ( [N+] ( = O ) [O-] ) c c 2 [nH] 1'],\n",
       "  ['C c 1 c c c 2 n c ( N C ( = N ) N ) [nH] c 2 c 1 C'],\n",
       "  ['C = C C ( S C # N ) c 1 c c c c c 1'],\n",
       "  ['O = C ( O ) C C 1 C = C C = C C 1 = N O C ( = O ) C ( Cl ) Cl'],\n",
       "  ['C O c 1 c c c ( C ( C # N ) c 2 c c n c 3 c c c c c 2 3 ) c c 1 O C'],\n",
       "  ['O C c 1 c n c 2 c c 3 c ( c c 2 c 1 O ) O C O 3'],\n",
       "  ['C C ( = O ) N 1 C C ( C C N 2 C C C ( c 3 c c c c c 3 ) C C 2 ) c 2 c c 3 c ( c c 2 1 ) O C O 3'],\n",
       "  ['Cl c 1 c c c ( C 2 ( C Br ) O C C ( C S c 3 c c c c c 3 ) O 2 ) c ( Cl ) c 1'],\n",
       "  ['N # C C 1 = C c 2 c c c c c 2 C ( N 2 C C N ( N = C c 3 c c c c n 3 ) C C 2 ) c 2 c c c c c 2 1'],\n",
       "  ['C c 1 c c c c 2 s c ( N N ) n c 1 2'],\n",
       "  ['O = C ( C C [C@H] ( N C ( = O ) O C c 1 c c c c c 1 ) C ( = O ) O ) O c 1 c ( Cl ) c ( Cl ) c ( Cl ) c ( Cl ) c 1 Cl'],\n",
       "  ['C C O N ( C ( = O ) C Cl ) C ( = O ) N C ( C ) C'],\n",
       "  ['C c 1 o c 2 c c ( O ) c c c 2 c 1 = O'],\n",
       "  ['O = C ( C C C Cl ) c 1 c c c 2 o c 3 c c c ( C ( = O ) C C C N 4 C C C C C 4 ) c c 3 c 2 c 1'],\n",
       "  ['O = C ( C C C N 1 C C C C C 1 ) c 1 c c c 2 c ( c 1 ) C c 1 c c ( C ( = O ) C C C N 3 C C C C C 3 ) c c c 1 O 2'],\n",
       "  ['C N C ( = O ) c 1 c c c ( - c 2 n n c ( - c 3 c c c ( C ( = O ) O C ) c c 3 ) o 2 ) c c 1'],\n",
       "  ['C [C@] 1 2 C C [C@@H] 3 [C@H] 4 C C C ( = O ) C = C 4 C C [C@H] 3 [C@@H] 1 C C [C@@H] 2 C C ( O ) C O'],\n",
       "  ['O C c 1 c c c c ( I ) c 1 I'],\n",
       "  ['C C ( = O ) O C C = C ( C ) C C C = C ( C ) C C C = C ( C ) C C C = C ( C ) C C C = C ( C ) C C C = C ( C ) C'],\n",
       "  ['C C ( = O ) N c 1 c c c ( O c 2 c ( N ) c c ( C ( = O ) O ) c c 2 S ( N ) ( = O ) = O ) c c 1'],\n",
       "  ['C C # C C S C 1 C ( = O ) N 1 C ( Cl ) C ( = O ) O C ( C ) ( C ) C'],\n",
       "  ['C c 1 c c c c ( C ) c 1 N C C 1 O C C O 1'],\n",
       "  ['C C ( C ) C 1 = C ( C ( = O ) O ) N 2 C ( = O ) C ( N C ( c 3 c c c c c 3 ) ( c 3 c c c c c 3 ) c 3 c c c c c 3 ) [C@H] 2 S C 1'],\n",
       "  ['Cl c 1 c c ( N C C C N 2 C C O C C 2 ) n c ( - c 2 c c c ( Br ) c c 2 ) n 1'],\n",
       "  ['O = C 1 c 2 c c c c 3 c c c c ( c 2 3 ) C ( = O ) N 1 C C C C C Br'],\n",
       "  ['C O C ( = O ) c 1 c c c ( O C C Br ) c c 1'],\n",
       "  ['C C ( C ) ( C ) N C C ( O ) C O c 1 c c c ( - c 2 c n n s 2 ) c c 1'],\n",
       "  ['C C N ( C C ) C C O C ( = O ) c 1 n n ( - c 2 c c c ( Cl ) c c 2 ) c 2 c 1 C C c 1 c c c c c 1 - 2'],\n",
       "  ['N C ( = O ) C 1 ( N 2 C ( = O ) c 3 c c c ( Br ) c ( Br ) c 3 C 2 = O ) C C C C C 1'],\n",
       "  ['C c 1 c c c ( O ) c ( C ( = O ) O ) c 1 C O'],\n",
       "  ['C c 1 c c 2 c ( c c 1 C ( = O ) C Br ) C ( C ) ( C ) C ( C ) C C 2 ( C ) C'],\n",
       "  ['C C C c 1 n c ( C ) c 2 n 1 N = C ( N ) N C 2 C c 1 c c c c c 1'],\n",
       "  ['C C C c 1 n c ( C ) c 2 n n c ( N ) n 2 n 1'],\n",
       "  ['C C 1 = C ( C O ) C ( C ) ( C ) C C 2 ( C 1 ) O C C O 2'],\n",
       "  ['C C O C ( = O ) C ( Br ) c 1 c c c ( - c 2 c c c c c 2 Cl ) c c 1'],\n",
       "  ['N N = C ( c 1 c c ( F ) c c c 1 N ) c 1 c c c c c 1'],\n",
       "  ['O = C ( c 1 c c c c c 1 Cl ) c 1 c c ( Cl ) c c c 1 - n 1 c c n c 1 C O'],\n",
       "  ['C c 1 c n c ( C O ) n 1 - c 1 c c c ( [N+] ( = O ) [O-] ) c c 1 C ( = O ) c 1 c c c c c 1 Cl'],\n",
       "  ['O = C ( c 1 c c c c c 1 Cl ) c 1 c c c c c 1 - n 1 c c n c 1 C N 1 C ( = O ) c 2 c c c c c 2 C 1 = O'],\n",
       "  ['O = C ( c 1 c c c c c 1 ) c 1 c c ( Br ) c c c 1 N C 1 = N C ( = O ) c 2 c c c c c 2 C 1'],\n",
       "  ['O = C ( c 1 c c c c c 1 Cl ) c 1 c c ( F ) c c c 1 N C 1 = N C ( = O ) c 2 c c c c c 2 C 1'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) N 1 C c 1 n c c n 1 - c 1 c c c ( Cl ) c c 1 C ( = O ) c 1 c c c c c 1'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) N 1 C c 1 n c c n 1 - c 1 c c c ( Cl ) c c 1 C ( = O ) c 1 c c c c c 1 Cl'],\n",
       "  ['O C 1 C c 2 c c c c c 2 C 1'],\n",
       "  ['C [C@] 1 2 C C ( = O ) [C@H] 3 [C@@H] ( C C [C@H] 4 C [C@H] ( N ) C [C@@] 4 3 C ) [C@@H] 1 C C [C@@H] 2 C ( N ) = O'],\n",
       "  ['O = C ( N C ( Cl ) ( Cl ) Cl ) N ( O ) c 1 c c c ( Cl ) c ( Cl ) c 1'],\n",
       "  ['N c 1 c c c ( N c 2 c c c c c 2 ) c c 1'],\n",
       "  ['O = C = N C 1 C C C c 2 s c c c 2 1'],\n",
       "  ['[N-] = [N+] = N S ( = O ) ( = O ) c 1 c c c c ( C ( = O ) Cl ) c 1'],\n",
       "  ['C C ( C ) ( C ( = O ) O ) c 1 c c c ( C 2 = C C C C C 2 ) c c 1'],\n",
       "  ['O = C c 1 c c c ( C 2 = C C C C C 2 ) c c 1'],\n",
       "  ['C C 1 C c 2 c c ( F ) c c c 2 C 1 = O'],\n",
       "  ['O = C ( N C C c 1 c c c 2 c ( c 1 ) O C O 2 ) C 1 2 C C 3 C C ( C C ( C 3 ) C 1 ) C 2'],\n",
       "  ['O = C ( N c 1 c c c ( C ( F ) ( F ) F ) c c 1 ) C 1 2 C C 3 C C ( C C ( C 3 ) C 1 ) C 2'],\n",
       "  ['C [C@H] ( C c 1 c c 2 c ( c c 1 C 1 = O ) O C O 1 ) O C O 2'],\n",
       "  ['C N ( C ) C ( = O ) N ( S Cl ) c 1 c c c ( Cl ) c ( Cl ) c 1'],\n",
       "  ['C C ( S c 1 n n n [nH] 1 ) C 1 = C ( C ( = O ) O ) N 2 C ( = O ) C ( N C ( = O ) C C 3 = C ( C N ) C C C C 3 ) [C@H] 2 S C 1'],\n",
       "  ['C O c 1 c c c ( C c 2 c c ( Cl ) c c ( C c 3 c c c ( O C ) c c 3 O ) c 2 O ) c ( O ) c 1'],\n",
       "  ['C N 1 C ( = O ) C 2 C C C N 2 C c 2 c c ( Br ) c c c 2 1'],\n",
       "  ['C N 1 C ( = O ) C 2 C C C N 2 C c 2 c c ( F ) c c c 2 1'],\n",
       "  ['O = C 1 N c 2 c c c c c 2 C ( = O ) N 2 C C C [C@@H] 1 2'],\n",
       "  ['C N C [C@@H] 1 C C [C@H] ( C ( = O ) O ) N 1 C c 1 c c c c c 1'],\n",
       "  ['C O C 1 ( C ) C C c 2 c ( C ) c ( O ) c ( C ) c ( C ) c 2 O 1'],\n",
       "  ['C C ( C = O ) C C C C ( C ) C C C C ( C ) C'],\n",
       "  ['C C 1 ( C ) c 2 c c c c c 2 - c 2 c c c c c 2 1'],\n",
       "  ['c 1 c c c ( O c 2 c c c c ( - c 3 c c c c c 3 ) c 2 ) c c 1'],\n",
       "  ['F c 1 c c ( - c 2 n c 3 c c c c c 3 o 2 ) c c c 1 C Br'],\n",
       "  ['N # C C c 1 c c c ( - c 2 n c 3 c c c c c 3 o 2 ) c c 1 F'],\n",
       "  ['C N ( C ) C c 1 n n c 2 n 1 - c 1 c c c ( C ( F ) ( F ) F ) c c 1 C ( c 1 c c c c c 1 Cl ) = N C 2'],\n",
       "  ['N C ( = O ) c 1 n c ( O ) n c 1 N'],\n",
       "  ['C O C c 1 c c c c n 1'],\n",
       "  ['C O C c 1 c c c c n 1'],\n",
       "  ['C O c 1 c c c ( - c 2 c c 3 c ( N 4 C C O C C 4 ) n n c ( N 4 C C O C C 4 ) c 3 c ( C ) n 2 ) c c 1'],\n",
       "  ['C O C C 1 = C ( C ( = O ) O C ( c 2 c c c c c 2 ) c 2 c c c c c 2 ) N 2 C ( = O ) [C@@H] ( N ) [C@H] 2 S C 1'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c ( C ( F ) ( F ) F ) c ( Cl ) c ( [N+] ( = O ) [O-] ) c 1 N N c 1 c c c c c 1'],\n",
       "  ['C O c 1 c c c 2 [nH] c 3 c ( c 2 c 1 ) C ( C ( = O ) O C c 1 c c c c c 1 ) C C C 3'],\n",
       "  ['O = C ( O c 1 c c c 2 c ( c 1 ) c 1 c ( n 2 C ( = O ) c 2 c c c c c 2 ) C C C ( C ( = O ) O ) C 1 ) c 1 c c c c c 1'],\n",
       "  ['C C ( C ) = C C C C ( C ) C C = C C ( C ) = C C ( = O ) O'],\n",
       "  ['O = C ( N C C n 1 c c n c 1 ) O c 1 c c c c c 1'],\n",
       "  ['C C O C ( = O ) C ( = C ( N ) N ( C ) N ) c 1 c c c c c 1'],\n",
       "  ['C O C ( = O ) [C@H] ( C c 1 c c c c c 1 ) N C ( = O ) [C@H] ( C C ( = O ) O ) N C ( = O ) O C 1 ( C ) C C C 1'],\n",
       "  ['O = C c 1 c c c ( O ) c ( O ) c 1'],\n",
       "  ['C C O C ( = O ) c 1 c c 2 c c c c ( C ( = O ) O C C ) c 2 [nH] 1'],\n",
       "  ['C N C ( = N C C N C c 1 c c [nH] n 1 ) N C # N'],\n",
       "  ['C C N C ( = N C C N C c 1 c c [nH] n 1 ) N C # N'],\n",
       "  ['O C C c 1 c c c n n 1'],\n",
       "  ['N C C N C c 1 n c c c n 1'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) N 1 C C C S c 1 n c c o 1'],\n",
       "  ['O = C ( C c 1 c c c s 1 ) N C 1 C ( = O ) N 2 C ( C ( = O ) O C ( c 3 c c c c c 3 ) c 3 c c c c c 3 ) = C ( F ) C S ( = O ) [C@H] 1 2'],\n",
       "  ['C c 1 c n n 2 c ( Cl ) c c n c 1 2'],\n",
       "  ['O = C ( N C ( = S ) N C C S C c 1 n c c c n 1 ) c 1 c c c c c 1'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) N 1 C C C S c 1 n c c o 1'],\n",
       "  ['C C C ( C ( = O ) C ( C ) C ( O ) C ( C ) C C C C ( = O ) c 1 c c c ( C ) c ( O ) c 1 C ( = O ) O C ( C ) = O ) C ( C ) = O'],\n",
       "  ['C C O C ( = O ) c 1 c c ( S C c 2 c c c c c 2 ) c ( S c 2 c c c c c 2 ) c ( S ( N ) ( = O ) = O ) c 1'],\n",
       "  ['C O c 1 c c c ( O c 2 c ( N ) c c ( C ( = O ) O ) c c 2 S ) c c 1'],\n",
       "  ['C O c 1 c c c ( O c 2 c ( N ) c c ( C ( = O ) N c 3 c c c c c 3 ) c c 2 S C c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C ( = O ) N c 1 c c ( C ( = O ) O ) c c ( [N+] ( = O ) [O-] ) c 1 - c 1 c c c c c 1'],\n",
       "  ['O = C ( O C c 1 c c c c c 1 ) c 1 c c ( O ) c ( - c 2 c c c c c 2 ) c ( [N+] ( = O ) [O-] ) c 1'],\n",
       "  ['C C ( = O ) N c 1 c c c ( C ( C ) = O ) c c 1 C N ( C ) C'],\n",
       "  ['C N ( C ) C c 1 c c ( C ( = O ) C Br ) c c c 1 N'],\n",
       "  ['C C O C ( = O ) c 1 c c c ( N C ( C ) = O ) c ( C N ( C C ) C C ) c 1'],\n",
       "  ['C C ( C ) ( C ) c 1 c c c ( O ) c ( O ) c 1'],\n",
       "  ['C C ( C ) ( C ) c 1 c c c ( O ) c ( O ) c 1'],\n",
       "  ['C C ( C ) ( C ) C C ( C ) ( C ) c 1 c c c ( O ) c ( C c 2 c c c c ( C O ) n 2 ) c 1'],\n",
       "  ['C C ( = O ) C C c 1 c c c ( - c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C ( = O ) C C c 1 c c c ( - c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C ( O ) C C c 1 c c c ( - c 2 c c c c c 2 ) c ( Cl ) c 1'],\n",
       "  ['C C ( = O ) C C ( C ) c 1 c c c ( C 2 C C C C C 2 ) c c 1'],\n",
       "  ['C C ( = O ) C C ( C ) c 1 c c c ( C 2 C C C C C 2 ) c ( Cl ) c 1'],\n",
       "  ['C C 1 C C N ( C [N+] ( = O ) [O-] ) C 1 = O'],\n",
       "  ['C C C C 1 C = C ( S c 2 c c c c ( O ) c 2 O ) C ( S ( = O ) ( = O ) O C ) = C 1'],\n",
       "  ['C C O C ( = O ) C = C ( N C c 1 c c c ( Cl ) c ( O C C ) c 1 ) O C C'],\n",
       "  ['C # C C C ( C C C C ) O C ( C ) = O'],\n",
       "  ['C C ( C # N ) c 1 c c c c c 1'],\n",
       "  ['C N C C 1 S C C C C S 1'],\n",
       "  ['C S c 1 n n c ( N ( C ) C ( = O ) Cl ) s 1'],\n",
       "  ['C C C C C C ( = O ) O C ( = O ) c 1 c c c ( O ) c c 1'],\n",
       "  ['O = C ( O ) C N ( C Cl ) C ( = O ) O C c 1 c c c c c 1'],\n",
       "  ['C C C S c 1 c c 2 c ( c c 1 Cl ) N = C N S 2 ( = O ) = O'],\n",
       "  ['C N ( C ) C ( = O ) O C 1 C C N ( C ) C ( = O ) N 1 c 1 n n c ( C ( F ) ( F ) F ) s 1'],\n",
       "  ['C C C C C C N ( C ) C ( = O ) O C 1 C C N ( C ) C ( = O ) N 1 c 1 n n c ( C ( F ) ( F ) F ) s 1'],\n",
       "  ['C O c 1 c c c c c 1 N 1 C C N ( C C ( O C ( C ) = O ) C O c 2 c c c c c 2 C C ( N ) = O ) C C 1'],\n",
       "  ['C c 1 n c c ( N ) n 1 C C c 1 c c n c c 1'],\n",
       "  ['C C ( = O ) n 1 c ( [N+] ( = O ) [O-] ) c n c 1 C = C N ( C ) C'],\n",
       "  ['C c 1 c c c c 2 c 1 C C c 1 c c c c c 1 C 2 C C C N ( C ) C'],\n",
       "  ['C C 1 ( c 2 c c c c 3 c 2 C C c 2 c c c c c 2 C 3 = O ) O C C O 1'],\n",
       "  ['C N ( C ) C C C O c 1 c c c 2 c ( c 1 ) C C c 1 c - 2 c c c c 1 C 1 ( C ) O C C O 1'],\n",
       "  ['C C O C ( = O ) C C C C c 1 c c c ( C c 2 c c c ( Cl ) c c 2 ) [nH] 1'],\n",
       "  ['C C O C ( = O ) C 1 C C C C C C C 1'],\n",
       "  ['Cl C ( Cl ) C ( Cl ) ( c 1 c c c ( C 2 C C C C C 2 ) c ( Cl ) c 1 ) C ( Cl ) Cl'],\n",
       "  ['C C ( = O ) [C@H] 1 C C [C@H] 2 [C@@H] 3 C C [C@H] 4 C [C@H] ( O ) [C@@H] ( C C O ) C [C@] 4 ( C ) [C@H] 3 C ( = O ) C [C@] 1 2 C'],\n",
       "  ['C C O [C@H] 1 C [C@@] 2 ( C ) [C@@H] ( C C [C@H] 3 [C@@H] 4 C C [C@H] ( C ( = O ) C I ) [C@@] 4 ( C ) C C ( = O ) [C@@H] 3 2 ) C [C@@H] 1 O C ( = O ) C I'],\n",
       "  ['C O [C@H] 1 C [C@@] 2 ( C ) [C@@H] ( C C [C@H] 3 [C@@H] 4 C C ( C ) = C ( C ( C ) O ) [C@@] 4 ( C ) C C ( = O ) [C@@H] 3 2 ) C [C@@H] 1 O'],\n",
       "  ['O = C ( c 1 c c ( Cl ) c c c 1 Cl ) N 1 C ( = O ) C 2 C C C ( Br ) C ( Br ) C 2 C 1 = O'],\n",
       "  ['C C N ( C C C ) C ( = S ) N ( C C C ) C C C'],\n",
       "  ['C [C@H] 1 C [C@@] 2 ( C ) [C@@H] ( C C [C@H] 3 [C@@H] 4 C C [C@H] ( C 5 ( C ) O C C O 5 ) [C@@] 4 ( C ) C C ( = O ) [C@@H] 3 2 ) C C 1 = O'],\n",
       "  ['C C O C ( = O ) c 1 c ( C ) n c ( N C C C N ( C ) C ) c ( [N+] ( = O ) [O-] ) c 1 Cl'],\n",
       "  ['C C N C ( = C [N+] ( = O ) [O-] ) N C C S C c 1 c c o n 1'],\n",
       "  ['C S ( = O ) ( = O ) c 1 c c ( C ( = O ) O ) c ( Cl ) c c 1 N C c 1 c c c ( C ( = O ) O ) o 1'],\n",
       "  ['C / C ( = C / C ( = O ) N C ( C ) ( C ) C ) O C ( = O ) c 1 c c ( S ( C ) ( = O ) = O ) c ( N N ) c c 1 N C c 1 c c c o 1'],\n",
       "  ['C S ( = O ) ( = O ) c 1 c c ( C ( = O ) O ) c c ( [N+] ( = O ) [O-] ) c 1 Cl'],\n",
       "  ['O = C 1 N c 2 c c c c c 2 C ( = O ) N 1 c 1 c c c ( N C ( = O ) c 2 c c c c c 2 O ) c c 1'],\n",
       "  ['O = C 1 N c 2 c c c c c 2 C ( = O ) N 1 c 1 c c c ( N C ( = O ) c 2 c c c c c 2 O ) c c 1'],\n",
       "  ['C N C ( = O ) C ( C O ) N C ( = O ) c 1 c ( I ) c ( N C ( = O ) C C N ( C ) C ( = O ) c 2 c ( I ) c ( N ) c ( I ) c ( C ( = O ) O ) c 2 I ) c ( I ) c ( C ( = O ) O ) c 1 I'],\n",
       "  ['C C C C C C ( C ) ( C C C Br ) C ( = O ) O C'],\n",
       "  ['C C C C C C ( C ) = C C C = C ( C ) C C # C C C C C C C Cl'],\n",
       "  ['C C C O c 1 c c c ( N ) c c 1'],\n",
       "  ['C C C ( O ) C O C c 1 c c c c c 1'],\n",
       "  ['S = C = N C 1 C C C c 2 c c c c c 2 1'],\n",
       "  ['C C ( = C c 1 c c c c ( F ) c 1 ) C ( = O ) O'],\n",
       "  ['C C ( C c 1 c c c c ( F ) c 1 ) C ( = O ) O'],\n",
       "  ['C C 1 ( c 2 c c c ( C ( O ) C ( C ) ( C ) C ) c c 2 ) O C C O 1'],\n",
       "  ['O = C ( O ) C C c 1 c ( C c 2 c c c c c 2 ) c c c c 1 C ( = O ) O'],\n",
       "  ['C C O C ( = O ) c 1 c c ( = O ) c 2 c ( O C C ( O ) C O c 3 c c c c ( C ) c 3 ) c c c c 2 o 1'],\n",
       "  ['C C ( = O ) c 1 c c c ( O C C ( O ) C O c 2 c c c c c 2 ) c c 1 O'],\n",
       "  ['C O C C O c 1 c c c c ( O ) c 1 C ( C ) = O'],\n",
       "  ['C [C@@H] ( N C ( = O ) C C ( N C ( = O ) O C c 1 c c c c c 1 ) C ( = O ) N [C@@H] ( C ) C ( = O ) N [C@@H] ( C C C ( = O ) O C ( C ) ( C ) C ) C ( = O ) O C ( C ) ( C ) C ) C ( = O ) O C ( C ) ( C ) C'],\n",
       "  ['C [C@H] ( N C ( = O ) O C c 1 c c c c c 1 ) C ( = O ) N [C@@H] ( C C C ( = O ) O C ( C ) ( C ) C ) C ( = O ) N [C@@H] ( C = O ) C O'],\n",
       "  ['C C N ( N c 1 c c c c c 1 ) c 1 c c ( Cl ) c ( C ( F ) ( F ) F ) c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C C N ( C ( C ) = O ) c 1 c c ( O C ) c ( C ( F ) ( F ) F ) c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C S ( = O ) ( = O ) O C C ( O ) ( C C ( Cl ) ( Cl ) Cl ) c 1 c c c c c 1'],\n",
       "  ['C N C 1 = N c 2 c c c ( Br ) c c 2 C ( c 2 c c c c n 2 ) = N C 1'],\n",
       "  ['C C O C ( = O ) C 1 ( C N ) C C C 1'],\n",
       "  ['O = c 1 [nH] c c ( F ) c ( = O ) [nH] 1'],\n",
       "  ['O = S ( = O ) ( O ) c 1 c c ( O ) c c c 1 O'],\n",
       "  ['C c 1 c c c 2 c ( c 1 ) C C ( N 1 C C N ( C C N 3 C C O C 3 = O ) C C 1 ) c 1 c c ( Cl ) c c c 1 S 2'],\n",
       "  ['C O c 1 c c c 2 c ( c 1 ) S c 1 c c c ( S C ) c c 1 C ( = O ) C 2 = O'],\n",
       "  ['C C O C ( = O ) N 1 C C N ( C 2 C c 3 c c c ( Cl ) c c 3 S c 3 c c c ( F ) c c 3 2 ) C C 1'],\n",
       "  ['C S c 1 c c 2 c ( n n 1 ) C C N ( C ( = O ) c 1 c c c c c 1 ) C 2'],\n",
       "  ['N N c 1 c c 2 c ( n n 1 ) C C N ( C ( = O ) c 1 c c c c c 1 ) C 2'],\n",
       "  ['C C c 1 c n c ( N C c 2 c n c ( [N+] ( = O ) [O-] ) n 2 C ) n 1 C C O'],\n",
       "  ['C n 1 c ( C = O ) c n c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C C ( C ) = C C C / C ( C ) = C / C C C ( = O ) C = C ( C ) C'],\n",
       "  ['C C ( C ) = C C C / C ( C ) = C / C C C ( = O ) C = C ( C ) C'],\n",
       "  ['C O C ( = O ) C C C C C C C 1 = C ( O C ) C ( O ) C C 1 = O'],\n",
       "  ['C c 1 c c c ( F ) c c 1 C ( O ) c 1 c c c ( S ( C ) = O ) c c 1'],\n",
       "  ['C N C 1 ( N C C 2 C C 2 ) c 2 c c c c c 2 C C c 2 c c c c c 2 1'],\n",
       "  ['C O c 1 c c c ( C ( = O ) c 2 c c c c ( C ( F ) ( F ) F ) c 2 ) c c 1 C'],\n",
       "  ['N # C c 1 c c c ( O C C C O c 2 c c c ( C ( = N O ) c 3 c c c c c 3 ) c c 2 ) c c 1'],\n",
       "  ['C c 1 c ( Cl ) c c c 2 c 1 [nH] c 1 c ( C ( = O ) N O ) c c c c 1 2'],\n",
       "  ['C c 1 c c c ( N C 2 C C C C ( C ( = O ) O ) C 2 = O ) c c 1 Cl'],\n",
       "  ['C c 1 c ( C N 2 C C N ( C ) C C 2 ) n c 2 n 1 - c 1 c c c c c 1 N ( C ) c 1 c c c c c 1 - 2'],\n",
       "  ['C C ( C ) C C ( O C ( = O ) c 1 c c n c c 1 ) C ( = O ) O'],\n",
       "  ['C C O C ( = O ) C 1 = C ( C ) N C ( C ) = C ( C ( = O ) O C C ) C 1 c 1 c c c c c 1 C ( F ) ( F ) F'],\n",
       "  ['C C O C ( = O ) C C C ( C c 1 c c c c c 1 Cl ) ( C c 1 c c c c c 1 Cl ) C ( C ) = O'],\n",
       "  ['C C 1 = C ( C C ( = O ) N ( C C ( = O ) O ) C c 2 c c c c c 2 ) c 2 c c ( C # N ) c c c 2 C 1 = C c 1 c c c ( S ( C ) = O ) c c 1'],\n",
       "  ['Cl c 1 c c c 2 n c ( N 3 C C C C C 3 ) n c ( N 3 C C C C C 3 ) c 2 c 1'],\n",
       "  ['C C N ( C C ) C C N c 1 n c ( N 2 C C C C 2 ) n c 2 c c c c c 1 2'],\n",
       "  ['O c 1 c c c c ( O c 2 c c c c ( O c 3 c c c c ( O c 4 c c c c c 4 ) c 3 ) c 2 ) c 1'],\n",
       "  ['C C C ( = C C C C ( C ) = C C O c 1 c c c 2 c ( c 1 ) O C C C 2 ) C C'],\n",
       "  ['C C ( = C C O c 1 c c c 2 c ( c 1 ) O C C C 2 ) C C C 1 O C 1 ( C ) C C ( C ) C'],\n",
       "  ['C C C 1 ( C ) C C C ( C ) ( C C O c 2 c c c 3 c ( c 2 ) O C C C 3 ) O 1'],\n",
       "  ['C C C C C C C C N ( C ) c 1 c c c ( C ( = O ) c 2 c c c c c 2 ) c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['O = C ( c 1 c c c c c 1 ) c 1 c c c ( N 2 C C O C C 2 ) c ( [N+] ( = O ) [O-] ) c 1'],\n",
       "  ['C C n 1 n c c 2 n c ( C ( = O ) O C C # N ) c ( C ) n c 2 1'],\n",
       "  ['C O c 1 n c ( - c 2 c c c c c 2 F ) c 2 c c s c 2 n 1'],\n",
       "  ['O = c 1 n c ( - c 2 c c c c c 2 ) c 2 c c ( [N+] ( = O ) [O-] ) s c 2 n 1 C C 1 C C 1'],\n",
       "  ['C C N ( C C ) C c 1 n n c ( C N ) n 1 - c 1 c c c ( Cl ) c c 1 C ( = O ) c 1 c c c c c 1'],\n",
       "  ['[N-] = [N+] = N c 1 c c ( Cl ) n c ( - c 2 c c c o 2 ) n 1'],\n",
       "  ['Cl c 1 c c c 2 c ( c 1 ) O C N 2'],\n",
       "  ['O = C ( C C ( = O ) C 1 C C 1 ) c 1 c c c c n 1'],\n",
       "  ['C O c 1 c c c c ( C C 2 ( Cl ) C ( N C C C Cl ) = N c 3 c c c c c 3 2 ) c 1'],\n",
       "  ['C # C C O C ( = O ) c 1 c c c ( O C C = C ( C ) C C C ( C ) = C ( C ) C ) c c 1'],\n",
       "  ['C # C C O C ( = O ) c 1 c c c ( O C ( C ) C C C ( C ) C C C ( C ) C ) c c 1'],\n",
       "  ['C # C C O C ( = O ) c 1 c c c ( O C ( C ) C C C = C ( C ) C C ) c c 1'],\n",
       "  ['C C ( C ) C C C C ( C ) O c 1 c c c ( C ( = O ) O ) c c 1'],\n",
       "  ['C C C C C C C C C C C C S C C O C ( = O ) C 1 C C 1'],\n",
       "  ['C C ( C ) = C 1 C C C ( C O ) C C 1'],\n",
       "  ['C C C ( O c 1 c c c ( Cl ) c c 1 ) C ( = O ) O'],\n",
       "  ['c 1 c c c 2 c ( c 1 ) C c 1 s c c c 1 N 2'],\n",
       "  ['C C C C C C C ( = O ) O C C N C ( = O ) C C C C C C'],\n",
       "  ['C C O C ( = O ) C ( C ( = O ) C = C c 1 c c c c 2 c c c c c 1 2 ) = C ( N ) N 1 C C C C 1'],\n",
       "  ['C C C C C [C@H] ( O ) / C = C / [C@H] 1 [C@H] ( O C ( = O ) O C C ( C ) C ) C C ( = O ) [C@@H] 1 C / C = C \\\\ C C C C ( = O ) O'],\n",
       "  ['O = C ( O ) C 1 C S [C@@H] 2 C ( N C ( = O ) C c 3 c c c s 3 ) C ( = O ) N 2 C 1 C c 1 c c c s 1'],\n",
       "  ['C C C 1 ( C C ( = O ) C C ( C ) C c 2 c c c c ( O C ) c 2 ) C ( = O ) C C C 1 = O'],\n",
       "  ['C C [C@] 1 2 C C [C@@H] 3 c 4 c c c ( O C ) c c 4 C C [C@H] 3 [C@@H] 1 C C [C@@H] 2 O'],\n",
       "  ['C # C [C@] 1 ( O ) C C [C@H] 2 [C@@H] 3 C C C 4 = C C ( = O ) C C [C@@H] 4 [C@H] 3 C C [C@@] 2 1 C C'],\n",
       "  ['C # C C 1 = C C [C@H] 2 [C@@H] 3 C C C 4 = C C ( = O ) C C [C@@H] 4 [C@H] 3 C C [C@] 1 2 C'],\n",
       "  ['C O C 1 = C C C 2 = C ( C C [C@@H] 3 [C@@H] 2 C C [C@] 2 ( C ) [C@@H] ( O ) C C [C@@H] 3 2 ) C 1'],\n",
       "  ['C O C 1 = C C C 2 = C ( C C [C@@H] 3 [C@@H] 2 C C [C@] 2 ( C ) [C@@H] ( O ) C C [C@@H] 3 2 ) C 1'],\n",
       "  ['C N 1 C C C 2 ( C C 1 ) O C ( c 1 c c c c c 1 ) c 1 c c c c c 1 2'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['C = C C 1 C ( = O ) c 2 c ( c c c c 2 [N+] ( = O ) [O-] ) C 1 = O'],\n",
       "  ['C N ( C ) C 1 C C c 2 [nH] c 3 c c c ( O c 4 c c c c c 4 ) c c 3 c 2 C 1'],\n",
       "  ['C N ( C ) C 1 C C c 2 [nH] c 3 c ( Br ) c c c c 3 c 2 C 1'],\n",
       "  ['C c 1 c c c ( C ) c ( C ( C ) S c 2 c c c c [n+] 2 [O-] ) c 1'],\n",
       "  ['C c 1 c c c ( C S c 2 c c c c [n+] 2 [O-] ) c ( C ) c 1 C'],\n",
       "  ['C O c 1 c c c ( C c 2 c c ( Cl ) c c ( C c 3 c c c ( O C ) c c 3 O ) c 2 O ) c ( O ) c 1'],\n",
       "  ['C C O C ( = O ) c 1 c n c 2 c ( O C c 3 c c c ( Cl ) c c 3 Br ) c c c n 2 c 1 = O'],\n",
       "  ['O = C ( C c 1 c c c s 1 ) N C 1 C ( = O ) N 2 C ( C ( = O ) O ) = C ( C O ) C S [C@H] 1 2'],\n",
       "  ['C # C c 1 c c c c c 1 C 1 = C C N ( C ) C C 1'],\n",
       "  ['C N C ( = O ) C C C ( = O ) O P ( = O ) ( O ) O'],\n",
       "  ['C C O C ( = O ) C ( C C ) = C ( C ) C C C 1 ( C ) O C 1 ( C ) C'],\n",
       "  ['C C ( C ) O c 1 c c c 2 c ( c 1 ) N c 1 c c c c ( S ( C ) ( = O ) = O ) c 1 S 2'],\n",
       "  ['C C ( = O ) N C ( = O ) c 1 c c c ( O C C N C C ( O ) C O c 2 c c c c ( N ) c 2 ) c c 1'],\n",
       "  ['C C C S C ( C ) ( C ) N = N C ( C ) ( C ) C'],\n",
       "  ['O = C ( C [N+] ( = O ) [O-] ) c 1 c c c c c 1'],\n",
       "  ['C c 1 c n ( C C S ( = O ) ( = O ) O ) c 2 c c c c c 1 2'],\n",
       "  ['C C O C ( = O ) N C ( C ) ( C ) C C ( = O ) N C ( = O ) C C ( C ) C'],\n",
       "  ['N # C N 1 C C C C ( C C ( = O ) c 2 c c c ( F ) c c 2 ) C 1'],\n",
       "  ['C N 1 C C C C ( C Cl ) C 1'],\n",
       "  ['C N 1 C C C C ( C C ( = O ) c 2 c c c ( F ) c c 2 ) C 1'],\n",
       "  ['C C O C ( = O ) c 1 n n n [nH] 1'],\n",
       "  ['C C O C ( = O ) c 1 n n n [nH] 1'],\n",
       "  ['C O c 1 c c c ( C C O ) c c 1 O C'],\n",
       "  ['C C 1 ( C ) C C ( = O ) N C ( = O ) C 1 C'],\n",
       "  ['C c 1 c c c c c 1 Cl'],\n",
       "  ['C C 1 = C ( C ) C N ( C ) C 1 = O'],\n",
       "  ['C n 1 c ( = O ) c ( = O ) [nH] c 2 c c c ( C ( F ) ( F ) F ) c c 2 1'],\n",
       "  ['O = C ( N C = S ) c 1 c c c c 2 c 1 N c 1 c c ( C ( F ) ( F ) F ) c c c 1 S 2'],\n",
       "  ['C n 1 c ( = S ) c 2 c c c c 3 s c 4 c c c ( C ( F ) ( F ) F ) c c 4 n ( c 1 = O ) c 3 2'],\n",
       "  ['C C ( = O ) O C C ( C n 1 c ( = O ) c 2 c c c c 3 s c 4 c c ( C ( F ) ( F ) F ) c c c 4 n ( c ( = O ) n 2 3 ) C 1 = O ) O C ( C ) = O'],\n",
       "  ['C C ( C ( = O ) O ) c 1 c c c 2 o c ( N ) n c 2 c 1'],\n",
       "  ['C C O C ( = O ) C ( C ) c 1 c c c ( O ) c ( N C = O ) c 1'],\n",
       "  ['C c 1 c c c c 2 c ( O ) c ( [N+] ( = O ) [O-] ) c ( = O ) [nH] c 1 2'],\n",
       "  ['C C O C ( = O ) c 1 c c c c c 1 N'],\n",
       "  ['N S ( = O ) ( = O ) c 1 c c c ( N C ( = O ) C N c 2 c c c c c 2 ) c ( Cl ) c 1'],\n",
       "  ['C c 1 c c ( S ( N ) ( = O ) = O ) c c c 1 N C ( = O ) C N ( C ) c 1 c c c c c 1'],\n",
       "  ['C C ( C ) N 1 C ( = O ) N C ( c 2 c c c c c 2 ) c 2 c c 3 c ( c c 2 1 ) C C O 3'],\n",
       "  ['N # C c 1 n c c c c 1 Br'],\n",
       "  ['O = c 1 c ( - c 2 c c c c n 2 ) c 2 c c c c c 2 o c 2 c c c ( O ) c c 1 2'],\n",
       "  ['C C N ( C C ) C C n 1 n c 2 c 3 c c c ( O C ) c c 3 s c 3 c ( C ) c c c 1 c 3 2'],\n",
       "  ['C S ( = O ) ( = O ) c 1 c n c ( N C ( = O ) C ( F ) ( F ) F ) c ( [N+] ( = O ) [O-] ) c 1'],\n",
       "  ['C S ( = O ) ( = O ) c 1 c n c ( N C ( = O ) C ( F ) ( F ) F ) c ( [N+] ( = O ) [O-] ) c 1'],\n",
       "  ['O = C ( N c 1 n c c ( Cl ) c c 1 [N+] ( = O ) [O-] ) C ( F ) ( F ) C ( F ) ( F ) C ( F ) ( F ) F'],\n",
       "  ['O = C ( N C 1 = N C C C 1 ) N c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C O c 1 c c c ( C = C c 2 c ( O ) c c ( C ) o c 2 = O ) c c 1 O'],\n",
       "  ['C C C C C C C ( C ) N c 1 c c c ( N C C C = O ) c c 1'],\n",
       "  ['C C O C ( = O ) C 1 = C ( N c 2 n c c c c 2 C ( = O ) O ) C C C 1'],\n",
       "  ['C N ( C ) c 1 c c ( C # N ) c n n 1'],\n",
       "  ['C # C C ( O ) C ( C C C C ) O C ( = O ) C ( C ) C'],\n",
       "  ['O = C 1 C N 2 C C O C 2 ( c 2 c c c c c 2 F ) c 2 c c ( Cl ) c c c 2 N 1 C C O'],\n",
       "  ['C C N ( C C ) N = C ( C ) O C ( C ) = O'],\n",
       "  ['O C [C@H] ( O ) [C@@H] ( O ) [C@H] ( O ) [C@H] ( O ) C O'],\n",
       "  ['C C O c 1 c ( C O ) c n c 2 c 1 c n n 2 C c 1 c c c o 1'],\n",
       "  ['O = C 1 N c 2 c c ( Br ) c ( Br ) c c 2 N ( C ( = O ) C Cl ) c 2 n c c c c 2 1'],\n",
       "  ['C c 1 c c 2 c ( c c 1 C ) N ( C ( = O ) C N 1 C C N ( C C O ) C C 1 ) c 1 n c c c c 1 C 2 = S'],\n",
       "  ['C C C ( C O ) N C ( = O ) c 1 c c c c c 1 C ( = O ) O'],\n",
       "  ['C C ( = O ) c 1 c c c ( O ) c ( C Cl ) c 1'],\n",
       "  ['O = C ( Cl ) C C C C C ( = O ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C O C ( = O ) N 1 C C N 2 C C N ( C C N 3 C C N ( C C N 4 C C N C C 4 ) C C 3 ) C C N ( C C N 1 ) C 2 = O'],\n",
       "  ['C C C c 1 c c c ( F ) c c 1 N ( C C = O ) C ( = O ) C Cl'],\n",
       "  ['O C C C C C C C 1 = C C C C 1 = O'],\n",
       "  ['O = C ( O ) C = C c 1 c c c ( - c 2 c c c c c 2 Cl ) c c 1'],\n",
       "  ['C C N 1 C ( = O ) C ( = C ( C ) O C ( C ) = O ) S C 1 = S'],\n",
       "  ['C C ( = O ) C 1 S C ( = S ) N ( c 2 c c c ( Cl ) c ( Cl ) c 2 ) C 1 = O'],\n",
       "  ['[O-] [n+] 1 c ( C Br ) c [n+] ( [O-] ) c 2 c c c c c 2 1'],\n",
       "  ['C O C C ( C ( N ) = S ) c 1 c c c c n 1'],\n",
       "  ['C O c 1 c c c 2 [nH] c ( C ) c ( C C ( = O ) O C ( O C ( C ) = O ) c 3 c c c ( Cl ) c c 3 ) c 2 c 1'],\n",
       "  ['C C 1 ( c 2 c c c s 2 ) C ( = O ) c 2 c c ( O ) c ( Cl ) c ( Cl ) c 2 C 1 = O'],\n",
       "  ['N C ( = O ) c 1 c ( O C C O c 2 c c c c c 2 ) c c c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C O c 1 c c c ( [C@@H] 2 C N ( C ) C C [C@@H] 2 N ) c c 1 O C'],\n",
       "  ['c 1 c c c ( - c 2 c c 3 c c c c c 3 n 2 - c 2 c c c ( O C C N 3 C C C C 3 ) c c 2 ) c c 1'],\n",
       "  ['C C ( O ) c 1 c c c ( C 2 C C C C C 2 ) c c 1'],\n",
       "  ['O = C 1 N c 2 c c c c c 2 C ( = O ) N 2 C C C [C@@H] 1 2'],\n",
       "  ['O = [N+] ( [O-] ) C = C 1 C N ( S c 2 c c c c c 2 ) C 1 = C [N+] ( = O ) [O-]'],\n",
       "  ['C O C 1 ( c 2 c c c c c 2 ) O C = C ( c 2 c c c ( C ( = O ) O ) c c 2 ) C 1 = O'],\n",
       "  ['C C C C C C C C C C C C C C C C C C ( = O ) O c 1 c c ( C ) c ( O ) c ( C ( C ) ( C ) C ) c 1'],\n",
       "  ['C C O C ( = O ) C ( = C N 1 c 2 c c c ( N ( C ) C ) c c 2 C C C 1 C ) C ( = O ) O C C'],\n",
       "  ['C c 1 c c c 2 n c ( C ) c c c 2 c 1'],\n",
       "  ['O = C 1 N C 2 C = C C = C C 2 ( C c 2 c c c ( F ) c c 2 ) C ( = O ) O 1'],\n",
       "  ['O = C 1 N C 2 C = C C = C C 2 ( C c 2 c c c ( F ) c c 2 ) C ( = O ) O 1'],\n",
       "  ['O = c 1 o c ( = O ) n ( C C 2 C C 2 ) c 2 c c c c c 1 2'],\n",
       "  ['C C ( C ) = C C = C C C C C = C C C C C'],\n",
       "  ['C O C ( C ) ( C ) O [C@H] 1 C C [C@H] ( [C@H] 2 C C c 3 c c ( O C ) c c c 3 C 2 ) C 1 ( C ) C C ( = O ) O'],\n",
       "  ['C # C C ( C C C C C ) O C 1 C C C C O 1'],\n",
       "  ['C C ( C ) = C C C C ( C ) C C = C C ( C ) = C C N'],\n",
       "  ['C O C ( C ) ( C ) C C C C ( C ) C C = C C ( C ) = C C Br'],\n",
       "  ['N # C C 1 c 2 c c ( Br ) c c c 2 - c 2 c c c ( C ( O ) c 3 c ( Br ) c c c 4 c 3 C c 3 c c ( Br ) c c c 3 - 4 ) c c 2 1'],\n",
       "  ['C C 1 ( C = O ) C O [C@@H] 2 C [C@H] 3 [C@@H] 4 C C C 5 = C C ( = O ) C = C [C@] 4 ( C ) [C@H] 3 [C@@H] ( O ) C [C@] 1 2 C'],\n",
       "  ['C C O c 1 c ( O C ) c 2 c ( c ( O C ) c 1 C ( = O ) C C c 1 c c c ( O ) c c 1 ) C C ( N 1 C C C C C 1 ) O 2'],\n",
       "  ['C C N c 1 c ( C ( = O ) O C C ) c ( C ) n c ( N 2 C C N ( C ) C C 2 ) c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C C ( C ) ( C ( O ) [C@H] 1 O [C@@H] ( n 2 c n c 3 c ( N ) n c n c 3 2 ) [C@H] ( O ) [C@@H] 1 O ) O'],\n",
       "  ['C O N = C 1 C = C C = C C 1 C C ( = O ) N [C@@H] 1 C ( = O ) N 2 C ( C ( = O ) O ) = C ( C O C ( = O ) N C C Cl ) C S [C@H] 1 2'],\n",
       "  ['C C C C N c 1 c c ( C ( = O ) O ) c c ( S ( N ) ( = O ) = O ) c 1 O c 1 c c c ( S ( = O ) ( = O ) Cl ) c c 1'],\n",
       "  ['C C C C N c 1 c c ( C ( = O ) N ( C C ) C C ) c c ( S ( N ) ( = O ) = O ) c 1 O c 1 c c c c c 1'],\n",
       "  ['C = C C N c 1 c c ( C ( = O ) O C C = C ) c c ( S ( N ) ( = O ) = O ) c 1 O c 1 c c c c c 1'],\n",
       "  ['C C C [C@@H] ( N C ( = O ) O C c 1 c c c c c 1 ) C ( = O ) N [C@@H] ( C C ( C ) C ) C ( = O ) N [C@@H] ( C C C N C ( = N ) N [N+] ( = O ) [O-] ) C ( = O ) N C C ( N ) = O'],\n",
       "  ['O = P ( Cl ) ( Cl ) N 1 C C C C 1'],\n",
       "  ['C C ( = O ) N 1 C = C N = C ( O C ( C ) = O ) C 1'],\n",
       "  ['C S ( = O ) ( = O ) N 1 C C N ( [Si] ( C ) ( C ) C ) C 1 = O'],\n",
       "  ['C c 1 s c c c 1 S ( = O ) ( = O ) Cl'],\n",
       "  ['C c 1 c c ( N ) n c ( N 2 C C S C C 2 ) n 1'],\n",
       "  ['C = C C 1 = C ( C ( = O ) O C ( C ) ( C ) C ) N 2 C ( = O ) [C@@H] ( N ) [C@H] 2 S C 1'],\n",
       "  ['C S c 1 c c c c ( N ( C C 2 C N ( C c 3 c c c c c 3 ) C C O 2 ) C ( = O ) O c 2 c c c c c 2 ) c 1'],\n",
       "  ['O = C ( O ) c 1 c c c ( C = C c 2 c c c ( - c 3 c c 4 c c c c c 4 s 3 ) c c 2 ) c c 1'],\n",
       "  ['C # C C S c 1 n c ( Cl ) c c ( N C c 2 c c c ( Cl ) c c 2 ) n 1'],\n",
       "  ['C C ( = O ) O C 1 ( C ( C ) C N ( C ) C ) C C C C C 1 c 1 c c c ( O C c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C S c 1 c c c ( N ) c ( C C ( C # N ) C ( N ) = O ) c 1'],\n",
       "  ['C C 1 = C ( C O ) C ( C ) ( C ) C C 2 ( C 1 ) O C C O 2'],\n",
       "  ['C C [C@H] ( C ) [C@H] ( N C ( = O ) [C@H] ( C c 1 c c c ( O ) c c 1 ) N C ( = O ) O C c 1 c c c c c 1 ) C ( = O ) O c 1 c c ( Cl ) c ( Cl ) c c 1 Cl'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['C S c 1 n c ( N 2 C C S C C 2 ) c ( [N+] ( = O ) [O-] ) c ( O ) n 1'],\n",
       "  ['C = C C ( = O ) N c 1 c c c ( N c 2 c c c c c 2 ) c c 1 S C C S C C S'],\n",
       "  ['O = C ( C C S ) N c 1 c c c ( N c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C N ( C C ) C C N C ( = O ) c 1 c c ( S ( C ) ( = O ) = O ) c c c 1 O C'],\n",
       "  ['C = C C ( C ) ( O ) C C C C ( C ) C C C C ( C ) C'],\n",
       "  ['C N ( C c 1 c c c c c 1 ) C 1 ( C N C ( = O ) c 2 c c c ( Cl ) c ( Cl ) c 2 ) C C C C C 1'],\n",
       "  ['C C ( C ) = C C N ( C C = C ( C ) C ) C ( C ) ( C ) C N C ( = O ) c 1 c c c c c 1'],\n",
       "  ['C = C C N ( C ) C ( C ) ( C ) C N C ( = O ) c 1 c c c c c 1'],\n",
       "  ['C O c 1 c c c c 2 c ( N 3 C C S C C 3 ) n c ( N 3 C C N ( S ( = O ) ( = O ) O ) C C 3 ) c c 1 2'],\n",
       "  ['O = C ( c 1 c c c n c 1 ) N 1 C C 2 C O C C 2 C 1'],\n",
       "  ['C # C c 1 c c c ( - c 2 c c c c c 2 F ) c c 1'],\n",
       "  ['C # C c 1 c c c ( - c 2 c c c c c 2 F ) c c 1'],\n",
       "  ['C C C C c 1 o c 2 c ( Br ) c c c c 2 c 1 C ( = O ) c 1 c c c ( O ) c c 1'],\n",
       "  ['C C ( = O ) O C C 1 = C ( C ( = O ) O C ( C ) ( C ) C ) N 2 C ( = O ) [C@H] ( N C ( = O ) C c 3 c c c c c 3 O C ( C ) ( C ) C ) C 2 S C 1'],\n",
       "  ['C C O C ( = O ) C ( C # N ) = N O C ( = O ) O c 1 c c ( Cl ) c ( Cl ) c c 1 Cl'],\n",
       "  ['C C O C ( = O ) C ( C # N ) = N O C ( = O ) O C c 1 c c c c c 1'],\n",
       "  ['C c 1 n n ( - c 2 c c ( C ( F ) ( F ) F ) c c c 2 C ( N ) = O ) c ( = N ) c 2 c c c ( C ( F ) ( F ) F ) c c 1 2'],\n",
       "  ['N # C c 1 c c c ( C ( F ) ( F ) F ) c c 1 N = N N ( C ( = O ) N c 1 c c c c c 1 ) c 1 c c ( C ( F ) ( F ) F ) c c c 1 C # N'],\n",
       "  ['C O c 1 c ( C ) c n c 2 c c c ( F ) c c 1 2'],\n",
       "  ['C C C / C = C / c 1 c c ( S ( = O ) ( = O ) c 2 c c c ( C ( C ) ( C ) C ) c ( C = C C C ) c 2 ) c c c 1 C ( C ) ( C ) C'],\n",
       "  ['C C ( C ) C 1 ( Br ) C c 2 c c ( O C C ( = O ) O ) c ( Cl ) c ( Cl ) c 2 C 1 O'],\n",
       "  ['N C ( C O ) c 1 c c c c c 1'],\n",
       "  ['N C ( C O ) c 1 c c c c c 1'],\n",
       "  ['N C ( C O ) c 1 c c c c c 1'],\n",
       "  ['N C ( C O ) c 1 c c c c c 1'],\n",
       "  ['N C ( C O ) c 1 c c c c c 1'],\n",
       "  ['O = C c 1 c c c ( O ) c ( O ) c 1 Cl'],\n",
       "  ['C c 1 c c c c c 1 N C ( = O ) C C Br'],\n",
       "  ['c 1 c c c ( C n 2 n c ( - c 3 c c n c c 3 ) n c 2 ) c c 1'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C N 1 C c 1 c c c s 1'],\n",
       "  ['C O C ( = O ) c 1 c c ( O C ) c c ( O C ) c 1 C C ( C ) C'],\n",
       "  ['C C O C ( = O ) C C ( c 1 c c c c c 1 ) ( c 1 c c c c c 1 ) c 1 n c c [nH] 1'],\n",
       "  ['O = C ( O ) C c 1 c c c ( - c 2 c n 3 c c c n c 3 n 2 ) c c 1'],\n",
       "  ['O C C C = C = C 1 C C C C C 1'],\n",
       "  ['C N C C C ( O ) ( c 1 c c c c ( C ( F ) ( F ) F ) c 1 ) C 1 C C 1'],\n",
       "  ['C C ( C ) = C C C C ( C ) C C = N ( C ) C'],\n",
       "  ['N C C C = C 1 c 2 c c c c c 2 C = C c 2 c c c c c 2 1'],\n",
       "  ['C N ( C ) C C C C 1 C 2 = C ( C C = C C 2 ) C C c 2 c c c c c 2 1'],\n",
       "  ['C N ( C ) C C C = C 1 c 2 c c c c c 2 C C c 2 c c c ( S ( C ) ( = O ) = O ) c c 2 1'],\n",
       "  ['O = C ( O ) c 1 c c c c c 1'],\n",
       "  ['O C C C C 1 C C C C 1'],\n",
       "  ['F C ( F ) ( c 1 c c c c c 1 ) c 1 c c c c c 1 C Br'],\n",
       "  ['C C ( = O ) O C C N C ( = O ) N ( C ) C 1 c 2 c c c c c 2 O c 2 c c c c c 2 1'],\n",
       "  ['O = S ( = O ) ( C C ( Cl ) C Cl ) O c 1 c c c c c 1'],\n",
       "  ['C C ( C ) O C ( = S ) O C ( C ) C'],\n",
       "  ['C C C N 1 N = C 2 C ( = C c 3 c c c ( Cl ) c c 3 ) C S ( = O ) ( = O ) C C 2 C 1 c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C C C C C C C C C C C C C C C C C C N C ( = O ) c 1 c c ( C ( C ) ( C ) C ) c ( O ) c ( C ( C ) ( C ) C ) c 1'],\n",
       "  ['C C C C C C C C N C ( = O ) c 1 c c c ( O C ( C ) ( C ) C ) c ( C ( C ) ( C ) C ) c 1'],\n",
       "  ['C C O C ( = O ) C ( C C ( O C ) O C ) C ( = O ) O'],\n",
       "  ['C C ( C ( N ) = O ) c 1 c c c 2 c ( = O ) c 3 c c c c c 3 c c c 2 c 1'],\n",
       "  ['O = C 1 N c 2 c c c ( Br ) c c 2 C 2 ( c 3 c c c c n 3 ) O C C N 1 2'],\n",
       "  ['C O c 1 c c c 2 c ( c 1 ) [C@@] 1 3 C C C C [C@@] 1 ( O ) [C@@H] ( C 2 ) N ( C ( = O ) C ( F ) ( F ) F ) C C 3'],\n",
       "  ['C C ( = O ) O c 1 c c c 2 c ( c 1 ) C C [C@H] 1 [C@@H] 3 C C C ( = O ) [C@@] 3 ( C ) C [C@H] ( O ) [C@] 2 1 O'],\n",
       "  ['C C 1 ( c 2 c c c ( - c 3 c c c ( F ) c c 3 ) c c 2 ) C C ( = O ) N N 1'],\n",
       "  ['C C N 1 C C C N 2 c 3 c c c c c 3 C C 2 C 1'],\n",
       "  ['O = C ( C ( = O ) N 1 C C C ( O ) C 1 ) c 1 c [nH] c 2 c c c c c 1 2'],\n",
       "  ['C C / C = C \\\\ C C 1 C ( O ) C C C 1 C C ( C ) = O'],\n",
       "  ['C C ( = C C ( = O ) O C C ) / C = C \\\\ C C ( C ) C C C C ( C ) C'],\n",
       "  ['O C 1 ( c 2 c c c 3 c ( c 2 ) O C O 3 ) C C N C C 1'],\n",
       "  ['C N C ( = O ) N C ( F ) ( F ) F'],\n",
       "  ['C = C ( C ) C N c 1 c c ( C ( = O ) O ) c c ( S ( N ) ( = O ) = O ) c 1 S c 1 c c c ( N ) c c 1'],\n",
       "  ['O = C ( Cl ) N 1 C C C ( C C C c 2 c c c c c 2 ) C C 1'],\n",
       "  ['C N 1 C C ( = O ) N C 1 = N C ( = O ) N c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C O C ( = O ) C 1 C C 2 ( C C 1 C ( = O ) O C ) O C C O 2'],\n",
       "  ['C O c 1 c c c c c 1 C O c 1 c c c ( C ( = O ) O ) c c 1'],\n",
       "  ['C S ( = O ) C 1 C ( = O ) N c 2 c c c ( C ( F ) ( F ) F ) c c 2 1'],\n",
       "  ['C # C C O C ( C ) ( C ) C C C C ( C ) C C O C'],\n",
       "  ['C C C ( C ) S C C C # N'],\n",
       "  ['C N C C ( C # N ) S C C O'],\n",
       "  ['C c 1 c c c ( N Cl ) c ( C ) c 1 N C 1 = C C ( = O ) C = C ( C ) C 1 = N'],\n",
       "  ['C C O C ( = O ) c 1 c ( C ) n c ( N C C C N ( C ) C ) c ( [N+] ( = O ) [O-] ) c 1 Cl'],\n",
       "  ['C N 1 C ( = O ) C ( C C C N C ( = O ) O C c 2 c c c c c 2 ) N C 1 = S'],\n",
       "  ['C C ( C ) N C C ( O ) C O c 1 c c c n 1 - c 1 c c c c c 1'],\n",
       "  ['C O C ( = O ) C C O c 1 c ( C ) c c c ( C ( C ) ( C ) C ) c 1 C'],\n",
       "  ['C C O C ( = O ) C ( C ) O C C C C Br'],\n",
       "  ['C C C C C C ( = N ) N C ( = O ) N c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C C 1 C O c 2 c c ( Cl ) c c 3 c ( = O ) c ( C ( = O ) O ) c n 1 c 2 3'],\n",
       "  ['C C C ( C ) C ( = O ) c 1 c c ( Br ) c ( O C ) c ( Cl ) c 1 Cl'],\n",
       "  ['C C 1 ( C 2 C C C C 2 ) C c 2 c c ( O C C ( = O ) Cl ) c ( Cl ) c ( Cl ) c 2 C 1 = O'],\n",
       "  ['S = C = N c 1 c c c ( Cl ) c n 1'],\n",
       "  ['C O c 1 c c c ( I ) c n 1'],\n",
       "  ['N [C@@H] ( C C C ( = O ) N [C@@H] ( C S C c 1 c c c c c 1 F ) C ( = O ) N C C ( = O ) O ) C ( = O ) O'],\n",
       "  ['N c 1 c c c 2 c ( c 1 ) C N 1 C C C C 1 C N 2 C ( = O ) C C N 1 C C C C 1'],\n",
       "  ['O = C ( c 1 c c c c c 1 ) N 1 C = C C = C C 1'],\n",
       "  ['C N ( C ) c 1 c c c ( C 2 O C ( = O ) N ( C ) c 3 c c c c c 3 2 ) c c 1'],\n",
       "  ['N c 1 c c c ( C ( = O ) O C ( = O ) N 2 C ( = O ) C ( N C ( = O ) C c 3 c c c s 3 ) C 2 ) c c 1'],\n",
       "  ['C c 1 c c 2 o c ( - c 3 c c c c n 3 ) n c 2 c c 1 N = C = S'],\n",
       "  ['C C O C ( = O ) C C c 1 c c ( Cl ) c ( C 2 C C C C C 2 ) c ( Cl ) c 1'],\n",
       "  ['C C O C ( = O ) C ( Br ) C c 1 c c c ( C 2 C C C C C 2 ) c ( Cl ) c 1'],\n",
       "  ['C C ( = O ) N C 1 C ( = O ) N 2 C ( C ( = O ) O ) = C ( C O C ( C ) = O ) C S [C@H] 1 2'],\n",
       "  ['C 1 = N c 2 c c c c 3 c c c c ( c 2 3 ) N 1'],\n",
       "  ['O = C N c 1 c c ( C ( = O ) c 2 c c c c n 2 ) c c c 1 O C c 1 c c c c c 1'],\n",
       "  ['N C ( = S ) N C 1 C c 2 c c c c c 2 C 1'],\n",
       "  ['C C N ( C C ) C C C O N = C 1 c 2 c c c c c 2 C S c 2 c c c c c 2 1'],\n",
       "  ['C C 1 ( C O C c 2 c c c c c 2 ) C C O C ( C ) ( C ( = O ) N c 2 c c c c c 2 ) O 1'],\n",
       "  ['N # C C = C c 1 c c c ( C ( = O ) O ) c c 1'],\n",
       "  ['N # C C = C c 1 c c c ( C ( = O ) O ) c c 1'],\n",
       "  ['C C O [Si] ( O C C ) ( O C C ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['Cl C C C 1 C C 2 C C N 1 C C 2'],\n",
       "  ['F C ( F ) ( F ) c 1 c c c 2 c ( c 1 ) N ( C C 1 C N 3 C C C 1 C C 3 ) c 1 c c c c c 1 S 2'],\n",
       "  ['N C 1 = C C S [C@@H] 2 [C@H] ( N C ( = O ) C c 3 c c c s 3 ) C ( = O ) N 2 C 1'],\n",
       "  ['C C ( O ) c 1 c n c 2 c ( c n n 2 C c 2 c c c o 2 ) c 1 N 1 C C C C C 1'],\n",
       "  ['O = C 1 C 2 C C C C C 2 C ( = O ) N 1 c 1 c c c ( Br ) c c 1 F'],\n",
       "  ['O = C 1 C 2 C C C C C 2 C ( = O ) N 1 c 1 c c c ( F ) c c 1 F'],\n",
       "  ['C c 1 c c c c c 1 N C ( = O ) C 1 ( C ) C C C C C 1'],\n",
       "  ['C C C O c 1 c c 2 c ( c c 1 Br ) C ( c 1 c c c c ( S ( = O ) C C ) c 1 ) = N C C ( = N N C ( = O ) C c 1 c c c c c 1 ) N 2'],\n",
       "  ['C C C C C C C C C C C C ( = O ) N 1 C C c 2 n c 3 c c c c c 3 c ( C ) c 2 C C 1'],\n",
       "  ['C c 1 c 2 c ( n c 3 c c c c c 1 3 ) C C N ( C ( = N ) N ) C C 2'],\n",
       "  ['C C O C ( = O ) C = C c 1 c c c 2 c ( O C ) c c c c 2 c 1'],\n",
       "  ['C # C c 1 c c c 2 c ( O C ) c c c c 2 c 1'],\n",
       "  ['C [Si] ( C ) ( C ) N 1 C C N C C N C C N C C 1'],\n",
       "  ['C C C C C C ( = O ) N c 1 n c 2 c c c ( Cl ) c c 2 n c 1 N C ( = O ) C C'],\n",
       "  ['C C C C C C O c 1 c c c c c 1 C ( = N ) O C C'],\n",
       "  ['C = C C O c 1 c c c c c 1 - c 1 n c ( N ) c ( N ) c ( = O ) [nH] 1'],\n",
       "  ['C = C C O c 1 c c c c c 1 - c 1 n c ( N ) c ( N = O ) c ( = O ) [nH] 1'],\n",
       "  ['C = C C ( = O ) N ( C C C C C C C ( = O ) O C C ) C C C C ( C C C C C ) O C ( C ) = O'],\n",
       "  ['C C O C ( = O ) C C C C C C C ( C ) ( C ) C ( C ( = O ) O C ) C ( C ) C C C C O'],\n",
       "  ['C c 1 c c ( O ) c c ( C ) c 1 O'],\n",
       "  ['N c 1 c c c ( C ( = O ) O ) c ( Cl ) c 1'],\n",
       "  ['N c 1 c ( C ( F ) ( F ) F ) c c c 2 c c n c c 1 2'],\n",
       "  ['O = C ( C Cl ) N = S ( c 1 c c c c c 1 ) c 1 c c c c c 1'],\n",
       "  ['C = C C S C ( = S ) c 1 c c ( C ( C ) ( C ) C ) c ( O ) c ( C ( C ) ( C ) C ) c 1'],\n",
       "  ['C O c 1 c c c ( O C ) c ( S ( = O ) O ) c 1'],\n",
       "  ['C C ( C ) ( C ) c 1 c c c ( N = C 2 c 3 c c c ( [N+] ( = O ) [O-] ) c c 3 C ( = O ) c 3 c c ( [N+] ( = O ) [O-] ) c c c 3 2 ) c c 1'],\n",
       "  ['C C N c 1 c c c ( C ( = O ) N C ( C ) = O ) c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C c 1 n c ( - c 2 n c c ( [N+] ( = O ) [O-] ) n 2 C ) n s 1'],\n",
       "  ['O = C ( O ) C O c 1 c c n c 2 c c c c c 1 2'],\n",
       "  ['C C O C ( = O ) C C c 1 n n n ( C c 2 c c c c c 2 ) c 1 O'],\n",
       "  ['C C O C ( = O ) C C c 1 n n n ( C c 2 c c c c c 2 ) c 1 Cl'],\n",
       "  ['C C ( C ) ( C ) C ( = O ) N ( C ( = O ) c 1 c c c c c 1 C c 1 c c c c c 1 ) C ( C ) ( C ) C'],\n",
       "  ['C C ( C ) O C ( = O ) C 1 = C ( N ) N C ( C ) = C C 1 c 1 c c c c c 1 Cl'],\n",
       "  ['C = C C C C ( = C C C = C ( C ) C ) C C C = C ( C ) C'],\n",
       "  ['C = C C C C ( = C C C = C ( C ) C ) C C C = C ( C ) C'],\n",
       "  ['C = C C O N ( C c 1 c c c c c 1 ) C ( C C ) = O'],\n",
       "  ['C C 1 ( C 2 = C C C C ( = O ) C 2 ) C ( = O ) C C C C 1 = O'],\n",
       "  ['C C C ( O ) C C C C ( O ) C C N C ( C ) c 1 c c c c c 1'],\n",
       "  ['C C O C C O C ( C = C I ) C ( C ) C'],\n",
       "  ['C C 1 C C ( n 2 c ( = O ) [nH] c 3 c c ( Cl ) c c c 3 2 ) C C N 1 C C C C ( c 1 c c c ( F ) c c 1 ) c 1 c c c ( F ) c c 1'],\n",
       "  ['C C ( = O ) O C C n 1 c ( [N+] ( = O ) [O-] ) c n n 1 C C O C ( = O ) N c 1 n n c 2 c c c ( - c 3 n n c ( [N+] ( = O ) [O-] ) n 3 C ) n 2 n 1'],\n",
       "  ['C n 1 c ( [N+] ( = O ) [O-] ) c n c 1 - c 1 n n c 2 c c c ( N C = N C C O ) n n 1 2'],\n",
       "  ['O = C ( O ) C ( O ) c 1 c c c 2 c ( c 1 ) C c 1 c c c c c 1 - 2'],\n",
       "  ['C C O C ( = O ) C ( C ) c 1 c c c 2 c ( c 1 ) C c 1 c c c c c 1 - 2'],\n",
       "  ['C C C C C C C C ( C C C N ( C # N ) C C C C C C C ( = O ) O ) O C ( C ) = O'],\n",
       "  ['O = C 1 N C ( = O ) N ( c 2 c c c c c 2 ) C 1 C C S C'],\n",
       "  ['C O c 1 c c 2 n c ( = O ) c 3 c ( c 2 c c 1 O C ) C C N ( C C N ( C ) C ) C 3'],\n",
       "  ['C N C ( = S ) N C 1 C C C c 2 c c ( C ) c n c 2 1'],\n",
       "  ['C C C N C c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C C ( c 1 c c c c c 1 ) n 1 c n c c 1 C O'],\n",
       "  ['C 1 = C ( N 2 C C C C 2 ) C C S C 1'],\n",
       "  ['C C C C N c 1 c c ( C ( = O ) O ) c c ( S ( N ) ( = O ) = O ) c 1 O c 1 c c c c c 1'],\n",
       "  ['C C 1 2 C = C C ( C = C 1 C O ) C ( C ) ( C ) C 2 = O'],\n",
       "  ['C c 1 c c c ( S ( = O ) ( = O ) O C C C C ( O C ( C ) = O ) C O c 2 c c c ( F ) c c 2 ) c c 1'],\n",
       "  ['C C ( C ) ( C ( = O ) c 1 c c c c c 1 ) c 1 c c c n c 1'],\n",
       "  ['C C 1 ( C ) N C ( c 2 c c c ( N ) c c 2 ) C C 1 c 1 c c n c c 1'],\n",
       "  ['C C C C C C ( C C C C C C C ( = O ) O C C ) O C c 1 c c c c c 1'],\n",
       "  ['O = C 1 c 2 c c c ( Cl ) c ( O ) c 2 C c 2 c ( O ) c c c ( O ) c 2 1'],\n",
       "  ['[N-] = [N+] = N C C ( = O ) C N 1 C ( = O ) c 2 c c c c c 2 C 1 = O'],\n",
       "  ['C N ( C ) C C ( O ) C N C 1 = N c 2 c c c ( Cl ) c c 2 C ( c 2 c c c c c 2 ) = N C 1'],\n",
       "  ['O = C ( Cl ) c 1 c n c 2 c ( Cl ) c c c c 2 c 1 O'],\n",
       "  ['C O c 1 c c c 2 c ( N = C = O ) n c c c 2 c 1'],\n",
       "  ['C N 1 C ( C ( = O ) N c 2 c c c ( F ) c c 2 ) = C ( O ) c 2 c c c 3 c c c c c 3 c 2 S 1 ( = O ) = O'],\n",
       "  ['C N 1 C ( C ( = O ) N c 2 c c c c ( Cl ) c 2 ) = C ( O ) c 2 c c c 3 c c c c c 3 c 2 S 1 ( = O ) = O'],\n",
       "  ['O = S ( = O ) ( c 1 c c ( C O ) c c c 1 F ) N 1 C C C C C 1'],\n",
       "  ['O = C ( O ) c 1 c c ( Cl ) c c ( Cl ) c 1 C 1 C 2 C C C ( C 2 ) C 1 C ( = O ) O'],\n",
       "  ['C C ( O C ( = O ) C c 1 c c c c c 1 ) S ( = O ) C ( = O ) C 1 N 2 C ( = O ) C ( N C ( = O ) C c 3 c c c c c 3 ) [C@H] 2 S C 1 ( C ) C'],\n",
       "  ['C C 1 ( C ) S [C@@H] 2 C ( N C ( = O ) C c 3 c c c c c 3 ) C ( = O ) N 2 C 1 C ( = O ) O'],\n",
       "  ['C C 1 ( C O C c 2 c c c c c 2 Cl ) C O C ( C c 2 c c c c c 2 ) O 1'],\n",
       "  ['C C ( C ( = O ) O ) S ( = O ) C ( C ) c 1 c c c ( - c 2 c c c c c 2 F ) c c 1'],\n",
       "  ['C C ( c 1 c c c ( - c 2 c c c c c 2 F ) c c 1 ) S ( = O ) ( = O ) C C ( = O ) N 1 C C C C C 1'],\n",
       "  ['C c 1 o c ( C ) c ( C ( = O ) N ( O ) C 2 C C C C 2 ) c 1 C'],\n",
       "  ['N c 1 c c c ( O C C O C c 2 c c c c c 2 ) c c 1 N'],\n",
       "  ['N C ( C ( = O ) O ) c 1 c c c 2 c c c c c 2 c 1'],\n",
       "  ['Cl c 1 c c c 2 c ( c 1 ) c ( - c 1 c c c c c 1 ) n n 2 C N 1 C C O C C 1'],\n",
       "  ['O = C 1 N C ( = C c 2 c c c c c 2 ) c 2 c c c c c 2 C 1 = O'],\n",
       "  ['C N C ( = S ) N C 1 C C C ( = O ) c 2 s c c c 2 1'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['C = C C 1 C ( = O ) c 2 c ( c c c c 2 [N+] ( = O ) [O-] ) C 1 = O'],\n",
       "  ['C = C C 1 C ( = O ) c 2 c ( c c c c 2 [N+] ( = O ) [O-] ) C 1 = O'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['C = C C 1 C ( = O ) c 2 c ( c c c c 2 [N+] ( = O ) [O-] ) C 1 = O'],\n",
       "  ['C = C C 1 C ( = O ) c 2 c ( c c c c 2 [N+] ( = O ) [O-] ) C 1 = O'],\n",
       "  ['C = C C 1 C ( = O ) c 2 c ( c c c c 2 [N+] ( = O ) [O-] ) C 1 = O'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) c 2 c 1 c c c c 2 [N+] ( = O ) [O-]'],\n",
       "  ['C C ( C ) = C C ( = O ) C ( = C C 1 = C C [C@H] 2 [C@@H] 3 C C = C 4 C [C@@H] ( O ) C C [C@] 4 ( C ) [C@H] 3 C C [C@] 1 2 C ) C ( C ) ( C ) O'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) N 1 C C c 1 n n c 2 n 1 - c 1 c c c ( Cl ) c c 1 C ( c 1 c c c c n 1 ) = N C 2'],\n",
       "  ['C N ( C ) C C c 1 n n c 2 n 1 - c 1 c c c ( F ) c c 1 C ( c 1 c c c c c 1 ) = N C 2'],\n",
       "  ['C C ( = O ) N c 1 n c n c 2 c ( - c 3 c c n c n 3 ) c ( - c 3 c c n n 3 ) n n 1 2'],\n",
       "  ['Br C c 1 n n c 2 n 1 - c 1 c c c ( Br ) c c 1 C ( c 1 c c c c n 1 ) = N C 2'],\n",
       "  ['C C O C ( = O ) C C 1 ( O ) C c 2 c c c c c 2 C 1'],\n",
       "  ['C O c 1 c c c ( C ( = O ) O ) c ( I ) c 1'],\n",
       "  ['C # C C ( O ) ( C = C I ) C C C C C'],\n",
       "  ['C C ( = C C Cl ) C C C ( C ) ( C C Cl ) c 1 c c c c c 1'],\n",
       "  ['C C N ( C C ) C C n 1 c ( - c 2 c c c c c 2 ) c 2 c c ( Cl ) c c c 2 c 1 C ( = O ) N C'],\n",
       "  ['C C O C ( = O ) c 1 c 2 c c c ( Cl ) c c 2 c ( - c 2 c c c c c 2 ) n 1 C C C N ( C ) c 1 c c c c c 1'],\n",
       "  ['C C 1 C S C 2 = C ( C C C C 2 ) S 1'],\n",
       "  ['C C O C ( = O ) c 1 n c 2 c c c ( Cl ) c c 2 n c 1 O C ( = O ) c 1 n c 2 c c c ( Cl ) c c 2 n c 1 O C'],\n",
       "  ['C c 1 n c c ( [N+] ( = O ) [O-] ) n 1 C C N 1 C ( = O ) c 2 c c c c ( [N+] ( = O ) [O-] ) c 2 C 1 = O'],\n",
       "  ['C O c 1 c c c 2 c ( c 1 ) C C [C@H] 1 [C@@H] 3 C C c 4 c c ( O ) c c c 4 [C@H] 3 C C [C@] 1 2 C'],\n",
       "  ['C c 1 c c c ( C 2 C C ( = O ) C C ( = O ) C 2 ) c c 1'],\n",
       "  ['C C 1 ( C ) [C@@H] ( C = C ( c 2 c c c c c 2 ) c 2 c c c c c 2 ) [C@@H] 1 C ( = O ) O'],\n",
       "  ['C C ( C ) c 1 c c ( S ( N ) ( = O ) = O ) c ( C ( C ) C ) c c 1 C Br'],\n",
       "  ['O C ( C N C C N ( c 1 c c c c c 1 ) c 1 c c c c c 1 ) C O c 1 c c c c c 1'],\n",
       "  ['C c 1 c n c 2 c ( c 1 ) C ( = O ) C C C 2'],\n",
       "  ['C O c 1 c c c c ( C 2 C C C C C 2 = N 2 C C C C 2 ) c 1'],\n",
       "  ['C C C ( = O ) [C@H] 1 C C [C@H] 2 [C@@H] 3 C C [C@H] 4 C [C@H] ( O ) C C [C@] 4 ( C ) [C@H] 3 C ( = O ) C [C@] 1 2 C'],\n",
       "  ['C C O C ( = O ) C C ( = O ) [C@H] 1 C C [C@H] 2 [C@@H] 3 C C [C@H] 4 C [C@H] ( O ) C C [C@] 4 ( C ) [C@H] 3 [C@H] ( O ) C [C@] 1 2 C'],\n",
       "  ['C C C ( = O ) N ( c 1 c c c c c 1 ) C 1 ( C O C ) C C N ( C C ( C ) Br ) C C 1'],\n",
       "  ['C C C ( = O ) N ( c 1 c c c c c 1 ) C 1 ( C O C ) C C N ( C C c 2 c c c ( N ) c c 2 ) C C 1'],\n",
       "  ['Cl c 1 n c ( - c 2 c c c c c 2 ) n c 2 c c c c c 1 2'],\n",
       "  ['C O c 1 c c c ( O C ) c ( C ( = O ) c 2 c c c c ( [N+] ( = O ) [O-] ) c 2 ) c 1'],\n",
       "  ['Cl c 1 c c c 2 c ( c 1 ) c ( - c 1 c c c c c 1 ) c c 1 n n c n 1 2'],\n",
       "  ['C c 1 n n c ( C O ) n 1 - c 1 c c c ( [N+] ( = O ) [O-] ) c c 1 C ( = O ) c 1 c c c c c 1 Cl'],\n",
       "  ['N N c 1 c c ( - c 2 c c c c c 2 ) c 2 c c ( Br ) c c c 2 n 1'],\n",
       "  ['C C C C C C C C / C = C \\\\ C C C C C C C C O c 1 c c ( C ( = O ) O ) c o 1'],\n",
       "  ['C = C C N C ( = O ) O c 1 c c c ( N = C = S ) c c 1'],\n",
       "  ['N C ( = O ) C 1 C C c 2 c c 3 c c c c c 3 n c 2 1'],\n",
       "  ['C c 1 c c 2 [nH] c 3 n c 4 c c ( C ( F ) ( F ) F ) c c c 4 c ( = O ) n 3 c 2 c c 1 C'],\n",
       "  ['C C O c 1 c c c 2 c ( = O ) n 3 c 4 c c ( C ) c ( C ) c c 4 n ( C C ( C ) C ) c 3 n c 2 c 1'],\n",
       "  ['N C ( = O ) c 1 c c ( C ( = O ) C N ( C c 2 c c c c c 2 ) C c 2 c c c c c 2 ) c c c 1 O C C C O'],\n",
       "  ['O = C 1 N C C ( C Cl ) O 1'],\n",
       "  ['C C ( = O ) N C ( = O ) c 1 c c ( C ( N C ( C ) = O ) C c 2 c c c c c 2 ) c c c 1 O C C O'],\n",
       "  ['C c 1 o c 2 c c c ( N ) c c 2 c 1 C'],\n",
       "  ['C N C ( = N C C S C c 1 [nH] c n c 1 C ) N S ( = O ) ( = O ) c 1 c c c ( Cl ) c ( Cl ) c 1'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c ( [N+] ( = O ) [O-] ) c 2 n c ( C ( F ) ( F ) F ) [nH] c 2 c 1 N 1 C C C C C 1'],\n",
       "  ['O = C ( C = C c 1 c c c c c 1 ) N 1 C C N C C 1'],\n",
       "  ['C N ( C ) C ( = O ) N c 1 c c c ( O c 2 c c n c ( Cl ) n 2 ) c c 1'],\n",
       "  ['C O c 1 c c 2 n c ( N 3 C C N C C 3 ) n c ( N ) c 2 c c 1 O C'],\n",
       "  ['C C C ( C ( = O ) N ( c 1 c c c c c 1 ) S ( C ) ( = O ) = O ) c 1 c c c c c 1'],\n",
       "  ['O c 1 c c 2 c ( c c 1 O ) C ( c 1 c c c c c 1 ) = N C C 2'],\n",
       "  ['C O c 1 c c c ( C C N C c 2 c c c ( F ) c c 2 ) c c 1 O C'],\n",
       "  ['O = C ( O ) C 1 = C ( C ( = O ) N c 2 c c c ( Br ) c c 2 F ) C C C C 1'],\n",
       "  ['C c 1 c c c ( S ( = O ) ( = O ) N c 2 c c ( N C ( = O ) c 3 c c c ( C ( = O ) O ) c ( - c 4 c c c c c 4 ) c 3 ) c ( N ) c c 2 N S ( = O ) ( = O ) c 2 c c c ( C ( = O ) O ) c c 2 ) c c 1'],\n",
       "  ['C C ( C ) = C C C C ( C ) = C C S c 1 c c c 2 c c c c c 2 c 1'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c c ( C ( F ) ( F ) C ( F ) ( F ) F ) c c 1'],\n",
       "  ['O = C 1 C C 2 ( C C ( = O ) O c 3 c c c c c 3 2 ) c 2 c c c c c 2 O 1'],\n",
       "  ['C C ( C ) C C C C C ( C ) C O C 1 C C C C O 1'],\n",
       "  ['C N C C n 1 c c ( - c 2 c c c c c 2 ) c 2 c c ( Cl ) c c c 2 1'],\n",
       "  ['C C O C ( C C C N ) O C C'],\n",
       "  ['C C ( = O ) c 1 c c c ( N C 2 = N c 3 c c c ( Cl ) c c 3 C S 2 ) c c 1'],\n",
       "  ['O = c 1 [nH] n c 2 n 1 C = C c 1 c c c c c 1 C 2 c 1 c n ( C ) c 2 n c c c c 1 2'],\n",
       "  ['C # C C S c 1 c c c ( N ) c ( N ) c 1'],\n",
       "  ['C C C ( C ) = C C C C ( C ) = C C O c 1 c c c 2 c ( c 1 ) O C C 2 C'],\n",
       "  ['O = C ( Cl ) c 1 c c c c c 1 O'],\n",
       "  ['O = C ( c 1 c c c c c 1 ) c 1 c c ( Cl ) c c c 1 - n 1 c n c ( C Br ) n 1'],\n",
       "  ['C C C C ( F ) ( F ) C ( F ) C ( F ) ( F ) F'],\n",
       "  ['C C O C ( = O ) C C ( = N ) N c 1 c c c c c 1 N c 1 c c c c c 1'],\n",
       "  ['C O c 1 c c c ( C C N C ( = O ) C C C 2 ( c 3 c c c ( O C ) c ( O C ) c 3 ) S C C C S 2 ) c c 1 O C'],\n",
       "  ['C # C C N 1 C C C ( O C ( = O ) c 2 c c c c c 2 ) N ( c 2 n n c ( C 3 C C 3 ) s 2 ) C 1 = O'],\n",
       "  ['N N C 1 = N O C c 2 c c c ( Cl ) c c 2 1'],\n",
       "  ['Cl C 1 = C ( Cl ) C ( Cl ) = C ( Cl ) C ( Cl ) = C 1 Cl'],\n",
       "  ['N # C C ( c 1 c c c c c 1 ) C ( = O ) c 1 c c ( I ) c c ( I ) c 1 O'],\n",
       "  ['C = C C C N 1 C C C ( O C ( = O ) c 2 c c c ( Cl ) c c 2 ) N ( c 2 n n c ( O C ) s 2 ) C 1 = O'],\n",
       "  ['C c 1 c c c 2 c ( c 1 ) C ( O ) c 1 c c c c c 1 S 2'],\n",
       "  ['c 1 c c c 2 c ( c 1 ) C C ( N 1 C C N C C 1 ) c 1 c c c c c 1 S 2'],\n",
       "  ['C O c 1 c c ( C ) c c ( C ) c 1 C ( = O ) c 1 n c c ( [N+] ( = O ) [O-] ) s 1'],\n",
       "  ['C C 1 C C c 2 c ( O ) c c ( = O ) [nH] c 2 C 1'],\n",
       "  ['C C C n 1 c 2 c ( c ( O ) c c 1 = O ) C C C C 2'],\n",
       "  ['C C ( = O ) N ( C 1 = C ( C ( = O ) O ) C C C C 1 ) C ( C ) C'],\n",
       "  ['C C ( C ) ( C ) c 1 c c ( C O ) c c ( C ( C ) ( C ) C ) c 1 O'],\n",
       "  ['C C O C ( = S ) C C N = [N+] = [N-]'],\n",
       "  ['C C C C C C C C O c 1 c c c ( C ( = O ) N N ) c ( O C C C ) n 1'],\n",
       "  ['N # C S C C 1 = C ( C ( = O ) O ) N 2 C ( = O ) C ( N C ( = O ) C C 3 = C S C C S 3 ) C 2 S C 1'],\n",
       "  ['C C C 1 C c 2 c c c ( C ( C ) C O ) c c 2 C 1 = O'],\n",
       "  ['N C ( = O ) C ( = N O C ( c 1 c c c c c 1 ) ( c 1 c c c c c 1 ) c 1 c c c c c 1 ) c 1 c c c s 1'],\n",
       "  ['C O C 1 ( N C ( = O ) C ( = N O C ( c 2 c c c c c 2 ) ( c 2 c c c c c 2 ) c 2 c c c c c 2 ) c 2 c c c c c 2 ) C ( = O ) N 2 C ( C ( = O ) O C ( C ) ( C ) C ) = C ( C O ) C S [C@H] 1 2'],\n",
       "  ['C [Si] ( C ) ( C ) N 1 C ( = O ) c 2 c c c c c 2 C 1 = O'],\n",
       "  ['C = C ( C Br ) C ( = C ( = O ) O C c 1 c c c ( [N+] ( = O ) [O-] ) c c 1 ) N 1 C ( = O ) C ( N C ( = O ) C O c 2 c c c c c 2 ) C 1 = O'],\n",
       "  ['O N c 1 c c c c ( - c 2 n c 3 n ( n 2 ) C c 2 c c c c c 2 - 3 ) c 1'],\n",
       "  ['C C O c 1 c c c c ( - c 2 n c 3 n ( n 2 ) C c 2 c c c c c 2 - 3 ) c 1'],\n",
       "  ['C C C C C C c 1 n n c ( N = C = O ) s 1'],\n",
       "  ['C N ( C C = O ) C ( = O ) N c 1 n n c ( C Cl ) s 1'],\n",
       "  ['N # C C C O'],\n",
       "  ['C O C ( C ) ( C ) C C C C ( C ) C C = C C ( C ) = C C ( = O ) O C ( C ) C'],\n",
       "  ['N # C S c 1 c c c ( O ) c c 1'],\n",
       "  ['C C O C ( = O ) C ( Cl ) c 1 c c c ( C 2 C C C C C 2 ) c ( Cl ) c 1'],\n",
       "  ['N c 1 c c c ( Cl ) c c 1 C ( F ) ( F ) F'],\n",
       "  ['N # C C ( Cl ) ( c 1 c c c c c 1 ) c 1 c c c c c 1'],\n",
       "  ['C C N 1 C N ( c 2 c c c c ( F ) c 2 ) C ( = O ) c 2 c c c n c 2 1'],\n",
       "  ['C N 1 C N ( c 2 c c c c ( [N+] ( = O ) [O-] ) c 2 ) C ( = O ) c 2 c c c n c 2 1'],\n",
       "  ['C C C C C = N N C 1 = N C C c 2 c c c c c 2 1'],\n",
       "  ['N # C C 1 C C C c 2 c c c ( - c 3 c c c c c 3 ) n c 2 1'],\n",
       "  ['N # C C 1 C C C c 2 c c c ( - c 3 c c c c c 3 ) n c 2 1'],\n",
       "  ['C N 1 C C N ( c 2 c ( N ) c ( F ) c ( F ) c ( C ( C ) = O ) c 2 F ) C C 1'],\n",
       "  ['C O C ( = O ) C C C C C C C C 1 = C C C ( O ) C 1'],\n",
       "  ['C C N ( C C ) c 1 c c c ( N C ( C ) = O ) c ( C ) c 1'],\n",
       "  ['C S c 1 c c c 2 c ( c 1 ) C ( N N ) = N C C ( = S ) N 2'],\n",
       "  ['C S c 1 c c c 2 c ( c 1 ) C ( c 1 c c c c c 1 ) N C c 1 n n c ( C N O C ( = O ) c 3 c c c c c 3 C 3 = O ) n 1 - 2'],\n",
       "  ['O = C 1 c 2 c c c c c 2 C ( = O ) N 1 O C c 1 n n c 2 n 1 - c 1 c c c ( Br ) c c 1 C ( c 1 c c c c n 1 ) = N C 2'],\n",
       "  ['C C N ( C C ) C c 1 n n c 2 n 1 - c 1 c c c ( Br ) c c 1 C ( c 1 c c c c c 1 Cl ) = N C 2'],\n",
       "  ['C O c 1 c c c ( N C C N c 2 c c c ( [N+] ( = O ) [O-] ) c c 2 ) c c 1'],\n",
       "  ['O c 1 c c c 2 c ( c 1 O ) C C C ( N C C C c 1 c c c c c 1 ) C 2 O'],\n",
       "  ['C C 1 = C C P ( = O ) ( C C C Cl ) C 1'],\n",
       "  ['C C ( C = O ) = C C 1 C ( C ) = C C C C 1 ( C ) C'],\n",
       "  ['c 1 c c c ( C 2 O c 3 c c c ( C N 4 C C N ( c 5 n c c c n 5 ) C C 4 ) c c 3 O 2 ) c c 1'],\n",
       "  ['O = C 1 C O C ( C C c 2 c c c c c 2 ) C N 1'],\n",
       "  ['C O c 1 c c c ( - c 2 c c c ( C C C # N ) c c 2 ) c c 1'],\n",
       "  ['C O c 1 c c c ( C ( = O ) N C C c 2 c c c ( - c 3 c c c ( O ) c c 3 ) c c 2 ) c ( O C ) c 1'],\n",
       "  ['C C O C ( = O ) C ( C ) O c 1 c c c ( - c 2 c c c ( C C N C ( = O ) c 3 c c ( Cl ) c c c 3 O C ) c c 2 ) c c 1'],\n",
       "  ['O = C ( Cl ) c 1 c c c 2 c c c c c 2 c 1 O'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c c 2 c ( c 1 ) C ( c 1 c c c c c 1 F ) = N C c 1 n n c ( C O ) n 1 - 2'],\n",
       "  ['C C C N ( C C ) C c 1 n n c 2 n 1 - c 1 c c c ( [N+] ( = O ) [O-] ) c c 1 C ( c 1 c c c c c 1 ) = N C 2'],\n",
       "  ['F c 1 c c c 2 c ( c 1 ) C ( c 1 c c c c c 1 ) = N C c 1 n n c ( C N 3 C C C C C 3 ) n 1 - 2'],\n",
       "  ['C N ( C c 1 n n c 2 n 1 - c 1 c c c ( Cl ) c c 1 C ( c 1 c c c c c 1 ) = N C 2 ) C C 1 C C 1 C C 1 C C 1 C C 1 C C 1 C C 3'],\n",
       "  ['C c 1 c n c 2 c ( c 1 ) C C C C 2 = N O'],\n",
       "  ['C C ( C ) ( C ) C c 1 c c c ( O c 2 c c c ( C ( O ) c 3 c c n c c 3 ) c c 2 ) c c 1'],\n",
       "  ['C N ( C ) C ( = S ) S C ( C [N+] ( = O ) [O-] ) c 1 c c c ( Cl ) c c 1 Cl'],\n",
       "  ['C N ( C ) C ( = S ) S c 1 c c c ( [N+] ( = O ) [O-] ) c c 1 C C [N+] ( = O ) [O-]'],\n",
       "  ['C C ( C ) c 1 c c c ( C ( C [N+] ( = O ) [O-] ) S C ( = S ) N ( C ) C ) c c 1'],\n",
       "  ['C N ( C ) C ( = S ) S C ( C [N+] ( = O ) [O-] ) c 1 c c c 2 c ( c 1 ) O C O 2'],\n",
       "  ['N C ( = O ) C O c 1 c c c c ( N ) c 1'],\n",
       "  ['C N ( C ) c 1 c c c 2 n c [nH] c ( = O ) c 2 c 1'],\n",
       "  ['O C C ( O ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C O c 1 c c c ( C C N C ( = O ) C ( O ) c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C ( C ) C C C C ( C ) C C C C ( C ) C C C C ( C ) C C Cl'],\n",
       "  ['C C ( C ) ( C C = C C O ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C C 1 ( c 2 n n c ( N C ( = O ) N ( C ) C C = O ) s 2 ) C C C C C 1'],\n",
       "  ['C c 1 n c ( / C = C / c 2 c c c ( C # N ) c c 2 ) n o 1'],\n",
       "  ['C = C ( C c 1 c c c c c 1 ) N C 1 C ( = O ) N 2 C ( C ( = O ) O C ( c 3 c c c c c 3 ) c 3 c c c c c 3 ) = C ( C ( = O ) O C ( c 3 c c c c c 3 ) c 3 c c c c c 3 ) C S [C@H] 1 2'],\n",
       "  ['C C O C C 1 = C S [C@@H] 2 C ( N C ( = O ) C c 3 c c c s 3 ) C ( = O ) N 2 C 1 C ( = O ) O C ( c 1 c c c c c 1 ) c 1 c c c c c 1'],\n",
       "  ['C C ( = O ) C 1 = C ( C ( = O ) O ) N 2 C ( = O ) C ( N C ( = O ) C c 3 c c c s 3 ) [C@H] 2 S C 1'],\n",
       "  ['C C ( = O ) C 1 = C ( C ( = O ) O ) N 2 C ( = O ) C ( N C ( = O ) C c 3 c c c s 3 ) [C@H] 2 S C 1'],\n",
       "  ['C C O C C 1 = C S [C@@H] 2 C ( N C ( = O ) C c 3 c c c s 3 ) C ( = O ) N 2 C 1 C ( = O ) O C ( c 1 c c c c c 1 ) c 1 c c c c c 1'],\n",
       "  ['C c 1 c c c ( S ( = O ) ( = O ) O C C O C c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C ( C ) ( C C c 1 c c c c c 1 ) N C C ( O ) c 1 c c c ( O ) c ( C ( N ) = O ) c 1'],\n",
       "  ['C c 1 c c c 2 c ( c 1 ) O C C = C 2 c 1 c c c c c 1'],\n",
       "  ['O = C ( O ) C C 1 C N ( c 2 n c n c 3 c c c c c 2 3 ) c 2 c c c c c 2 1'],\n",
       "  ['C O C ( = O ) C c 1 n n c 2 n 1 - c 1 c c c ( Cl ) c c 1 C ( c 1 c c c c c 1 ) = N C 2'],\n",
       "  ['O = C C c 1 n n c 2 n 1 - c 1 c c c c c 1 C ( c 1 c c c c c 1 Cl ) = N C 2'],\n",
       "  ['N c 1 n c 2 c c c ( C ( = O ) O ) c c 2 s 1'],\n",
       "  ['C C N ( C C N = [N+] = [N-] ) c 1 c c c c c 1'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c ( Cl ) c c c 1 C Br'],\n",
       "  ['C N 1 C C N ( c 2 c c c c ( N C ( = O ) c 3 n n n n 3 C c 3 c c c c c 3 ) c 2 C # N ) C C 1'],\n",
       "  ['O = C N [C@@H] 1 C ( = O ) N 2 C ( C ( = O ) O ) = C ( C O ) C S [C@H] 1 2'],\n",
       "  ['C C C ( = O ) N 1 C C c 2 c c ( O ) c ( O C ) c c 2 C 1'],\n",
       "  ['O = C ( C c 1 c c c c c 1 ) N [C@@H] 1 C ( = O ) N 2 C ( C ( = O ) O C ( c 3 c c c c c 3 ) c 3 c c c c c 3 ) = C ( O ) C S [C@H] 1 2'],\n",
       "  ['C C n 1 n c c 2 c ( O ) c ( C ( = O ) O ) c ( C C c 3 c c c c c 3 ) n c 2 1'],\n",
       "  ['C c 1 c c ( C ( = O ) N c 2 c c c c c 2 ) c ( C ) o 1'],\n",
       "  ['O = C ( N c 1 c c c c c 1 ) C 1 = C ( O ) c 2 c c c c c 2 S ( = O ) ( = O ) C 1'],\n",
       "  ['N # C C c 1 c c c c 2 c ( - c 3 c c c ( F ) c c 3 ) c o c 1 2'],\n",
       "  ['O = C ( O ) C C C O c 1 c c c ( C 2 = C C C C C C C 2 ) c c 1'],\n",
       "  ['N # C C N ( C ( = O ) N C ( = O ) c 1 c ( Cl ) c c c c 1 Cl ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C [C@] 1 2 C C [C@H] 3 [C@@H] ( C C C 4 = C C ( = O ) C C [C@@] 4 3 C = O ) [C@@H] 1 C C C 2 = O'],\n",
       "  ['C S C ( = C [N+] ( = O ) [O-] ) N C C S C c 1 [nH] c n c 1 C'],\n",
       "  ['c 1 c c c ( C O c 2 c c c ( O C C 3 C O 3 ) c c 2 O C c 2 c c c c c 2 ) c c 1'],\n",
       "  ['C C ( = O ) C ( C ) ( C ( = S ) N c 1 c c c c c 1 ) C ( C ) ( O ) C ( C ) C ( = S ) N c 1 c c c c c 1'],\n",
       "  ['C C C ( C ) N c 1 n c ( C ) c ( [N+] ( = O ) [O-] ) c ( Cl ) n 1'],\n",
       "  ['C = C C O c 1 n c ( S C ) n c ( N C ( C ) C ) c 1 [N+] ( = O ) [O-]'],\n",
       "  ['N # C C C C ( = O ) c 1 c c c c c 1'],\n",
       "  ['C n 1 c 2 c ( c 3 c c c c c 3 1 ) c 1 c c c c c 1 c 1 n c ( Cl ) c c c 1 2'],\n",
       "  ['C c 1 c c 2 c ( c c 1 C ) C ( = N c 1 c c c c c 1 ) C = C C 2 = O'],\n",
       "  ['C C ( = O ) O c 1 c c c ( C = C C ( = O ) N = [N+] = [N-] ) c c 1'],\n",
       "  ['C C C C O C ( = O ) N C = C c 1 c c c ( O C C ( O ) C N C ( C ) C ) c c 1'],\n",
       "  ['C O c 1 c c ( C = C N C ( C ) = O ) c c c 1 O C C ( O ) C N C ( C ) C'],\n",
       "  ['O = C ( C C N 1 C ( = O ) c 2 c c c c c 2 C 1 = O ) N c 1 c ( I ) c c ( I ) c ( C ( = O ) O ) c 1 I'],\n",
       "  ['C C ( C c 1 c c c c ( N ) c 1 ) N C = O'],\n",
       "  ['O = C ( C Cl ) N 1 C C C C 2 ( C C 1 ) C C 1 C C C 2 C C 1'],\n",
       "  ['C C 1 = C ( C ( = O ) O C C ( Cl ) ( Cl ) Cl ) N 2 C ( = O ) [C@@H] ( N C ( = O ) c 3 c c ( C ( C ) ( C ) C ) c ( O ) c ( C ( C ) ( C ) C ) c 3 ) [C@H] 2 S C 1'],\n",
       "  ['Cl c 1 c c c 2 c ( c 1 ) C ( c 1 c c c c c 1 Cl ) = N C c 1 n n c c 1 - 2'],\n",
       "  ['C N C ( = O ) c 1 c c c c c 1 - c 1 n c ( - c 2 c c c c c 2 ) n o 1'],\n",
       "  ['C = C C n 1 c ( = O ) c 2 c c c c c 2 n ( - c 2 c c c c ( [N+] ( = O ) [O-] ) c 2 ) c 1 = O'],\n",
       "  ['C C C n 1 c ( = O ) c 2 c c c c c 2 n ( - c 2 c c c c ( [N+] ( = O ) [O-] ) c 2 ) c 1 = O'],\n",
       "  ['N # C C C N ( C C C # N ) C C C O'],\n",
       "  ['C C [C@@] 1 2 C = C C [C@H] 1 [C@@H] 1 C C c 3 c c ( O ) c c c 3 [C@H] 1 C C 2'],\n",
       "  ['C C ( C ) N C C ( O ) C O c 1 c c c n 1 - c 1 c c c c c 1'],\n",
       "  ['O = C ( C ( = C c 1 c c c ( Br ) c c 1 ) c 1 c c c c c 1 ) c 1 c c c ( O ) c c 1'],\n",
       "  ['C c 1 c c ( [N+] ( = O ) [O-] ) c c c 1 C ( = O ) N c 1 c c c 2 c c c ( Cl ) n c 2 n 1'],\n",
       "  ['C C C C C c 1 c c c ( - c 2 c c c ( C ( = O ) O ) c c 2 ) c c 1'],\n",
       "  ['C C ( C ) N ( S c 1 n c 2 c c c c c 2 s 1 ) C ( C ) C'],\n",
       "  ['C C ( C ) = C C C C ( C ) C C O C ( = O ) n 1 c ( - c 2 c s c n 2 ) n c 2 c c c c c 2 1'],\n",
       "  ['C O c 1 c c c ( C N O ) c c 1'],\n",
       "  ['C C ( = N N C 1 = N c 2 c c c ( Cl ) c c 2 C ( c 2 c c c c c 2 ) = N C 1 ) C ( = O ) O'],\n",
       "  ['C C N ( C C ) C C C C ( = N N C 1 = N c 2 c c c ( Cl ) c c 2 C ( c 2 c c c c c 2 ) = N C 1 ) C ( = O ) O C'],\n",
       "  ['C C O C ( = O ) C C C ( = O ) N = N C 1 = N c 2 c c c ( Cl ) c c 2 C ( c 2 c c c c c 2 ) = N C 1'],\n",
       "  ['C O C ( = O ) C ( C ) = N N C 1 = N c 2 c c c ( [N+] ( = O ) [O-] ) c c 2 C ( c 2 c c c c c 2 Cl ) = N C 1'],\n",
       "  ['C O C ( = O ) C ( C ) = N N C 1 = N c 2 c c c c c 2 C ( c 2 c c c c c 2 Cl ) = N C 1'],\n",
       "  ['C C C C C C C C C C C C ( = O ) N ( C ) C C O C ( = O ) c 1 c c c c c 1'],\n",
       "  ['C C C C C C C C C C C C ( = O ) N ( C ) C C O C ( = O ) c 1 c c c ( C ) c c 1'],\n",
       "  ['C C ( C ) ( C ) O N = C ( C ( = O ) O ) c 1 c c c s 1'],\n",
       "  ['C C ( C ) ( C ) O N C ( = O ) C ( = O ) c 1 c c 2 c c c c c 2 s 1'],\n",
       "  ['C N C ( = O ) N ( C ) C ( = O ) N C ( = O ) N C ( = O ) N C'],\n",
       "  ['C C 1 C N ( C ) C ( = O ) N = C ( c 2 c c c c c 2 ) c 2 c c ( Cl ) c c c 2 1'],\n",
       "  ['C C O C ( = O ) c 1 c n ( C C Cl ) c 2 c c ( Cl ) c c c 2 c 1 = O'],\n",
       "  ['O = C ( O ) c 1 c c c ( O ) c 2 n c c c c 1 2'],\n",
       "  ['C C O C ( = O ) C ( Cl ) = C ( C ) N c 1 s c ( C C ) c c 1 C ( = O ) c 1 c c c c c 1 Cl'],\n",
       "  ['C C ( = O ) O C ( C # C C O ) C O c 1 c c c c ( C ( F ) ( F ) F ) c 1'],\n",
       "  ['C O C 1 ( C ) C C c 2 c ( C ) c ( O ) c ( C ) c ( C ) c 2 O 1'],\n",
       "  ['C N 1 C C ( O C c 2 c c c c c 2 ) C 1'],\n",
       "  ['Cl C C C 1 ( C C C 2 ( C C Cl ) C C C C C 2 ) C C C C C 1'],\n",
       "  ['C C C C O c 1 c c ( C ( = O ) O ) c c 2 c 1 C ( = O ) c 1 c c c c c 1 C 2 = O'],\n",
       "  ['C C ( = O ) O C C O C 1 C C N ( C c 2 c c c c c 2 ) C C 1'],\n",
       "  ['C C C C O C ( = S ) N ( C C # N ) C ( = O ) O C C'],\n",
       "  ['C C O c 1 n c ( O P ( = O ) ( O C ) O C ) c ( C # N ) s 1'],\n",
       "  ['C C C ( O c 1 c c c ( C ( C ) ( C ) C C ) c c 1 C ( C ) ( C ) C C ) c 1 n n ( - c 2 c c c c ( C ( = O ) N C 3 = N N ( c 4 c ( Cl ) c c ( Cl ) c c 4 Cl ) c ( = O ) c 3 [n+] 2 [O-] ) c 2 ) c 1 C'],\n",
       "  ['C c 1 c c c c ( N ) n 1'],\n",
       "  ['C C C C 1 = C ( C ( = O ) O C C ) C ( C ( = O ) N C 2 C C 2 ) N C ( C C C ) = C ( C C C ) N 1'],\n",
       "  ['Cl c 1 c c c 2 c ( c 1 ) C ( c 1 c c c c c 1 Cl ) = N C c 1 n n c ( C N C 3 C C 3 ) n 1 - 2'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c c 2 c ( c 1 ) C ( c 1 c c c c c 1 Cl ) = N C c 1 n n c ( C N C C 3 C C 3 ) n 1 - 2'],\n",
       "  ['C N ( C c 1 n n c 2 n 1 - c 1 c c c c c 1 C ( c 1 c c c c c 1 Cl ) = N C 2 ) C ( = O ) C 1 C C 1'],\n",
       "  ['C C S c 1 c c c c 2 c 1 C ( c 1 c c c c ( Cl ) c 1 ) = N C c 1 n n c ( C N C 3 C C 3 ) n 1 - 2'],\n",
       "  ['C = C C N C c 1 n n c 2 n 1 - c 1 c c c ( Cl ) c c 1 C ( c 1 c ( F ) c c c c 1 F ) = N C 2'],\n",
       "  ['N c 1 c c c ( - c 2 c c c ( C ( = O ) C C C ( = O ) O ) c c 2 ) c c 1'],\n",
       "  ['C C ( O ) C N C [C@H] ( O ) [C@@H] ( O ) [C@H] ( O ) [C@H] ( O ) C O'],\n",
       "  ['O C [C@@H] ( O ) [C@@H] ( O ) [C@H] ( O ) [C@@H] ( O ) C N C C N 1 C C O C C 1'],\n",
       "  ['C O c 1 c c c 2 c ( c 1 ) c ( C C N 1 C C C C C 1 C C C C C 2 ) c ( C ) n 2 C ( = O ) c 1 c c c ( F ) c c 1'],\n",
       "  ['C [C@H] ( C C C C ( C ) ( C ) O C 1 = C C N ( c 2 c c c c c 2 ) C ( = O ) N 1 ) [C@H] 1 C C [C@H] 2 C 3 = C C = C 4 C [C@@H] ( O ) C = C [C@] 4 ( C ) [C@H] 3 C C [C@@] 2 1 C'],\n",
       "  ['C C c 1 n c 2 n c 3 c c ( Cl ) c c c 3 n c 2 n 1 C C ( Cl ) ( Cl ) Cl'],\n",
       "  ['C C ( C ) N C C ( = O ) c 1 c c c ( O ) c 2 [nH] c ( = O ) c c c 1 2'],\n",
       "  ['C C O C ( = O ) c 1 c ( C ) n c ( N C ( C ) ( C ) C ) c ( [N+] ( = O ) [O-] ) c 1 Cl'],\n",
       "  ['O = C 1 c 2 c c c ( Cl ) c c 2 C ( = O ) N 1 C C N 1 C C C C 1'],\n",
       "  ['C C C C C C ( O ) C C N 1 C ( = O ) C S ( = O ) C 1 C C C C C C C ( = O ) O'],\n",
       "  ['C C C C C [C@@H] ( C C = O ) O C c 1 c c c c c 1'],\n",
       "  ['C C C C C C ( O C ( C ) = O ) C C N 1 C ( = O ) C S C 1 C C C C C C C ( = O ) O'],\n",
       "  ['C N C ( = O ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C C C C C C C ( O ) C N C C C C C C N C C ( O ) C C C C C C'],\n",
       "  ['C C C C C C C C C ( O ) C N C C C N C C ( O ) C C C C C C C C'],\n",
       "  ['C C C C C C ( O ) C N C C C C C C C N C C ( O ) C C C C C'],\n",
       "  ['C C C C C ( O ) C N C C C C C C C C C N C C ( O ) C C C C'],\n",
       "  ['N c 1 c c c ( - c 2 n c ( - c 3 c c c n c 3 ) n o 2 ) c c 1'],\n",
       "  ['O = C ( O ) c 1 c c ( S C C O ) c c c 1 O'],\n",
       "  ['C O c 1 c c ( C ( = O ) N c 2 c c c c 3 c 2 C N ( C ) C C 3 ) c c ( O C ) c 1 O C'],\n",
       "  ['C N 1 C C c 2 c ( c c c c 2 N C ( = O ) c 2 c c c c c 2 Br ) C 1'],\n",
       "  ['C [C@] 1 2 C C [C@H] 3 [C@@H] ( C C C 4 = C C ( = O ) C C [C@@] 4 3 C = O ) [C@@H] 1 C C C 2 = O'],\n",
       "  ['C [C@H] 1 C C 2 ( C = C 3 C C [C@H] 4 [C@@H] 5 C C C ( = O ) [C@@] 5 ( C ) C C [C@@H] 4 [C@] 3 1 C O ) O C C S 2'],\n",
       "  ['C [C@H] 1 C C C = C 2 C C [C@H] 3 [C@@H] 4 C C C ( = O ) [C@@] 4 ( C ) C C [C@@H] 3 [C@] 2 1 C = O'],\n",
       "  ['C O [C@H] 1 C C [C@H] 2 [C@@H] 3 C C C 4 = C C ( = O ) C C [C@@H] 4 [C@H] 3 C C [C@] 1 2 C'],\n",
       "  ['C [C@] 1 2 C C [C@H] 3 [C@@H] ( C C C 4 = C C ( = O ) C C [C@@] 4 3 C = O ) [C@@H] 1 C C C 2 = O'],\n",
       "  ['C O c 1 c c c ( O ) c ( C ) c 1 N'],\n",
       "  ['N c 1 c c c c 2 c 1 C ( = O ) c 1 c ( - c 3 c c c 4 c c c c c 4 3 ) c c c c 1 n 2 O'],\n",
       "  ['C O c 1 c c ( C = O ) c c ( F ) c 1 N'],\n",
       "  ['N c 1 c c c ( C = C ( Cl ) Cl ) o 1'],\n",
       "  ['C C O C ( = O ) C = P ( c 1 c c c c c 1 ) ( c 1 c c c c c 1 ) c 1 c c c c c 1'],\n",
       "  ['C N ( C ) C ( = O ) N 1 C C c 2 c c c ( S ( N ) ( = O ) = O ) c c 2 C C 1'],\n",
       "  ['C C C C C C ( O ) / C = C / C 1 C C C 2 ( O C C ( C ) ( C ) C O 2 ) C 1 C C = O'],\n",
       "  ['O = C ( N c 1 c c ( Br ) c ( O ) c ( Br ) c 1 ) c 1 c c c c c 1 O C c 1 c c c c c 1'],\n",
       "  ['C O c 1 c ( Cl ) c c ( N C ( = O ) c 2 c c c c c 2 O ) c c 1 Cl'],\n",
       "  ['C O c 1 c c ( O C ) c 2 c ( c 1 ) O C ( C ( C ) ( C ) C ( N ) = O ) C C 2 = O'],\n",
       "  ['C O [C@@H] 1 C [C@H] ( N ) [C@H] ( O C ( = O ) c 2 c c c c c 2 ) [C@@H] ( C Br ) O 1'],\n",
       "  ['C = C 1 O [C@H] ( O C ) C [C@H] ( N C ( C ) = O ) [C@@H] 1 O C ( = O ) c 1 c c c c c 1'],\n",
       "  ['C O C ( = O ) c 1 c c c ( O ) c n 1'],\n",
       "  ['C C n 1 c ( = C 2 C ( = O ) c 3 c c c c c 3 C 2 = O ) C = C c 2 c c c c c 2 1'],\n",
       "  ['C c 1 c c c ( N ) c c 1 N 1 C ( = O ) c 2 c c c ( Cl ) c c 2 C 1 = O'],\n",
       "  ['C C C c 1 c c ( C ( = O ) O C C ) c ( C ) o 1'],\n",
       "  ['N = C ( N C C S C c 1 n c c c c 1 Br ) N C C S C c 1 n c c c c 1 Br'],\n",
       "  ['C C C N ( C C C ) c 1 c ( [N+] ( = O ) [O-] ) c c ( C ) c ( C ) c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C S ( = O ) ( = O ) N c 1 c c c c ( [N+] ( = O ) [O-] ) c 1'],\n",
       "  ['C N ( C ) C ( = O ) c 1 c c c ( N c 2 c c n c 3 c c ( C ( F ) ( F ) F ) c c c 2 3 ) c c 1'],\n",
       "  ['C C ( = O ) N C C n 1 n n n c 1 S c 1 c c c ( [N+] ( = O ) [O-] ) c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C C ( = O ) N c 1 c c ( [N+] ( = O ) [O-] ) c c c 1 O'],\n",
       "  ['C C ( = O ) C 1 = C ( O ) C ( = C ( C ) N c 2 c c c ( O ) c ( [N+] ( = O ) [O-] ) c 2 ) C ( = O ) O C 1 = O'],\n",
       "  ['C c 1 c c c c ( C ) c 1 N = C N'],\n",
       "  ['N c 1 c c c ( C 2 = N N C ( = O ) C C 2 ) c c 1 N'],\n",
       "  ['C C C ( C ) N C ( C ) C ( = O ) c 1 c c c ( O ) c 2 [nH] c ( = O ) c c c 1 2'],\n",
       "  ['C c 1 n n c ( S C C 2 = C ( C ( = O ) O ) N 3 C ( = O ) C ( N C ( = O ) C N c 4 c c c ( C Cl ) c c 4 ) C 3 S C 2 ) s 1'],\n",
       "  ['C C N ( C C ) C C c 1 c c c 2 s c 3 c c c c c 3 c 2 c 1'],\n",
       "  ['C C C C ( = O ) C C C C ( = O ) C C C'],\n",
       "  ['N c 1 c c c ( C ( = O ) c 2 c c c c n 2 ) c c 1 N'],\n",
       "  ['N c 1 c c c ( C ( = O ) c 2 c c c n c 2 ) c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C O C ( = O ) c 1 c c c c c 1'],\n",
       "  ['O C C O C c 1 c c c c n 1'],\n",
       "  ['C C N ( C C ) C ( = O ) c 1 n c ( C N ) n ( - c 2 c c c ( Cl ) c c 2 C ( = O ) c 2 c c c c c 2 ) n 1'],\n",
       "  ['C c 1 c c c c c 1 O C C ( O ) C N C C O c 1 n c c n c 1 C'],\n",
       "  ['C C O C ( = O ) C ( C # N ) = C c 1 c c c ( C ) c c 1'],\n",
       "  ['C C ( C ) ( C ) N = N C 1 ( N C ( = S ) N N C ( N ) = O ) C C C C C 1'],\n",
       "  ['C N ( N ) C ( = O ) N C 1 ( N = N C ( C ) ( C ) C ) C C C C C 1'],\n",
       "  ['C O C ( = S ) N C 1 ( N = N C ( C ) ( C ) C ) C C C C C 1'],\n",
       "  ['C C ( C ) ( C ) N = N C ( C ) ( C ) N = C = S'],\n",
       "  ['C c 1 c ( C ( = O ) C S ( C ) ( = O ) = O ) c ( O ) c 2 c c c c c 2 n 1 C'],\n",
       "  ['C C ( C ) = C C C C ( C ) = C C C C ( C ) = C C C C ( = O ) O C ( C ) C'],\n",
       "  ['O = C ( N C C Cl ) N ( C C Cl ) N = O'],\n",
       "  ['O = C ( O ) C c 1 c c c ( C ( = O ) C 2 C C C 2 ) c c 1'],\n",
       "  ['C C O C ( = O ) C ( C ) ( C ( = O ) O C C ) c 1 c c c c c 1 C ( = O ) C 1 C C C C C 1'],\n",
       "  ['C N ( C ( = O ) C N C ( = O ) [C@H] ( C ) N C ( = O ) O C ( C ) ( C ) C ) C ( = O ) N C C ( = O ) N [C@H] ( C C ( C ) C ) C ( = O ) O'],\n",
       "  ['O = S ( = O ) ( O ) C C C N C C O'],\n",
       "  ['C C O C ( N ( C ) C C O ) c 1 n c ( Cl ) c ( Cl ) n c 1 Cl'],\n",
       "  ['C C C ( C ) N ( C ( = O ) C Cl ) C ( C ) C C'],\n",
       "  ['C O c 1 n c n c 2 c ( Cl ) c ( N C ( = S ) N C C N ) c c c 1 2'],\n",
       "  ['C C C ( C ) ( C ) c 1 c c ( Cl ) c ( O ) c ( C N ) c 1'],\n",
       "  ['O = C ( N C 1 C C N ( C C ( O ) c 2 c c c 3 c ( c 2 ) C C C C 3 ) C C 1 ) c 1 c c c c c 1'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c c ( N N = C ( C ( F ) ( F ) F ) C ( F ) ( F ) F ) c c 1'],\n",
       "  ['O = C ( N N c 1 c c c ( [N+] ( = O ) [O-] ) c c 1 ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C C 1 N = C ( c 2 c c c c c 2 Cl ) c 2 c c ( [N+] ( = O ) [O-] ) c c c 2 N C 1 = S'],\n",
       "  ['C O C ( = O ) C ( C C ( O ) c 1 c c c c c 1 ) C C c 1 c c c c c 1'],\n",
       "  ['C = C C O c 1 c c c c c 1 - c 1 n c c ( C ( = O ) O C C ) c ( = O ) [nH] 1'],\n",
       "  ['C O C ( C C ( O C ) C ( C ) O ) O C'],\n",
       "  ['C c 1 c c c ( O ) c c 1 N'],\n",
       "  ['O = c 1 c 2 c c c c c 2 c c ( O C C 2 C O 2 ) c 2 s c c c 1 2'],\n",
       "  ['F c 1 c c c ( O C 2 C N ( C C c 3 c [nH] c 4 c c c c c 3 4 ) C 2 ) c c 1'],\n",
       "  ['C C O c 1 c c c ( C ( C ) ( C ) C 2 = N C C C N 2 C ) c ( C ( = O ) N 2 C C C C 2 = O ) c 1'],\n",
       "  ['C C 1 C C S C ( C ) ( C ) O 1'],\n",
       "  ['O = C ( c 1 c c c ( F ) c c 1 [N+] ( = O ) [O-] ) N 1 C C C 2 ( C C 1 ) C ( = O ) N C N 2 c 1 c c c c c 1'],\n",
       "  ['C C O C C O C C O C C C ( = O ) C ( C C C Br ) C ( C ) = O'],\n",
       "  ['C C C c 1 n s c ( N = C ( Cl ) N ( C ) C ) c 1 C # N'],\n",
       "  ['C c 1 c c c c c 1 - n 1 n c ( C ( = O ) O ) c ( - c 2 c c c c c 2 N ) n 1'],\n",
       "  ['O = C ( N c 1 c c c c c 1 ) n 1 c c ( F ) c ( = O ) [nH] c 1 = O'],\n",
       "  ['C C C C C C ( O ) C C n 1 c ( = O ) s c ( = O ) n 1 C C C C C = C C ( = O ) O'],\n",
       "  ['C O C ( = O ) N c 1 n c 2 c c ( C 3 ( c 4 c c c ( F ) c c 4 ) O C C O 3 ) c c c ( N ) c 2 [nH] 1'],\n",
       "  ['C c 1 c c c ( O c 2 c c c ( C ) c c 2 ) c ( C ( = O ) O ) c 1'],\n",
       "  ['Cl C C C O c 1 c c c c c 1 C 1 C N C C N 1 C 1 C c 2 c c ( Cl ) c c c 2 O c 2 c c c c c 2 1'],\n",
       "  ['C c 1 c c c 2 c ( c 1 ) C C ( = O ) c 1 c c ( S C ) c c c 1 O 2'],\n",
       "  ['C C ( = O ) N c 1 c c c ( C ( = O ) c 2 c c c c c 2 C ( = O ) O ) c ( N ( C ) C ) c 1'],\n",
       "  ['C # C C ( C ) ( O ) C C C = C ( C ) C C C = C ( C ) C'],\n",
       "  ['C c 1 n c c ( C = C C ( = O ) O ) s 1'],\n",
       "  ['C C C c 1 n c c ( C = O ) s 1'],\n",
       "  ['C N 1 C ( = O ) C 2 N C C N 2 C 2 C C c 3 c c c c c 3 N 1 2'],\n",
       "  ['C C ( = O ) C 1 = C ( O ) C ( = C ( C ) N c 2 c c c ( O C ( = O ) N C c 3 c c c c c 3 ) c c 2 ) C ( = O ) O C 1 = O'],\n",
       "  ['C C C C C C S ( = O ) C 1 ( C C ) C C C C C 1 C C'],\n",
       "  ['C = C ( C C ( C ) C ) C ( = O ) N C O C ( = S ) S C C C ( O C ) O C C'],\n",
       "  ['N C C C C [C@H] ( N C C ( O ) C ( O ) C ( O ) C ( O ) C O ) C ( = O ) O'],\n",
       "  ['N C C C C [C@H] ( N C C ( O ) C ( O ) C ( O ) C ( O ) C O ) C ( = O ) O'],\n",
       "  ['C C C ( N S ( = O ) ( = O ) c 1 c c c ( Cl ) c c 1 ) P ( = O ) ( O c 1 c c c c c 1 ) O c 1 c c c c c 1'],\n",
       "  ['C O c 1 c c c 2 c ( c 1 ) C ( = O ) C ( C = C c 1 c c c ( C # N ) c c 1 ) S ( C ) = O'],\n",
       "  ['O = c 1 o c 2 c c ( Cl ) c n c 2 n 1 C O'],\n",
       "  ['O = C ( O ) C c 1 c s c ( - c 2 c c c c c 2 ) n 1'],\n",
       "  ['C C C C C [C@@H] ( O ) C # C C N ( C C C C C C C ( = O ) O ) S ( C ) ( = O ) = O'],\n",
       "  ['C c 1 c c ( Cl ) c c c 1 N'],\n",
       "  ['C c 1 c c ( Cl ) c c c 1 N'],\n",
       "  ['C C C C C C C C C C C C C C C C C C N ( C C C C C C C C C C C C C C C C C C ) C ( = O ) N C C N ( C C O ) C C O'],\n",
       "  ['O = C ( c 1 c c c ( F ) c c 1 ) C 1 C C N ( C C C n 2 c ( = O ) [nH] c 3 c c c ( Cl ) c c 3 2 ) C C 1'],\n",
       "  ['O = C ( O ) C c 1 c c c ( Cl ) c ( C ( = O ) C c 2 c c c ( Cl ) c c 2 ) c 1'],\n",
       "  ['C O C ( = O ) C 1 C C C c 2 c 1 c c c ( O C ) c 2 O C'],\n",
       "  ['N # C c 1 c ( O C c 2 c c c c c 2 ) c c c 2 c 1 C C [C@@H] ( O ) [C@@H] 2 O'],\n",
       "  ['C C O C ( = O ) C C 1 ( C ) O c 2 c c c ( Cl ) c c 2 - n 2 c c c c 2 1'],\n",
       "  ['C C C C N C ( = O ) O c 1 c c c c ( N 2 C ( = O ) C C ( C ) ( O C ) O C 2 = O ) c 1'],\n",
       "  ['C C C C C C C C C C C C C C C N 1 C C N C C 1'],\n",
       "  ['C C C C C C ( = O ) C = C c 1 c c ( O ) c 2 c ( c 1 ) O C ( C ) ( C ) C 1 C C C ( C ) ( C ) C 1 C C 2 ( C ) C'],\n",
       "  ['C C C O C C ( O ) C Cl'],\n",
       "  ['C O C ( = O ) c 1 c c c ( [N+] ( = O ) [O-] ) c n 1 S C c 1 c c c c ( Cl ) c 1'],\n",
       "  ['C O C ( = O ) c 1 c c c ( S C c 2 c c c c ( C ( F ) ( F ) F ) c 2 ) c ( C ) n 1'],\n",
       "  ['C C C C C C C C C C C C O C C ( Cl ) C n 1 c c n c 1'],\n",
       "  ['C C C C C C C C C C O C ( C O c 1 c c c ( Cl ) c c 1 ) C n 1 c c n c 1'],\n",
       "  ['O = C ( N O ) N C 1 C C C c 2 s c c c 2 1'],\n",
       "  ['O = C ( C C [C@H] ( N C ( = O ) c 1 c c c c c 1 ) C ( = O ) O ) N C 1 C C C c 2 s c c c 2 1'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c c c c c 1 - c 1 c c c c c 1 [N+] ( = O ) [O-]'],\n",
       "  ['C C # C C N c 1 c c c ( O C 2 3 C C 4 C C ( C C ( C 4 ) C 2 ) C 3 ) c c 1'],\n",
       "  ['C C 1 ( O c 2 c c c ( N ) c c 2 ) C 2 C C 3 C C ( C 2 ) C C 1 C 3'],\n",
       "  ['F c 1 c c ( O C 2 3 C C 4 C C ( C C ( C 4 ) C 2 ) C 3 ) c c c 1 N 1 C C C C 1'],\n",
       "  ['C c 1 c c ( C ) n ( C ) c ( = O ) c 1 C # N'],\n",
       "  ['C C O C ( = O ) c 1 c n ( C C C ( C ) C ) c 2 n n ( C ) c c 2 c 1 = O'],\n",
       "  ['C O C ( = O ) C C C C C C C C ( = O ) C C C ( = O ) C ( O C ) O C'],\n",
       "  ['C C ( C ) ( N = C = O ) C 1 C C C C C 1'],\n",
       "  ['C 1 C N 2 C C N 3 C C N ( C C N C 1 ) P 2 3'],\n",
       "  ['O = S ( c 1 c c c c c 1 F ) c 1 c c c c c 1 F'],\n",
       "  ['O = C ( c 1 c c c c c 1 ) c 1 n c 2 n c c c c 2 o 1'],\n",
       "  ['O = [N+] ( [O-] ) c 1 c n c c c 1 O'],\n",
       "  ['C N ( C ) C C = N c 1 c c c c ( - c 2 n c 3 n c c c c 3 o 2 ) c 1'],\n",
       "  ['C C C C O C ( = O ) N C = C c 1 c c c ( O C C ( O ) C N C ( C ) C ) c c 1'],\n",
       "  ['C C O C ( = O ) N C = C c 1 c c c ( O C C ( O ) C N C ( C ) ( C ) C ) c c 1'],\n",
       "  ['C C C O C ( = O ) N c 1 c c c c ( N C ( = O ) / C = C ( / C ) O C ) c 1'],\n",
       "  ['C C C ( = O ) N c 1 c c c c ( N 2 C ( = O ) C C ( C ) ( O C ) O C 2 = O ) c 1'],\n",
       "  ['C C C C C ( = O ) N c 1 c c c c ( N ( C ) C ( = O ) C C ( C ) = O ) c 1'],\n",
       "  ['C N c 1 c c ( C ( = O ) O C ) c c ( N 2 C C C C C 2 ) c 1 N 1 C C C C C 1'],\n",
       "  ['C C C C C C C C C N c 1 c c c ( C ) c c 1 N'],\n",
       "  ['C N 1 C C C ( O C ( = O ) O c 2 c c c c ( [N+] ( = O ) [O-] ) c 2 ) N ( c 2 n n c ( C ( C ) ( C ) C ) s 2 ) C 1 = O'],\n",
       "  ['C O C 1 ( c 2 c c c c c 2 ) N = C C ( = O ) N ( C ) c 2 c c c ( [N+] ( = O ) [O-] ) c c 2 1'],\n",
       "  ['C O C ( = O ) c 1 n c ( C ) c ( C ) n c 1 O C'],\n",
       "  ['N C ( = O ) C S c 1 n n c ( C S ( = O ) ( = O ) C C 2 = C ( C ( = O ) O ) N 3 C ( = O ) C ( N ) [C@H] 3 S C 2 ) o 1'],\n",
       "  ['O = C ( O ) c 1 c c c 2 c ( c 1 ) C C c 1 c c ( Br ) c c c 1 C 2 = O'],\n",
       "  ['O = C 1 O C C O C 1 c 1 c c c ( C O ) c c 1'],\n",
       "  ['C = C C O P ( = O ) ( C C Cl ) O C C'],\n",
       "  ['Cl c 1 c c c ( - n 2 n c 3 c ( c 2 Br ) C S C C 3 ) c c 1'],\n",
       "  ['O = C ( O ) C C ( = O ) N c 1 c c c c c 1 S ( = O ) ( = O ) N C 1 C C C C C 1'],\n",
       "  ['C n 1 n c ( - c 2 c c c ( Cl ) c c 2 ) c 2 c c 3 c c c c c 3 c c 2 1'],\n",
       "  ['C = C ( C ) C C ( C ( = O ) O C ( C # N ) c 1 c c c c ( O c 2 c c c c c 2 ) c 1 ) C ( C ) C'],\n",
       "  ['O = C ( O ) c 1 c ( I ) c ( I ) c ( C ( = O ) O ) c ( I ) c 1 I'],\n",
       "  ['O = C ( O ) c 1 c ( I ) c ( I ) c ( C ( = O ) O ) c ( I ) c 1 I'],\n",
       "  ['C O C ( = O ) C 1 = C ( C ) N C ( C ) = C ( C ( = O ) O C c 2 c c c c ( Cl ) c 2 ) C 1 c 1 c c c c ( N ) c 1'],\n",
       "  ['C c 1 c c ( F ) c c c 1 - c 1 c c c ( O ) c c 1'],\n",
       "  ['C O c 1 c c ( F ) c c c 1 - c 1 c c c ( O C ( C ) = O ) c ( O c 2 c c c c c 2 ) c 1'],\n",
       "  ['C = C C O c 1 c c c ( - c 2 c c c ( F ) c c 2 ) c c 1 C ( = O ) O C'],\n",
       "  ['C C S N ( C ) C ( = O ) N c 1 c c c ( Cl ) c ( Cl ) c 1'],\n",
       "  ['C C ( C ) ( C ) O c 1 c c c ( C ( C ) ( C ) C ) c c 1'],\n",
       "  ['C C C C C C C S C ( C C C n 1 c c n c 1 ) c 1 c c c ( Cl ) c c 1'],\n",
       "  ['C N 1 C C O C 2 ( C C 1 ) C c 1 c c c c c 1 2'],\n",
       "  ...])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from onmt.translate.translator import build_translator\n",
    "from onmt.new_opts  import TranslateOpts\n",
    "import onmt.inputters\n",
    "import onmt.translate\n",
    "import onmt\n",
    "import onmt.model_builder\n",
    "import onmt.modules\n",
    "import onmt.opts\n",
    "\n",
    "opt = TranslateOpts(\n",
    "   gpu=-1,\n",
    "    models= [\"demo_model_step_29000.pt\"],\n",
    "    src=\"data/USP_src-valid.txt\", \n",
    "    tgt=\"data/USP_tgt-valid.txt\", \n",
    "    replace_unk=True, \n",
    "    verbose=True,\n",
    "    output=\"data/USP_demo_pred.txt\",\n",
    "          )\n",
    "translator = build_translator(opt, report_score=True)\n",
    "translator.translate(src_path=opt.src,\n",
    "                     tgt_path=opt.tgt,\n",
    "                     src_dir=opt.src_dir,\n",
    "                     batch_size=opt.batch_size,\n",
    "                     attn_debug=opt.attn_debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = Opts(\n",
    "        encoder_type=\"transformer\", \n",
    "        decoder_type=\"transformer\",\n",
    "        layers=4,\n",
    "        norma\n",
    "        heads=4, \n",
    "        rnn_size=256, \n",
    "        word_vec_size=256, \n",
    "        transformer_ff=256,\n",
    "        data=\"data/USP\", \n",
    "        save_model=\"CNN\", \n",
    "        epochs=10, \n",
    "        batch_size=64, \n",
    "        gpu_ranks=[0], \n",
    "        valid_steps=100, \n",
    "        valid_batch_size=64, \n",
    "        decay_steps=1000, \n",
    "        start_decay_steps=10000, \n",
    "        learning_rate_decay=0.9, \n",
    "        optim=\"adam\",\n",
    "        learning_rate=0.001,\n",
    "        save_checkpoint_steps=1000,\n",
    "        \n",
    "          )\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Sci",
   "language": "python",
   "name": "sci"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}