Deep Learning- RNN, LSTM, CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:04.688331Z",
     "start_time": "2020-10-19T12:37:02.812675Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "import sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:05.195896Z",
     "start_time": "2020-10-19T12:37:04.690135Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>Probably my all-time favorite movie, a story o...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>I sure would like to see a resurrection of a u...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>This show was an amazing, fresh &amp; innovative i...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Encouraged by the positive comments about this...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>If you like original gut wrenching laughter yo...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  One of the other reviewers has mentioned that ...  positive\n",
       "1  A wonderful little production. <br /><br />The...  positive\n",
       "2  I thought this was a wonderful way to spend ti...  positive\n",
       "3  Basically there's a family where a little boy ...  negative\n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...  positive\n",
       "5  Probably my all-time favorite movie, a story o...  positive\n",
       "6  I sure would like to see a resurrection of a u...  positive\n",
       "7  This show was an amazing, fresh & innovative i...  negative\n",
       "8  Encouraged by the positive comments about this...  negative\n",
       "9  If you like original gut wrenching laughter yo...  positive"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/IMDB Dataset.csv\")\n",
    "print (data.shape)\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:05.209664Z",
     "start_time": "2020-10-19T12:37:05.197540Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "review       0\n",
       "sentiment    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:05.222925Z",
     "start_time": "2020-10-19T12:37:05.212237Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    25000\n",
       "negative    25000\n",
       "Name: sentiment, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sentiment.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:05.236258Z",
     "start_time": "2020-10-19T12:37:05.229835Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_html(text):\n",
    "    bs = BeautifulSoup(text, \"html.parser\")\n",
    "    return ' ' + bs.get_text() + ' '\n",
    "\n",
    "def keep_only_letters(text):\n",
    "    text=re.sub(r'[^a-zA-Z\\s]',' ',text)\n",
    "    return text\n",
    "\n",
    "def convert_to_lowercase(text):\n",
    "    return text.lower()\n",
    "\n",
    "def clean_reviews(text):\n",
    "    text = remove_html(text)\n",
    "    text = keep_only_letters(text)\n",
    "    text = convert_to_lowercase(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:12.904296Z",
     "start_time": "2020-10-19T12:37:05.238413Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['review'] = data['review'].apply(lambda review: clean_reviews(review))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:13.638559Z",
     "start_time": "2020-10-19T12:37:12.906283Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['review_word_length'] = [len(review.split()) for review in data['review']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:14.559963Z",
     "start_time": "2020-10-19T12:37:13.639910Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEICAYAAACeSMncAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAeRElEQVR4nO3df5QdZZ3n8ffHID8UkATaGJJgggZm\nI6sBmh9HBZlFQgA14FEMOhIZhugBjnJ0VgO6C6uyi47gmNHBCZIlcfghCkiUMNhkFXRGIB2M+QHE\nNCEs3TRJS9AgsMHgd/+o55JKc7u7utL3Xm7u53VOna761lNVz3Mr3C/PU3WrFBGYmZmV8ZpGV8DM\nzJqXk4iZmZXmJGJmZqU5iZiZWWlOImZmVpqTiJmZleYkYrssSZdJ+tcB1p0gqbvedUrHHrBeBbf/\nhaS/S/Mfk/SzEazbGkknpPmdqmeVfV8i6XsjtT97dXASsbqQdLGkO/vF1g0Qm1Xf2tVOrZNVRFwf\nEdML1OM6SV8tsL+3RcQvdrZe1dodEf8zIv5uZ/dtry5OIlYv9wLvlDQKQNI44LXA4f1ib01lC1PG\n/5Z3gqTdGl0Ha07+D8/qZRlZ0piWlo8Dfg6s7Rd7NCKeBJD0TknLJP0x/X1nZWdpSOdySf8OPA8c\nLGmypHskPSupAzigaOUkHSjpFkl9kh6T9Oncussk3SxpUdr3GkntufVHSPpNWvdDST+Q9FVJrwfu\nBA6U9Kc0HZg2232g/VWp20mSHkmfw7cB5dZ9QtKv0rwkfVPSJklbJK2SdJikOcDHgM+nOvwkld8g\n6QuSVgLPSdotxd6bO/yeqT3PSnpQ0jtyxw5Jb80tXzdYu/sPj0n6QGr7H9L5/E+5dRsk/b2kland\nP5C0Z9HzafXjJGJ1EREvAvcDx6fQ8cAvgV/1i90LIGkMcAcwD9gfuAq4Q9L+ud1+HJgD7AM8DtwA\nLCdLHl8BZhepW+rF/AT4LTAeOBG4SNLJuWIfAG4C9gMWA99O2+4O3AZcB4wBbgTOSG1+DjgFeDIi\n9k7Tk4Ptr0rdDgBuBb6U2vUo8K4BmjKd7DM8BHgDcCbwdETMB64Hvp7q8P7cNmcBpwH7RcS2Kvuc\nCfwwte0G4MeSXjvA8SnQ7kq7DiH7rC4C2oAlwE/S51lxJjADmAy8HfjEYMe1xnASsXq6h+0J4ziy\nJPLLfrF70vxpwLqI+H5EbIuIG4FHgPwX4HURsSZ9+Y0DjgL+W0RsjYh7yRJDEUcBbRHx5Yh4MSLW\nA9cA+Wszv4qIJRHxEvB9oPJ/5McCuwHzIuLPEXEr8ECBYw60v/5OBdZExI8i4s/APwJPDVD2z2QJ\n9a8ARcTDEdE7RD3mRcQTEfHCAOuX5459FbAnWZt31keAOyKiI+37G8BewDtzZeZFxJMRsZnsXE6r\nsh9rMCcRq6d7gXenXkZbRKwD/oPsWskY4DC2Xw85kKx3kfc4WU+h4onc/IHAM+n/gvPli3gz2dDL\nHyoTcAkwNlcm/8X9PNkwz27puD2x45NM8/UayED76+/A/P7ScaruPyL+D1mP5jvAJknzJe07RD2G\nqmv+2H8BulOddtYO5zft+wl2PL/9P6O9R+C4NsKcRKyefk02zHIe8O8AEbEFeDLFnoyIx1LZJ8m+\n3PMOAnpyy/kv7l5gdBqPz5cv4gngsYjYLzftExGnFti2FxgvSbnYxAHqWEZvfn/pOBMHKhwR8yLi\nSGAq2bDWfx2iHkPVL3/s1wATyM4NZF/sr8uVfdMw9rvD+c21q2fALexVyUnE6iYNmXQCnyUbxqr4\nVYrl78paAhwi6aPpgu9HyL4YfzrAvh9P+/4fknaX9G52HPoazAPAs+ki816SRqUL0kcV2PbXwEvA\nhameM4Gjc+s3AvtLekPBuvR3B/A2SR9MPZVPs+OX9cskHSXpmHTN4jng/wF/ydXj4BLHPzJ37IuA\nrcB9ad0K4KPp85oBvCe33VDtvhk4TdKJqb6fS/v+jxJ1tAZyErF6uwd4I1niqPhlir2cRCLiaeB9\nZF8uTwOfB94XEb8fZN8fBY4BNgOXAouKVChdl3gf2Zj7Y8Dvge+R9ZqG2vZF4IPAucAfgL8hS3Rb\n0/pHyC4gr09DZcMaCkrt/TBwBdnnMIXUi6tiX7JrOc+QDRU9DfxDWnctMDXV4cfDqMLtZNcvniG7\nkeGD6RoGwGfIEvUfyO7+enm/Q7U7ItaSfVb/RPZ5vx94f/o8rYnIL6UyG1mS7ge+GxH/u9F1Mas1\n90TMdpKk90h6UxrOmk12O+q/NbpeZvXgX6ma7bxDycb4Xw+sBz5U4NZas12Ch7PMzKw0D2eZmVlp\nLTecdcABB8SkSZMaXQ0zs6ayfPny30dEW/94zZKIpIlkt1iOJfvh0fyI+Fb6ZfIPgEnABuDMiHgm\n/djoW2SPeXge+EREPJj2NZvs2UEAX42IhSl+JNkzi/Yi+13BZ2KI8blJkybR2dk5gi01M9v1Sar6\nBIhaDmdtAz4XEVPJnrVzgaSpwFxgaURMAZamZcge2DYlTXOAq1PFx5Dd838M2Y+4LpU0Om1zNdkv\nnSvbzahhe8zMrJ+aJZGI6K30JCLiWeBhsufizAQWpmILgdPT/ExgUWTuA/ZT9n6Jk4GOiNgcEc8A\nHcCMtG7fiLgv9T4W5fZlZmZ1UJcL65ImAYeTPQp8bO72x6fY/pC78ez4MLjuFBss3l0lXu34cyR1\nSurs6+vbqbaYmdl2NU8ikvYGbgEuSg/be1nqQdT8HuOImB8R7RHR3tb2iutCZmZWUk2TSHqw2i3A\n9ek9CwAb01BU5XWom1K8hx2fTjohxQaLT6gSNzOzOqlZEkl3W10LPBwRV+VWLWb7G+dmkz3grRI/\nO73i81jgj2nY6y5guqTR6YL6dOCutG6LpGPTsc7O7cvMzOqglr8TeRfZUz9XSVqRYpeQPY30Zknn\nkj1p9My0bgnZ7b1dZLf4ngMQEZslfYXsHd0AX05vOgM4n+23+N6ZJjMzq5OWe+xJe3t7+HciZmbD\nI2l5RLT3j/uxJ2ZmVlrLPfakHibNvaNQuQ1XnFbjmpiZ1ZZ7ImZmVpqTiJmZleYkYmZmpTmJmJlZ\naU4iZmZWmpOImZmV5iRiZmalOYmYmVlpTiJmZlaak4iZmZXmJGJmZqU5iZiZWWlOImZmVpqTiJmZ\nleYkYmZmpTmJmJlZaTVLIpIWSNokaXUu9gNJK9K0ofLudUmTJL2QW/fd3DZHSlolqUvSPElK8TGS\nOiStS39H16otZmZWXS17ItcBM/KBiPhIREyLiGnALcCtudWPVtZFxKdy8auB84Apaarscy6wNCKm\nAEvTspmZ1VHNkkhE3AtsrrYu9SbOBG4cbB+SxgH7RsR9ERHAIuD0tHomsDDNL8zFzcysThp1TeQ4\nYGNErMvFJkv6jaR7JB2XYuOB7lyZ7hQDGBsRvWn+KWDsQAeTNEdSp6TOvr6+EWqCmZk1KomcxY69\nkF7goIg4HPgscIOkfYvuLPVSYpD18yOiPSLa29raytbZzMz62a3eB5S0G/BB4MhKLCK2AlvT/HJJ\njwKHAD3AhNzmE1IMYKOkcRHRm4a9NtWj/mZmtl0jeiLvBR6JiJeHqSS1SRqV5g8mu4C+Pg1XbZF0\nbLqOcjZwe9psMTA7zc/Oxc3MrE5qeYvvjcCvgUMldUs6N62axSsvqB8PrEy3/P4I+FREVC7Knw98\nD+gCHgXuTPErgJMkrSNLTFfUqi1mZlZdzYazIuKsAeKfqBK7heyW32rlO4HDqsSfBk7cuVqamdnO\n8C/WzcysNCcRMzMrzUnEzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9KcRMzMrDQnETMz\nK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnEzMxKcxIxM7PSnETMzKw0JxEzMyutlu9YXyBpk6TV\nudhlknokrUjTqbl1F0vqkrRW0sm5+IwU65I0NxefLOn+FP+BpN1r1RYzM6uulj2R64AZVeLfjIhp\naVoCIGkqMAt4W9rmnyWNkjQK+A5wCjAVOCuVBfha2tdbgWeAc2vYFjMzq6JmSSQi7gU2Fyw+E7gp\nIrZGxGNAF3B0mroiYn1EvAjcBMyUJOC/AD9K2y8ETh/RBpiZ2ZAacU3kQkkr03DX6BQbDzyRK9Od\nYgPF9wf+EBHb+sWrkjRHUqekzr6+vpFqh5lZy6t3ErkaeAswDegFrqzHQSNifkS0R0R7W1tbPQ5p\nZtYSdqvnwSJiY2Ve0jXAT9NiDzAxV3RCijFA/GlgP0m7pd5IvryZmdVJXXsiksblFs8AKnduLQZm\nSdpD0mRgCvAAsAyYku7E2p3s4vviiAjg58CH0vazgdvr0QYzM9uuZj0RSTcCJwAHSOoGLgVOkDQN\nCGAD8EmAiFgj6WbgIWAbcEFEvJT2cyFwFzAKWBARa9IhvgDcJOmrwG+Aa2vVFjMzq65mSSQizqoS\nHvCLPiIuBy6vEl8CLKkSX09295aZmTWIf7FuZmalOYmYmVlpTiJmZlaak4iZmZXmJGJmZqU5iZiZ\nWWlOImZmVpqTiJmZleYkYmZmpTmJmJlZaU4iZmZWmpOImZmV5iRiZmalOYmYmVlpTiJmZlaak4iZ\nmZXmJGJmZqXVLIlIWiBpk6TVudg/SHpE0kpJt0naL8UnSXpB0oo0fTe3zZGSVknqkjRPklJ8jKQO\nSevS39G1aouZmVVXy57IdcCMfrEO4LCIeDvwO+Di3LpHI2Jamj6Vi18NnAdMSVNln3OBpRExBVia\nls3MrI5qlkQi4l5gc7/YzyJiW1q8D5gw2D4kjQP2jYj7IiKARcDpafVMYGGaX5iLm5lZnTTymsjf\nAnfmlidL+o2keyQdl2Ljge5cme4UAxgbEb1p/ilg7EAHkjRHUqekzr6+vhGqvpmZFUoikv7zSB5U\n0heBbcD1KdQLHBQRhwOfBW6QtG/R/aVeSgyyfn5EtEdEe1tb207U3MzM8or2RP5Z0gOSzpf0hp05\noKRPAO8DPpa+/ImIrRHxdJpfDjwKHAL0sOOQ14QUA9iYhrsqw16bdqZeZmY2fIWSSEQcB3wMmAgs\nl3SDpJOGezBJM4DPAx+IiOdz8TZJo9L8wWQX0Nen4aotko5Nd2WdDdyeNlsMzE7zs3NxMzOrk8LX\nRCJiHfAl4AvAe4B56XbdD1YrL+lG4NfAoZK6JZ0LfBvYB+jodyvv8cBKSSuAHwGfiojKRfnzge8B\nXWQ9lMp1lCuAkyStA96bls3MrI52K1JI0tuBc4DTyG7TfX9EPCjpQLJEcWv/bSLirCq7urba/iPi\nFuCWAdZ1AodViT8NnFik/mZmVhuFkgjwT2S9gUsi4oVKMCKelPSlmtTMzMxe9YomkdOAFyLiJQBJ\nrwH2jIjnI+L7NaudmZm9qhW9JnI3sFdu+XUpZmZmLaxoEtkzIv5UWUjzr6tNlczMrFkUTSLPSTqi\nsiDpSOCFQcqbmVkLKHpN5CLgh5KeBAS8CfhIzWplZmZNoVASiYhlkv4KODSF1kbEn2tXLTMzawZF\neyIARwGT0jZHSCIiFtWkVmZm1hSK/tjw+8BbgBXASylceTS7mZm1qKI9kXZgauWBiWZmZlA8iawm\nu5jeO1RBK27S3DsKldtwxWk1romZWTlFk8gBwEOSHgC2VoIR8YGa1MrMzJpC0SRyWS0rYWZmzano\nLb73SHozMCUi7pb0OmBUbatmZmavdkVfj3se2Xs+/iWFxgM/rlWlzMysORR97MkFwLuALfDyC6re\nWKtKmZlZcyiaRLZGxIuVBUm7kf1OxMzMWljRJHKPpEuAvdK71X8I/KR21TIzs2ZQNInMBfqAVcAn\ngSVk71sflKQFkjZJWp2LjZHUIWld+js6xSVpnqQuSSv7PTV4diq/TtLsXPxISavSNvMkqWB7zMxs\nBBRKIhHxl4i4JiI+HBEfSvNFhrOuA2b0i80FlkbEFGBpWgY4BZiSpjnA1ZAlHeBS4BjgaODSSuJJ\nZc7Lbdf/WGZmVkNF7856TNL6/tNQ20XEvcDmfuGZwMI0vxA4PRdfFJn7gP0kjQNOBjoiYnNEPAN0\nADPSun0j4r6U0Bbl9mVmZnUwnGdnVewJfBgYU/KYYyOi8viUp4CxaX488ESuXHeKDRbvrhJ/BUlz\nyHo3HHTQQSWrbWZm/RUdzno6N/VExD8CO/1Ap9SDqPldXhExPyLaI6K9ra2t1oczM2sZRR8Ff0Ru\n8TVkPZPhvIskb6OkcRHRm4akNqV4DzAxV25CivUAJ/SL/yLFJ1Qpb2ZmdVL07qwrc9P/Ao4Ezix5\nzMVA5Q6r2cDtufjZ6S6tY4E/pmGvu4DpkkanC+rTgbvSui2Sjk13ZZ2d25eZmdVB0Wdn/XWZnUu6\nkawXcYCkbrK7rK4AbpZ0LvA425PREuBUoAt4HjgnHXuzpK8Ay1K5L0dE5WL9+WR3gO0F3JkmMzOr\nk6LDWZ8dbH1EXDVA/KwBNjmxStkge7xKtf0sABZUiXcChw1WNzMzq53h3J11FNmQE8D7gQeAdbWo\nlJmZNYeiSWQCcEREPAsg6TLgjoj4m1pVzMzMXv2KXlgfC7yYW36R7b/vMDOzFlW0J7IIeEDSbWn5\ndLb/6tzMzFpU0buzLpd0J3BcCp0TEb+pXbXMzKwZFB3OAngdsCUivgV0S5pcozqZmVmTKPoAxkuB\nLwAXp9BrgX+tVaXMzKw5FO2JnAF8AHgOICKeBPapVaXMzKw5FE0iL+Yflijp9bWrkpmZNYuiSeRm\nSf9C9o6P84C7gWtqVy0zM2sGRe/O+kZ6t/oW4FDgv0dER01rZmZmr3pDJhFJo4C700MYnTjMzOxl\nQw5nRcRLwF8kvaEO9TEzsyZS9BfrfwJWSeog3aEFEBGfrkmtzMysKRRNIremyczM7GWDJhFJB0XE\n/40IPyfLzMxeYahrIj+uzEi6pcZ1MTOzJjNUElFu/uBaVsTMzJrPUEkkBpgvTdKhklbkpi2SLpJ0\nmaSeXPzU3DYXS+qStFbSybn4jBTrkjR3JOpnZmbFDXVh/R2StpD1SPZK86TliIh9h3vAiFgLTIOX\nf4PSA9wGnAN8MyK+kS8vaSowC3gbcCBwt6RD0urvACcB3cAySYsj4qHh1snMzMoZNIlExKgaH/9E\n4NGIeFzSQGVmAjdFxFbgMUldwNFpXVdErAeQdFMq6yRiZlYnw3mfSC3MAm7MLV8oaaWkBZJGp9h4\n4Ilcme4UGyj+CpLmSOqU1NnX1zdytTcza3ENSyKSdid7vPwPU+hq4C1kQ129wJUjdayImB8R7RHR\n3tbWNlK7NTNreUV/bFgLpwAPRsRGgMpfAEnXAD9Niz3AxNx2E1KMQeJmZlYHjRzOOovcUJakcbl1\nZwCr0/xiYJakPdIreacADwDLgCmSJqdezaxU1szM6qQhPZH0UquTgE/mwl+XNI3sVuINlXURsUbS\nzWQXzLcBF6SHQiLpQuAuYBSwICLW1K0RZmbWmCQSEc8B+/eLfXyQ8pcDl1eJLwGWjHgFzcyskEbf\nnWVmZk3MScTMzEpzEjEzs9KcRMzMrDQnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnEzMxK\ncxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9KcRMzMrLRGvmPdCpo0947CZTdccVoNa2JmtqOG\n9UQkbZC0StIKSZ0pNkZSh6R16e/oFJekeZK6JK2UdERuP7NT+XWSZjeqPWZmrajRw1l/HRHTIqI9\nLc8FlkbEFGBpWgY4BZiSpjnA1ZAlHeBS4BjgaODSSuIxM7Paa3QS6W8msDDNLwROz8UXReY+YD9J\n44CTgY6I2BwRzwAdwIx6V9rMrFU1MokE8DNJyyXNSbGxEdGb5p8Cxqb58cATuW27U2yg+A4kzZHU\nKamzr69vJNtgZtbSGnlh/d0R0SPpjUCHpEfyKyMiJMVIHCgi5gPzAdrb20dkn2Zm1sAkEhE96e8m\nSbeRXdPYKGlcRPSm4apNqXgPMDG3+YQU6wFO6Bf/Ra3qPJy7pMzMWkFDhrMkvV7SPpV5YDqwGlgM\nVO6wmg3cnuYXA2enu7SOBf6Yhr3uAqZLGp0uqE9PMTMzq4NG9UTGArdJqtThhoj4N0nLgJslnQs8\nDpyZyi8BTgW6gOeBcwAiYrOkrwDLUrkvR8Tm+jXDzKy1NSSJRMR64B1V4k8DJ1aJB3DBAPtaACwY\n6TqamdnQXm23+JqZWRNxEjEzs9KcRMzMrDQnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnE\nzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpr5JsNrQaKvjhrwxWn1bgmZtYK3BMxM7PSnETMzKw0\nJxEzMyvNScTMzEqrexKRNFHSzyU9JGmNpM+k+GWSeiStSNOpuW0ultQlaa2kk3PxGSnWJWluvdti\nZtbqGnF31jbgcxHxoKR9gOWSOtK6b0bEN/KFJU0FZgFvAw4E7pZ0SFr9HeAkoBtYJmlxRDxUl1aY\nmVn9k0hE9AK9af5ZSQ8D4wfZZCZwU0RsBR6T1AUcndZ1RcR6AEk3pbJOImZmddLQayKSJgGHA/en\n0IWSVkpaIGl0io0Hnsht1p1iA8WrHWeOpE5JnX19fSPYAjOz1tawJCJpb+AW4KKI2AJcDbwFmEbW\nU7lypI4VEfMjoj0i2tva2kZqt2ZmLa8hv1iX9FqyBHJ9RNwKEBEbc+uvAX6aFnuAibnNJ6QYg8TN\nzKwOGnF3loBrgYcj4qpcfFyu2BnA6jS/GJglaQ9Jk4EpwAPAMmCKpMmSdie7+L64Hm0wM7NMI3oi\n7wI+DqyStCLFLgHOkjQNCGAD8EmAiFgj6WayC+bbgAsi4iUASRcCdwGjgAURsaaeDTEza3WKiEbX\noa7a29ujs7Oz1LZFH264K/GDGs0MQNLyiGjvH/cv1s3MrDQnETMzK81JxMzMSnMSMTOz0pxEzMys\nNCcRMzMrzUnEzMxKa8hjT6x5FP1tjH9PYtaa3BMxM7PSnETMzKw0JxEzMyvN10RsRPjaiVlrck/E\nzMxKcxIxM7PSPJxldeVhL7Ndi3siZmZWmnsi9qrkHotZc3BPxMzMSmv6noikGcC3yN6z/r2IuKLB\nVbI6qsUri927MSuuqZOIpFHAd4CTgG5gmaTFEfFQY2tmzcxDaWbFNftw1tFAV0Ssj4gXgZuAmQ2u\nk5lZy2jqnggwHngit9wNHNO/kKQ5wJy0+CdJa0sc6wDg9yW2a3at2O5CbdbX6lCT+mnF8wyt2e6y\nbX5ztWCzJ5FCImI+MH9n9iGpMyLaR6hKTaMV2+02t45WbPdIt7nZh7N6gIm55QkpZmZmddDsSWQZ\nMEXSZEm7A7OAxQ2uk5lZy2jq4ayI2CbpQuAuslt8F0TEmhodbqeGw5pYK7bbbW4drdjuEW2zImIk\n92dmZi2k2YezzMysgZxEzMysNCeRAiTNkLRWUpekuY2uz0iStEHSKkkrJHWm2BhJHZLWpb+jU1yS\n5qXPYaWkIxpb++IkLZC0SdLqXGzY7ZQ0O5VfJ2l2I9pS1ABtvkxSTzrfKySdmlt3cWrzWkkn5+JN\n8+9f0kRJP5f0kKQ1kj6T4rvsuR6kzfU51xHhaZCJ7IL9o8DBwO7Ab4Gpja7XCLZvA3BAv9jXgblp\nfi7wtTR/KnAnIOBY4P5G138Y7TweOAJYXbadwBhgffo7Os2PbnTbhtnmy4C/r1J2avq3vQcwOf2b\nH9Vs//6BccARaX4f4HepbbvsuR6kzXU51+6JDK0VH60yE1iY5hcCp+fiiyJzH7CfpHGNqOBwRcS9\nwOZ+4eG282SgIyI2R8QzQAcwo/a1L2eANg9kJnBTRGyNiMeALrJ/+0317z8ieiPiwTT/LPAw2ZMt\ndtlzPUibBzKi59pJZGjVHq0y2AlqNgH8TNLy9HgYgLER0ZvmnwLGpvld7bMYbjt3lfZfmIZuFlSG\nddgF2yxpEnA4cD8tcq77tRnqcK6dROzdEXEEcApwgaTj8ysj6//u8veBt0o7gauBtwDTgF7gysZW\npzYk7Q3cAlwUEVvy63bVc12lzXU5104iQ9ulH60SET3p7ybgNrIu7cbKMFX6uykV39U+i+G2s+nb\nHxEbI+KliPgLcA3Z+YZdqM2SXkv2ZXp9RNyawrv0ua7W5nqdayeRoe2yj1aR9HpJ+1TmgenAarL2\nVe5GmQ3cnuYXA2enO1qOBf6YGyJoRsNt513AdEmj09DA9BRrGv2uYZ1Bdr4ha/MsSXtImgxMAR6g\nyf79SxJwLfBwRFyVW7XLnuuB2ly3c93oOwuaYSK7g+N3ZHcufLHR9RnBdh1MdgfGb4E1lbYB+wNL\ngXXA3cCYFBfZS8AeBVYB7Y1uwzDaeiNZl/7PZGO955ZpJ/C3ZBciu4BzGt2uEm3+fmrTyvQFMS5X\n/oupzWuBU3Lxpvn3D7ybbKhqJbAiTafuyud6kDbX5Vz7sSdmZlaah7PMzKw0JxEzMyvNScTMzEpz\nEjEzs9KcRMzMrDQnETMzK81JxMzMSvv/0cS/PoDXOWgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['review_word_length'] = [len(review.split()) for review in data['review']]\n",
    "data['review_word_length'].plot(kind='hist', bins=30)\n",
    "plt.title('Word length distribution')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting into Training and Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:14.564778Z",
     "start_time": "2020-10-19T12:37:14.561465Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imdb_train = data[:40000]\n",
    "imdb_test = data[40000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:16.939263Z",
     "start_time": "2020-10-19T12:37:14.567853Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(92279, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>the</td>\n",
       "      <td>533072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>and</td>\n",
       "      <td>259694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>a</td>\n",
       "      <td>258257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>of</td>\n",
       "      <td>230754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>to</td>\n",
       "      <td>213961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    key   value\n",
       "2   the  533072\n",
       "35  and  259694\n",
       "47    a  258257\n",
       "1    of  230754\n",
       "58   to  213961"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "counter = Counter([words for reviews in imdb_train['review'] for words in reviews.split()])\n",
    "df = pd.DataFrame()\n",
    "df['key'] = counter.keys()\n",
    "df['value'] = counter.values()\n",
    "df.sort_values(by='value', ascending=False, inplace=True)\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:16.945792Z",
     "start_time": "2020-10-19T12:37:16.940836Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9477437089109378"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:10000].value.sum()/df.value.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:37:16.951287Z",
     "start_time": "2020-10-19T12:37:16.947528Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_10k_words = list(df[:10000].key.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input_Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:41:01.237138Z",
     "start_time": "2020-10-19T12:37:16.953155Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 500) (10000, 500)\n"
     ]
    }
   ],
   "source": [
    "def get_encoded_input(review):\n",
    "    words = review.split()\n",
    "    if len(words) > 500:\n",
    "        words = words[:500]\n",
    "    encoding = []\n",
    "    for word in words:\n",
    "        try:\n",
    "            index = top_10k_words.index(word)\n",
    "        except:\n",
    "            index = 10000\n",
    "        encoding.append(index)\n",
    "    while len(encoding) < 500:\n",
    "        encoding.append(10001)\n",
    "    return encoding\n",
    "\n",
    "training_data = np.array([get_encoded_input(review) for review in imdb_train['review']])\n",
    "testing_data = np.array([get_encoded_input(review) for review in imdb_test['review']])\n",
    "print (training_data.shape, testing_data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:41:01.252757Z",
     "start_time": "2020-10-19T12:41:01.238666Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000,) (10000,)\n"
     ]
    }
   ],
   "source": [
    "train_labels = [1 if sentiment=='positive' else 0 for sentiment in imdb_train['sentiment']]\n",
    "test_labels = [1 if sentiment=='positive' else 0 for sentiment in imdb_test['sentiment']]\n",
    "train_labels = np.array(train_labels)\n",
    "test_labels = np.array(test_labels)\n",
    "print (train_labels.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:41:03.507487Z",
     "start_time": "2020-10-19T12:41:01.254173Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 500)]             0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, 500, 32)           320064    \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 100)               8300      \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1)                 101       \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 328,465\n",
      "Trainable params: 328,465\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow\n",
    "from tensorflow.keras.layers import Activation\n",
    "\n",
    "input_data = tensorflow.keras.layers.Input(shape=(500))\n",
    "\n",
    "data = tensorflow.keras.layers.Embedding(input_dim=10002, output_dim=32, input_length=500)(input_data)\n",
    "\n",
    "data = tensorflow.keras.layers.Bidirectional(tensorflow.keras.layers.SimpleRNN(50))(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Dense(1)(data)\n",
    "output_data = tensorflow.keras.layers.Activation('sigmoid')(data)\n",
    "\n",
    "model = tensorflow.keras.models.Model(inputs=input_data, outputs=output_data)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics='accuracy')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T12:49:38.669043Z",
     "start_time": "2020-10-19T12:41:03.508784Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "157/157 [==============================] - 48s 305ms/step - loss: 0.6839 - accuracy: 0.5548 - val_loss: 0.6907 - val_accuracy: 0.5250\n",
      "Epoch 2/10\n",
      "157/157 [==============================] - 55s 352ms/step - loss: 0.6743 - accuracy: 0.5792 - val_loss: 0.6689 - val_accuracy: 0.5985\n",
      "Epoch 3/10\n",
      "157/157 [==============================] - 49s 311ms/step - loss: 0.6374 - accuracy: 0.6433 - val_loss: 0.6313 - val_accuracy: 0.6435\n",
      "Epoch 4/10\n",
      "157/157 [==============================] - 53s 339ms/step - loss: 0.5445 - accuracy: 0.7265 - val_loss: 0.5861 - val_accuracy: 0.6838\n",
      "Epoch 5/10\n",
      "157/157 [==============================] - 51s 323ms/step - loss: 0.3828 - accuracy: 0.8332 - val_loss: 0.4239 - val_accuracy: 0.8258\n",
      "Epoch 6/10\n",
      "157/157 [==============================] - 52s 334ms/step - loss: 0.2664 - accuracy: 0.8942 - val_loss: 0.5263 - val_accuracy: 0.7429\n",
      "Epoch 7/10\n",
      "157/157 [==============================] - 52s 334ms/step - loss: 0.2234 - accuracy: 0.9125 - val_loss: 0.5065 - val_accuracy: 0.7780\n",
      "Epoch 8/10\n",
      "157/157 [==============================] - 50s 321ms/step - loss: 0.1269 - accuracy: 0.9573 - val_loss: 0.4620 - val_accuracy: 0.8289\n",
      "Epoch 9/10\n",
      "157/157 [==============================] - 51s 325ms/step - loss: 0.0713 - accuracy: 0.9796 - val_loss: 0.5079 - val_accuracy: 0.8270\n",
      "Epoch 10/10\n",
      "157/157 [==============================] - 49s 310ms/step - loss: 0.0426 - accuracy: 0.9895 - val_loss: 0.5668 - val_accuracy: 0.8280\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x12dbb28d0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(training_data, train_labels, epochs=10, batch_size=256, validation_data=(testing_data, test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-13T14:40:44.700752Z",
     "start_time": "2020-10-13T14:40:44.180731Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_25\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_17 (InputLayer)        [(None, 500)]             0         \n",
      "_________________________________________________________________\n",
      "embedding_16 (Embedding)     (None, 500, 32)           320064    \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 100)               33200     \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 1)                 101       \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 353,365\n",
      "Trainable params: 353,365\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow\n",
    "from tensorflow.keras.layers import Activation\n",
    "\n",
    "input_data = tensorflow.keras.layers.Input(shape=(500))\n",
    "\n",
    "data = tensorflow.keras.layers.Embedding(input_dim=10002, output_dim=32, input_length=500)(input_data)\n",
    "\n",
    "data = tensorflow.keras.layers.Bidirectional(tensorflow.keras.layers.LSTM(50))(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Dense(1)(data)\n",
    "output_data = tensorflow.keras.layers.Activation('sigmoid')(data)\n",
    "\n",
    "model = tensorflow.keras.models.Model(inputs=input_data, outputs=output_data)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics='accuracy')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-13T15:07:32.037299Z",
     "start_time": "2020-10-13T14:40:46.595546Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "313/313 [==============================] - 140s 449ms/step - loss: 0.5606 - accuracy: 0.7155 - val_loss: 0.4979 - val_accuracy: 0.7681\n",
      "Epoch 2/10\n",
      "313/313 [==============================] - 142s 453ms/step - loss: 0.3372 - accuracy: 0.8625 - val_loss: 0.3041 - val_accuracy: 0.8714\n",
      "Epoch 3/10\n",
      "313/313 [==============================] - 142s 455ms/step - loss: 0.2705 - accuracy: 0.8979 - val_loss: 0.3145 - val_accuracy: 0.8777\n",
      "Epoch 4/10\n",
      "313/313 [==============================] - 155s 496ms/step - loss: 0.2132 - accuracy: 0.9235 - val_loss: 0.3785 - val_accuracy: 0.8484\n",
      "Epoch 5/10\n",
      "313/313 [==============================] - 153s 488ms/step - loss: 0.2182 - accuracy: 0.9210 - val_loss: 0.3360 - val_accuracy: 0.8748\n",
      "Epoch 6/10\n",
      "313/313 [==============================] - 159s 507ms/step - loss: 0.1941 - accuracy: 0.9291 - val_loss: 0.3379 - val_accuracy: 0.8725\n",
      "Epoch 7/10\n",
      "313/313 [==============================] - 181s 577ms/step - loss: 0.1592 - accuracy: 0.9448 - val_loss: 0.3238 - val_accuracy: 0.8800\n",
      "Epoch 8/10\n",
      "313/313 [==============================] - 164s 523ms/step - loss: 0.1471 - accuracy: 0.9496 - val_loss: 0.3347 - val_accuracy: 0.8791\n",
      "Epoch 9/10\n",
      "313/313 [==============================] - 166s 531ms/step - loss: 0.1371 - accuracy: 0.9535 - val_loss: 0.3633 - val_accuracy: 0.8805\n",
      "Epoch 10/10\n",
      "313/313 [==============================] - 196s 628ms/step - loss: 0.1229 - accuracy: 0.9598 - val_loss: 0.4345 - val_accuracy: 0.8161\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x13730acf8>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(training_data, train_labels, epochs=10, batch_size=128, validation_data=(testing_data, test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T13:28:48.017530Z",
     "start_time": "2020-10-19T13:28:47.887932Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         [(None, 500)]             0         \n",
      "_________________________________________________________________\n",
      "embedding_4 (Embedding)      (None, 500, 32)           320064    \n",
      "_________________________________________________________________\n",
      "conv1d_9 (Conv1D)            (None, 498, 50)           4850      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_9 (MaxPooling1 (None, 249, 50)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_10 (Conv1D)           (None, 247, 40)           6040      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_10 (MaxPooling (None, 123, 40)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_11 (Conv1D)           (None, 121, 30)           3630      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling (None, 60, 30)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 58, 30)            2730      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_12 (MaxPooling (None, 29, 30)            0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 870)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 20)                17420     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 20)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 1)                 21        \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 354,755\n",
      "Trainable params: 354,755\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow\n",
    "\n",
    "input_data = tensorflow.keras.layers.Input(shape=(500))\n",
    "\n",
    "data = tensorflow.keras.layers.Embedding(input_dim=10002, output_dim=32, input_length=500)(input_data)\n",
    "\n",
    "data = tensorflow.keras.layers.Conv1D(50, kernel_size=3, activation='relu')(data)\n",
    "data = tensorflow.keras.layers.MaxPool1D(pool_size=2)(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Conv1D(40, kernel_size=3, activation='relu')(data)\n",
    "data = tensorflow.keras.layers.MaxPool1D(pool_size=2)(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Conv1D(30, kernel_size=3, activation='relu')(data)\n",
    "data = tensorflow.keras.layers.MaxPool1D(pool_size=2)(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Conv1D(30, kernel_size=3, activation='relu')(data)\n",
    "data = tensorflow.keras.layers.MaxPool1D(pool_size=2)(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Flatten()(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Dense(20)(data)\n",
    "data = tensorflow.keras.layers.Dropout(0.5)(data)\n",
    "\n",
    "data = tensorflow.keras.layers.Dense(1)(data)\n",
    "output_data = tensorflow.keras.layers.Activation('sigmoid')(data)\n",
    "\n",
    "model = tensorflow.keras.models.Model(inputs=input_data, outputs=output_data)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics='accuracy')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-19T13:33:41.004298Z",
     "start_time": "2020-10-19T13:28:49.310816Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "157/157 [==============================] - 28s 180ms/step - loss: 0.6393 - accuracy: 0.5771 - val_loss: 0.3717 - val_accuracy: 0.8485\n",
      "Epoch 2/10\n",
      "157/157 [==============================] - 27s 175ms/step - loss: 0.2787 - accuracy: 0.8913 - val_loss: 0.2644 - val_accuracy: 0.8974\n",
      "Epoch 3/10\n",
      "157/157 [==============================] - 28s 177ms/step - loss: 0.1885 - accuracy: 0.9319 - val_loss: 0.2611 - val_accuracy: 0.9004\n",
      "Epoch 4/10\n",
      "157/157 [==============================] - 30s 193ms/step - loss: 0.1398 - accuracy: 0.9523 - val_loss: 0.3670 - val_accuracy: 0.8766\n",
      "Epoch 5/10\n",
      "157/157 [==============================] - 30s 191ms/step - loss: 0.1009 - accuracy: 0.9664 - val_loss: 0.3944 - val_accuracy: 0.8888\n",
      "Epoch 6/10\n",
      "157/157 [==============================] - 29s 184ms/step - loss: 0.0651 - accuracy: 0.9788 - val_loss: 0.4523 - val_accuracy: 0.8902\n",
      "Epoch 7/10\n",
      "157/157 [==============================] - 29s 183ms/step - loss: 0.0435 - accuracy: 0.9866 - val_loss: 0.4840 - val_accuracy: 0.8872\n",
      "Epoch 8/10\n",
      "157/157 [==============================] - 28s 177ms/step - loss: 0.0261 - accuracy: 0.9919 - val_loss: 0.6312 - val_accuracy: 0.8873\n",
      "Epoch 9/10\n",
      "157/157 [==============================] - 29s 182ms/step - loss: 0.0287 - accuracy: 0.9911 - val_loss: 0.7780 - val_accuracy: 0.8845\n",
      "Epoch 10/10\n",
      "157/157 [==============================] - 31s 200ms/step - loss: 0.0107 - accuracy: 0.9973 - val_loss: 0.9143 - val_accuracy: 0.8835\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x12dfe90f0>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(training_data, train_labels, epochs=10, batch_size=256, validation_data=(testing_data, test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}