news_category.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "news category.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "nn0bOsDsRtk5",
        "outputId": "639c4345-91d6-44bd-877b-1bd9cd73582d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from nltk.corpus import stopwords \n",
        "from nltk import download\n",
        "import string\n",
        "import re\n",
        "from gensim.models import Word2Vec\n",
        "from collections import Counter\n",
        "from sklearn.model_selection import KFold, StratifiedKFold\n",
        "from sklearn.metrics import accuracy_score\n",
        "from fuzzywuzzy import fuzz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/fuzzywuzzy/fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
            "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nlduG32MblE_",
        "outputId": "dc9e4ed6-557c-4aa0-aa8f-5cbe9c55a251",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 743
        }
      },
      "source": [
        "try:\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n",
        "  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\n",
        "except ValueError:\n",
        "  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n",
        "\n",
        "tf.config.experimental_connect_to_cluster(tpu)\n",
        "tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running on TPU  ['10.116.253.138:8470']\n",
            "INFO:tensorflow:Initializing the TPU system: grpc://10.116.253.138:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.116.253.138:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dA8djeqTR7C3",
        "outputId": "eb2d0230-7aa4-4e63-9507-28d1124fc02a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "download('stopwords')\n",
        "stop_words = set(stopwords.words('english'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zlc4kqhhTCxh"
      },
      "source": [
        "df = pd.read_json('./drive/My Drive/edustack/news.json', orient = 'records', lines = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GbKteKkUTWBx",
        "outputId": "e063db8a-4e78-41a8-fb43-5054d322188f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['category', 'headline', 'authors', 'link', 'short_description', 'date'], dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJeyn32MSS5C"
      },
      "source": [
        "'''Some contractions, rules and token replace dictionaries which will help in text preprocessing'''\n",
        "contractions = {\n",
        "\"ain't\": \"is not\",\n",
        "\"aren't\": \"are not\",\n",
        "\"can't\": \"cannot\",\n",
        "\"can't've\": \"cannot have\",\n",
        "\"'cause\": \"because\",\n",
        "\"could've\": \"could have\",\n",
        "\"couldn't\": \"could not\",\n",
        "\"couldn't've\": \"could not have\",\n",
        "\"didn't\": \"did not\",\n",
        "\"doesn't\": \"does not\",\n",
        "\"don't\": \"do not\",\n",
        "\"hadn't\": \"had not\",\n",
        "\"hadn't've\": \"had not have\",\n",
        "\"hasn't\": \"has not\",\n",
        "\"haven't\": \"have not\",\n",
        "\"he'd\": \"he would\",\n",
        "\"he'd've\": \"he would have\",\n",
        "\"he'll\": \"he will\",\n",
        "\"he'll've\": \"he he will have\",\n",
        "\"he's\": \"he is\",\n",
        "\"how'd\": \"how did\",\n",
        "\"how'd'y\": \"how do you\",\n",
        "\"how'll\": \"how will\",\n",
        "\"how's\": \"how is\",\n",
        "\"I'd\": \"I would\",\n",
        "\"i'd've\": \"I would have\",\n",
        "\"i'll\": \"I will\",\n",
        "\"i'll've\": \"I will have\",\n",
        "\"i'm\": \"I am\",\n",
        "\"i've\": \"I have\",\n",
        "\"i'd\": \"i would\",\n",
        "\"i'd've\": \"i would have\",\n",
        "\"i'll\": \"i will\",\n",
        "\"i'll've\": \"i will have\",\n",
        "\"i'm\": \"i am\",\n",
        "\"i've\": \"i have\",\n",
        "\"isn't\": \"is not\",\n",
        "\"its\":\"it is\",\n",
        "\"it'd\": \"it would\",\n",
        "\"it'd've\": \"it would have\",\n",
        "\"it'll\": \"it will\",\n",
        "\"it'll've\": \"it will have\",\n",
        "\"it's\": \"it is\",\n",
        "\"let's\": \"let us\",\n",
        "\"ma'am\": \"madam\",\n",
        "\"mayn't\": \"may not\",\n",
        "\"might've\": \"might have\",\n",
        "\"mightn't\": \"might not\",\n",
        "\"mightn't've\": \"might not have\",\n",
        "\"must've\": \"must have\",\n",
        "\"mustn't\": \"must not\",\n",
        "\"mustn't've\": \"must not have\",\n",
        "\"needn't\": \"need not\",\n",
        "\"needn't've\": \"need not have\",\n",
        "\"o'clock\": \"of the clock\",\n",
        "\"oughtn't\": \"ought not\",\n",
        "\"oughtn't've\": \"ought not have\",\n",
        "\"shan't\": \"shall not\",\n",
        "\"sha'n't\": \"shall not\",\n",
        "\"shan't've\": \"shall not have\",\n",
        "\"she'd\": \"she would\",\n",
        "\"she'd've\": \"she would have\",\n",
        "\"she'll\": \"she will\",\n",
        "\"she'll've\": \"she will have\",\n",
        "\"she's\": \"she is\",\n",
        "\"should've\": \"should have\",\n",
        "\"shouldn't\": \"should not\",\n",
        "\"shouldn't've\": \"should not have\",\n",
        "\"so've\": \"so have\",\n",
        "\"so's\": \"so as\",\n",
        "\"that'd\": \"that would\",\n",
        "\"that'd've\": \"that would have\",\n",
        "\"that's\": \"that is\",\n",
        "\"there'd\": \"there would\",\n",
        "\"there'd've\": \"there would have\",\n",
        "\"there's\": \"there is\",\n",
        "\"they'd\": \"they would\",\n",
        "\"they'd've\": \"they would have\",\n",
        "\"they'll\": \"they will\",\n",
        "\"they'll've\": \"they will have\",\n",
        "\"they're\": \"they are\",\n",
        "\"they've\": \"they have\",\n",
        "\"to've\": \"to have\",\n",
        "\"wasn't\": \"was not\",\n",
        "\"we'd\": \"we would\",\n",
        "\"we'd've\": \"we would have\",\n",
        "\"we'll\": \"we will\",\n",
        "\"we'll've\": \"we will have\",\n",
        "\"we're\": \"we are\",\n",
        "\"we've\": \"we have\",\n",
        "\"weren't\": \"were not\",\n",
        "\"what'll\": \"what will\",\n",
        "\"what'll've\": \"what will have\",\n",
        "\"what're\": \"what are\",\n",
        "\"what's\": \"what is\",\n",
        "\"what've\": \"what have\",\n",
        "\"when's\": \"when is\",\n",
        "\"when've\": \"when have\",\n",
        "\"where'd\": \"where did\",\n",
        "\"where's\": \"where is\",\n",
        "\"where've\": \"where have\",\n",
        "\"who'll\": \"who will\",\n",
        "\"who'll've\": \"who will have\",\n",
        "\"who's\": \"who is\",\n",
        "\"who've\": \"who have\",\n",
        "\"why's\": \"why is\",\n",
        "\"why've\": \"why have\",\n",
        "\"will've\": \"will have\",\n",
        "\"won't\": \"will not\",\n",
        "\"won't've\": \"will not have\",\n",
        "\"would've\": \"would have\",\n",
        "\"wouldn't\": \"would not\",\n",
        "\"wouldn't've\": \"would not have\",\n",
        "\"y'all\": \"you all\",\n",
        "\"y'all'd\": \"you all would\",\n",
        "\"y'all'd've\": \"you all would have\",\n",
        "\"y'all're\": \"you all are\",\n",
        "\"y'all've\": \"you all have\",\n",
        "\"you'd\": \"you would\",\n",
        "\"you'd've\": \"you would have\",\n",
        "\"you'll\": \"you will\",\n",
        "\"you'll've\": \"you will have\",\n",
        "\"you're\": \"you are\",\n",
        "\"you've\": \"you have\"\n",
        "}\n",
        "\n",
        "rules = {\n",
        "    \"'t\": \" not\",\n",
        "    \"'cause\": \" because\",\n",
        "    \"'ve\": \" have\",\n",
        "    \"'t\": \" not\",\n",
        "    \"'s\": \" is\",\n",
        "    \"'d\": \" had\"\n",
        "}\n",
        "\n",
        "punctuations = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", \n",
        "                '$', '&', '/', '[', ']','>', '%', '=', '#', '*', '+', '\\\\', '•', '~', \n",
        "                '@', '£', '·', '_', '{', '}', '©', '^','®', '`', '<', '→', '°', '€', '™', \n",
        "                '›', '♥', '←', '×', '§', '″', '′', 'Â', '█','½', 'à', '…', '“', '★', '”', \n",
        "                '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶','↑', '±', '¿', '▾', '═', \n",
        "                '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼','⊕', '▼', '▪', '†',\n",
        "                '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲','è', '¸', '¾', \n",
        "                'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪','╩', \n",
        "                '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', \n",
        "                'ï', 'Ø', '¹', '≤', '‡', '√'] + list(string.punctuation)\n",
        "token_replace = {\n",
        "    'usepackage':'latex',\n",
        "    'orf19':'gene',\n",
        "    'documentclass':'latex',\n",
        "    'magento':'open-source e-commerce',\n",
        "    'appium':'web-app',\n",
        "    'tikz':'programming language',\n",
        "    'tikzpicture':'programming language',\n",
        "    'openvpn':'vpn',\n",
        "    'httpclient':'http client',\n",
        "    'arraylist':'array list',\n",
        "    'jsonobject': 'json',\n",
        "    'artifactid':'xml',\n",
        "    'hwnd':'os'\n",
        "}\n",
        "punctuations_in_embeddings = {',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', \n",
        "                '>', '%', '=', '#', '*', '+', '\\\\', '•', '~', '@', '£', '·', '{', '}', '©', '^', '®', \n",
        "                '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n",
        "                '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '═', \n",
        "                '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', \n",
        "                '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n",
        "                '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔',\n",
        "                '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', \n",
        "                '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\\\', ']', '^', '{', \n",
        "                '|', '}', '~'}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8KAc2JUSrLP"
      },
      "source": [
        "'''Pipeline for text preprocessing. Check coverage computes what %age of tokens in the text data \n",
        "are covered by Embeddings.\n",
        "preprocess_text is used to clean text.\n",
        "'''\n",
        "def improve_text(dframes):\n",
        "    def preprocess_text(s):\n",
        "        s = s.lower()\n",
        "        '''Expanding contractions...'''\n",
        "        for key, value in contractions.items():\n",
        "            s = s.replace(key, f' {value} ')\n",
        "        for key, value in rules.items():\n",
        "            s = s.replace(key, f' {value} ')\n",
        "        '''Fixing punctuations...'''\n",
        "        for punct in punctuations:\n",
        "            if punct in punctuations_in_embeddings:\n",
        "                s = s.replace(punct, f' {punct} ')\n",
        "            else:\n",
        "                s = s.replace(punct, ' ')\n",
        "        '''Replacing few tokens with its similar word/group of words...'''\n",
        "        for key, value in token_replace.items():\n",
        "            s = s.replace(key, value)\n",
        "        '''Removing HTML tags'''\n",
        "        s = re.sub('<.*?>', ' ', s)\n",
        "        s = re.sub('\\s+', ' ', s)\n",
        "        return s\n",
        "    print(\"Applying Preprocessing.....\")\n",
        "    for each_df in dframes:\n",
        "        each_df['clean_headline'] = each_df['headline'].apply(preprocess_text)\n",
        "        each_df['clean_desc'] = each_df['short_description'].apply(preprocess_text)\n",
        "    print(\"Preprocessing Finished\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "16hUsrAkTBbY",
        "outputId": "f714e772-eb24-43ba-a962-feaba40522e6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "improve_text([df])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Applying Preprocessing.....\n",
            "Preprocessing Finished\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bUCer3t2vo2Z",
        "outputId": "af0cfbd7-2a8c-47e6-d54a-87f5d6336e0c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "!pip install fuzzywuzzy"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting fuzzywuzzy\n",
            "  Downloading https://files.pythonhosted.org/packages/43/ff/74f23998ad2f93b945c0309f825be92e04e0348e062026998b5eefef4c33/fuzzywuzzy-0.18.0-py2.py3-none-any.whl\n",
            "Installing collected packages: fuzzywuzzy\n",
            "Successfully installed fuzzywuzzy-0.18.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dql9l18msjre"
      },
      "source": [
        "def fetch_words(s, use_stopwords = False, ignore_stopwords = False):\n",
        "    if ignore_stopwords == True:\n",
        "        return {word for word in s.split() if word not in stop_words}\n",
        "    elif use_stopwords:\n",
        "        return {word for word in s.split() if word in stop_words}\n",
        "    else:\n",
        "        return set(s.split())\n",
        "def text_features(row, use_stopwords, is_token, return_max):\n",
        "    t1, t2 = None, None\n",
        "    if use_stopwords == True:\n",
        "        t1 = fetch_words(row[0], use_stopwords = True)\n",
        "        t2 = fetch_words(row[1], use_stopwords = True)\n",
        "    elif is_token == True:\n",
        "        t1 = fetch_words(row[0], ignore_stopwords = True)\n",
        "        t2 = fetch_words(row[1], ignore_stopwords = True)\n",
        "    else:\n",
        "        t1 = fetch_words(row[0])\n",
        "        t2 = fetch_words(row[1])\n",
        "    if return_max:\n",
        "        try:\n",
        "            ans = len(t1.intersection(t2))/max(len(t1), len(t2))\n",
        "        except:\n",
        "            return 0\n",
        "    else:\n",
        "        try:\n",
        "            ans = len(t1.intersection(t2))/min(len(t1), len(t2))\n",
        "        except:\n",
        "            return 0\n",
        "    return ans\n",
        "\n",
        "def get_ratio(row, is_token = False):\n",
        "    if is_token == True:\n",
        "        t1 = len(set([word for word in row[0].split() if word not in stop_words]))\n",
        "        t2 = len(set([word for word in row[1].split() if word not in stop_words]))\n",
        "    t1 = len(set([word for word in row[0].split()]))\n",
        "    t2 = len(set([word for word in row[1].split()]))\n",
        "    try:\n",
        "        return t1/t2\n",
        "    except:\n",
        "        return 0\n",
        "def get_fuzzy_ratio(row, ratio, partial_ratio, token_sort_ratio, token_set_ratio):\n",
        "    if ratio:\n",
        "        return fuzz.ratio(row[0], row[1])/100\n",
        "    elif partial_ratio:\n",
        "        return fuzz.partial_ratio(row[0], row[1])/100\n",
        "    elif token_sort_ratio:\n",
        "        return fuzz.token_sort_ratio(row[0], row[1])/100\n",
        "    else:\n",
        "        return fuzz.token_set_ratio(row[0], row[1])/100"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xSuUzZJhw-TU",
        "outputId": "9b7d60a3-9d53-4484-8393-5c87ec80c82c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2009, 22)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y_1D7nd7wHLi"
      },
      "source": [
        "# temp_df = df.copy()\n",
        "#df = df.sample(frac = 0.01)\n",
        "df = temp_df.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7myygzPbtKnw"
      },
      "source": [
        "feature_engg_cols = ['cwc_min_headline_desc',\n",
        "    'cwc_max_headline_desc', 'csc_min_headline_desc',\n",
        "    'csc_max_headline_desc', 'ctc_min_headline_desc',\n",
        "    'ctc_max_headline_desc', \n",
        "    'word_headline_desc_ratio', 'token_headline_desc_ratio',\n",
        "    'ratio_headline_desc', 'partial_ratio_headline_desc',\n",
        "    'token_sort_ratio_headline_desc',\n",
        "    'token_set_ratio_headline_desc']\n",
        "\n",
        "df['cwc_min_headline_desc'] = df[['clean_headline', 'clean_desc']].apply(func = text_features, \n",
        "                                                    axis = 1, args = (False, False, False))\n",
        "df['cwc_max_headline_desc'] = df[['clean_headline', 'clean_desc']].apply(func = text_features, \n",
        "                                                    axis = 1, args = (False, False, True))\n",
        "df['csc_min_headline_desc'] = df[['clean_headline', 'clean_desc']].apply(func = text_features, \n",
        "                                                    axis = 1, args = (True, False, False))\n",
        "df['csc_max_headline_desc'] = df[['clean_headline', 'clean_desc']].apply(func = text_features, \n",
        "                                                    axis = 1, args = (True, False, True))\n",
        "df['ctc_min_headline_desc'] = df[['clean_headline', 'clean_desc']].apply(func = text_features, \n",
        "                                                    axis = 1, args = (False, True, False))\n",
        "df['ctc_max_headline_desc'] = df[['clean_headline', 'clean_desc']].apply(func = text_features, \n",
        "                                                    axis = 1, args = (False, True, True))\n",
        "\n",
        "df['word_headline_desc_ratio'] = df[['clean_headline', 'clean_desc']].apply(\n",
        "        func = get_ratio, axis = 1, args = (False, ))\n",
        "df['token_headline_desc_ratio'] = df[['clean_headline', 'clean_desc']].apply(\n",
        "        func = get_ratio, axis = 1, args = (True, ))\n",
        "df['ratio_headline_desc'] = \\\n",
        "df[['clean_headline', 'clean_desc']].apply(func = get_fuzzy_ratio, \n",
        "                                                    axis = 1, args = (True, False, False, False))\n",
        "df['partial_ratio_headline_desc'] = \\\n",
        "df[['clean_headline', 'clean_desc']].apply(func = get_fuzzy_ratio, \n",
        "                                                    axis = 1, args = (False, True, False, False))\n",
        "df['token_sort_ratio_headline_desc'] = \\\n",
        "df[['clean_headline', 'clean_desc']].apply(func = get_fuzzy_ratio, \n",
        "                                                    axis = 1, args = (False, False, True, False))\n",
        "df['token_set_ratio_headline_desc'] = \\\n",
        "df[['clean_headline', 'clean_desc']].apply(func = get_fuzzy_ratio, \n",
        "                                                    axis = 1, args = (False, False, False, True))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3qjuL9yY0WFa"
      },
      "source": [
        "# df.to_csv('./drive/My Drive/edustack/data1.csv', index = False)\n",
        "fe = df[feature_engg_cols].values.astype(\"float32\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0wj7GYniTlQd",
        "outputId": "0c99d4b5-d2a5-4a81-bb85-ea29a61458da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "df['combine_text'] = df.clean_headline + ' ' + df.clean_desc \n",
        "all_texts = df['combine_text'].tolist() \n",
        "all_texts = [text.split() for text in all_texts]\n",
        "model = Word2Vec(size = 300, min_count = 1, window = 5)\n",
        "model.build_vocab(all_texts)\n",
        "model.intersect_word2vec_format('./drive/My Drive/google_qa/fasttext-wiki-news-subwords-300.gz', lockf = 1)\n",
        "model.train(all_texts, total_examples = model.corpus_count, epochs = 5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:254: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(26761715, 34919320)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "elOCHDtHsLGd"
      },
      "source": [
        "def category_cleaner(x):\n",
        "    if x == 'THE WORLDPOST':\n",
        "        return 'WORLDPOST'\n",
        "    elif x == 'PARENTING':\n",
        "        return 'PARENTS'\n",
        "    elif x == 'ARTS' or x == 'CULTURE & ARTS':\n",
        "        return 'ARTS & CULTURE'\n",
        "    elif x == 'STYLE':\n",
        "        return 'STYLE & BEAUTY'\n",
        "    elif x == 'COLLEGE':\n",
        "        return 'EDUCATION'\n",
        "    elif x == 'TASTE':\n",
        "        return 'FOOD & DRINK'\n",
        "    else:\n",
        "        return x\n",
        "    \n",
        "df['category'] = df.category.apply(category_cleaner)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zgrMxcyPbTmT"
      },
      "source": [
        "counter = Counter()\n",
        "for text in all_texts:\n",
        "  counter.update(text)\n",
        "vocab = {}\n",
        "vocab['token2id'] = {key:id+1 for id, (key, _) in enumerate(counter.items())}\n",
        "vocab['id2token'] = {value:key for key, value in vocab['token2id'].items()}\n",
        "vocab['word_freq'] = dict(counter)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-8QhTgN0ULqF"
      },
      "source": [
        "def build_embedding_matrix(vocab, texts, embedd_size, model):\n",
        "  n = len(vocab['token2id'])+1\n",
        "  embedding_matrix = np.zeros((n, embedd_size))\n",
        "  for text in texts:\n",
        "    for key in text:\n",
        "      word = key\n",
        "      try:\n",
        "        embedding_matrix[vocab['token2id'][word]] = model.wv[word]\n",
        "        continue\n",
        "      except:\n",
        "        pass\n",
        "  return embedding_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fQbBF6ibZoYL"
      },
      "source": [
        "def get_token_ids(texts, max_length):\n",
        "  tokens = []\n",
        "  for text in texts:\n",
        "    tmp_tokens = []\n",
        "    if len(text.split()) > max_length:\n",
        "      for each in (text.split()[:(max_length//2)] + text.split()[-(max_length//2):]):\n",
        "        tmp_tokens.append(vocab['token2id'].get(each, 0))\n",
        "      tokens.append(tmp_tokens)\n",
        "    else:\n",
        "      for each in (text.split()[:max_length]):\n",
        "        tmp_tokens.append(vocab['token2id'].get(each, 0))\n",
        "      tokens.append(tmp_tokens)\n",
        "  return tf.keras.preprocessing.sequence.pad_sequences(tokens, padding=\"post\", maxlen=max_length)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mZWXVpYHdBnA"
      },
      "source": [
        "embedding_matrix = build_embedding_matrix(vocab, all_texts, 300, model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SihRSfQ6vGuB"
      },
      "source": [
        "#df['clean_combined'] = df['clean_headline'] + ' ' + df['clean_desc']\n",
        "df['clean_headline'] = df.clean_headline.apply(lambda x: x.strip())\n",
        "df['clean_desc'] = df.clean_desc.apply(lambda x: x.strip())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rXqqNIrHZ06d"
      },
      "source": [
        "MAX_LENGTH = 60\n",
        "data = {}\n",
        "data['headline'] = get_token_ids(df['clean_headline'], MAX_LENGTH)\n",
        "data['desc'] = get_token_ids(df['clean_desc'], MAX_LENGTH)\n",
        "targets = df.category.astype('category').cat.codes.values\n",
        "targets = tf.keras.utils.to_categorical(targets, num_classes = 34)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zL5qqu9iFq4B"
      },
      "source": [
        "class Accuracy(tf.keras.metrics.Metric):\n",
        "    def __init__(self, name = 'accuracy', batch_size = 32, **kwrgs):\n",
        "        super(Accuracy, self).__init__(name = name, **kwrgs)\n",
        "        self.correct_predictions = self.add_weight(name = 'one',initializer = 'zeros')\n",
        "        self.total_predictions = self.add_weight(name = 'two', initializer = 'zeros')\n",
        "        self.batch_size = batch_size\n",
        "    def update_state(self, y_true, y_pred, sample_weight = None):\n",
        "        predictions = tf.argmax(y_pred, axis = 1)\n",
        "        true_predictions = tf.argmax(y_true, axis = 1)\n",
        "        correct_values = (tf.cast(true_predictions, \"int32\") == tf.cast(predictions, \"int32\"))\n",
        "        correct_values = tf.cast(correct_values, \"float32\")\n",
        "        self.correct_predictions.assign_add(tf.reduce_sum(correct_values))\n",
        "    def result(self):\n",
        "        self.total_predictions.assign_add(tf.constant(self.batch_size, dtype = \"float32\"))\n",
        "        return self.correct_predictions/self.total_predictions\n",
        "    def reset_state(self):\n",
        "        self.correct_predictions.assign(0)\n",
        "        self.total_predictions.assign(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hxxLJ1LMHEwz"
      },
      "source": [
        "def get_generator(X, batch_size = 32, training = True):\n",
        "    Y = X[1]\n",
        "    N = X[0][0].shape[0]\n",
        "    indexes = np.arange(N)\n",
        "    if training == True:\n",
        "        indexes = np.random.permutation(indexes)\n",
        "    def generator():\n",
        "        for i in indexes:\n",
        "            yield {\"input_1\": X[0][0][i], \"input_2\":X[0][1][i], \"input_3\":X[0][2][i]}, Y[i]\n",
        "    return tf.data.Dataset.from_generator(generator, \n",
        "                    output_types = ({\"input_1\": tf.int32, \"input_2\":tf.int32, \"input_3\":tf.float32}, tf.float32)).repeat().batch(batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ww6RMg0EcwAA"
      },
      "source": [
        "def c_lstm(max_length):\n",
        "  i1 = tf.keras.layers.Input(shape = (max_length), name = 'input_1')\n",
        "  i2 = tf.keras.layers.Input(shape = (max_length), name = 'input_2')\n",
        "  fe_ = tf.keras.layers.Input(shape = (fe.shape[1]), name = 'input_3')\n",
        "  e1 = tf.keras.layers.Embedding(input_dim = embedding_matrix.shape[0], output_dim = embedding_matrix.shape[1], weights = [embedding_matrix], trainable = False)(i1)\n",
        "  e2 = tf.keras.layers.Embedding(input_dim = embedding_matrix.shape[0], output_dim = embedding_matrix.shape[1], weights = [embedding_matrix], trainable = False)(i2)\n",
        "  \n",
        "  \n",
        "  lstm_head = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=True))(e1)\n",
        "  lstm_head_2 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=False))(tf.keras.layers.SpatialDropout1D(rate = 0.2)(lstm_head))\n",
        "  \n",
        "  lstm_desc = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=True))(e2)\n",
        "  lstm_desc_2 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=False))(tf.keras.layers.SpatialDropout1D(rate = 0.2)(lstm_desc))\n",
        "  \n",
        "\n",
        "\n",
        "#   avg_head = tf.keras.layers.GlobalAveragePooling1D()(lstm_head)\n",
        "#   avg_desc = tf.keras.layers.GlobalAveragePooling1D()(lstm_desc)\n",
        "  \n",
        "  #concat = tf.keras.layers.Concatenate()([lstm_head, lstm_desc_2)\n",
        "  \n",
        "  #drop = tf.keras.layers.Dropout(rate = 0.2)(concat)\n",
        "  \n",
        "  d1 = tf.keras.layers.Dense(512, activation = 'relu')(tf.keras.layers.Dropout(rate = 0.2)(lstm_head_2))\n",
        "  d2 = tf.keras.layers.Dense(512, activation = 'relu')(tf.keras.layers.Dropout(rate = 0.2)(lstm_desc_2))\n",
        "  concat = tf.keras.layers.Concatenate()([d1, d2, fe_])\n",
        "  out = tf.keras.layers.Dense(34, activation = 'softmax')(concat)\n",
        "\n",
        "  model = tf.keras.Model(inputs = [i1, i2, fe_], outputs = [out])\n",
        "\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bLKFaq_afoVc",
        "outputId": "4b5aaeca-86f7-40bc-e426-12b51d9ef820",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "kf = StratifiedKFold(n_splits = 5).split(X = df.link, y = df.category)\n",
        "lr_sched = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-3)#*(0.7 ** ((epoch%4))))\n",
        "for fold, (train_idx, valid_idx) in enumerate(kf):\n",
        "    if fold != 1:\n",
        "        continue\n",
        "    tf.keras.backend.clear_session()\n",
        "    BATCH_SIZE = 128\n",
        "    train_inputs = ((data['headline'][train_idx],data['desc'][train_idx], fe[train_idx]), targets[train_idx]) \n",
        "    valid_inputs = ((data['headline'][valid_idx],data['desc'][valid_idx], fe[valid_idx]), targets[valid_idx])\n",
        "    train_dataset = get_generator(train_inputs, BATCH_SIZE)\n",
        "    valid_dataset = get_generator(valid_inputs, BATCH_SIZE, False)\n",
        "    with tpu_strategy.scope():\n",
        "        model = c_lstm(MAX_LENGTH)\n",
        "        model.compile(tf.keras.optimizers.Adam(), \n",
        "                    loss = tf.keras.losses.CategoricalCrossentropy(),\n",
        "                    metrics = ['accuracy'])\n",
        "                    #metrics = [Accuracy(batch_size = BATCH_SIZE)])\n",
        "    # model.fit(train_dataset, steps_per_epoch = train_idx.shape[0]//BATCH_SIZE, validation_data = valid_dataset,\n",
        "    #           validation_steps = valid_idx.shape[0]//BATCH_SIZE, epochs = 10)\n",
        "    model.fit(x = train_inputs[0], y = train_inputs[1], validation_data = valid_inputs, batch_size = BATCH_SIZE, epochs = 5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "   2/1256 [..............................] - ETA: 53:14 - loss: 3.4760 - accuracy: 0.0938  WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0029s vs `on_train_batch_end` time: 0.0515s). Check your callbacks.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0029s vs `on_train_batch_end` time: 0.0515s). Check your callbacks.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1256/1256 [==============================] - ETA: 0s - loss: 1.5124 - accuracy: 0.5786WARNING:tensorflow:Callbacks method `on_test_batch_end` is slow compared to the batch time (batch time: 0.0018s vs `on_test_batch_end` time: 0.0231s). Check your callbacks.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Callbacks method `on_test_batch_end` is slow compared to the batch time (batch time: 0.0018s vs `on_test_batch_end` time: 0.0231s). Check your callbacks.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1256/1256 [==============================] - 88s 70ms/step - loss: 1.5124 - accuracy: 0.5786 - val_loss: 1.3514 - val_accuracy: 0.6102\n",
            "Epoch 2/5\n",
            "1256/1256 [==============================] - 76s 60ms/step - loss: 1.1874 - accuracy: 0.6561 - val_loss: 1.2193 - val_accuracy: 0.6500\n",
            "Epoch 3/5\n",
            "1256/1256 [==============================] - 76s 60ms/step - loss: 1.0344 - accuracy: 0.6940 - val_loss: 1.1915 - val_accuracy: 0.6587\n",
            "Epoch 4/5\n",
            "1256/1256 [==============================] - 75s 60ms/step - loss: 0.9010 - accuracy: 0.7260 - val_loss: 1.2004 - val_accuracy: 0.6535\n",
            "Epoch 5/5\n",
            "1256/1256 [==============================] - 76s 60ms/step - loss: 0.7694 - accuracy: 0.7606 - val_loss: 1.2388 - val_accuracy: 0.6464\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YhFvS4Kbc3IO"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}