NER_CHAR_CRF.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndata = pd.read_csv(\"../input/ner-dataset/ner_datasetreference.csv\", encoding=\"latin1\")","metadata":{"_uuid":"8260b78cf150c8da6e4d8adb8c421a00012ffde2","scrolled":true,"execution":{"iopub.status.busy":"2021-11-02T14:56:10.864025Z","iopub.execute_input":"2021-11-02T14:56:10.864321Z","iopub.status.idle":"2021-11-02T14:56:11.624826Z","shell.execute_reply.started":"2021-11-02T14:56:10.864267Z","shell.execute_reply":"2021-11-02T14:56:11.624086Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"data = data.fillna(method=\"ffill\")","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:13.592249Z","iopub.execute_input":"2021-11-02T14:56:13.592557Z","iopub.status.idle":"2021-11-02T14:56:14.037634Z","shell.execute_reply.started":"2021-11-02T14:56:13.592494Z","shell.execute_reply":"2021-11-02T14:56:14.036924Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:16.112505Z","iopub.execute_input":"2021-11-02T14:56:16.112807Z","iopub.status.idle":"2021-11-02T14:56:16.154770Z","shell.execute_reply.started":"2021-11-02T14:56:16.112754Z","shell.execute_reply":"2021-11-02T14:56:16.154000Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"words = list(set(data[\"Word\"].values))\nn_words = len(words); n_words","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:22.653106Z","iopub.execute_input":"2021-11-02T14:56:22.653394Z","iopub.status.idle":"2021-11-02T14:56:22.731028Z","shell.execute_reply.started":"2021-11-02T14:56:22.653335Z","shell.execute_reply":"2021-11-02T14:56:22.730268Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"tags = list(set(data[\"Tag\"].values))\nn_tags = len(tags); n_tags","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:26.224596Z","iopub.execute_input":"2021-11-02T14:56:26.224885Z","iopub.status.idle":"2021-11-02T14:56:26.262350Z","shell.execute_reply.started":"2021-11-02T14:56:26.224830Z","shell.execute_reply":"2021-11-02T14:56:26.261625Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"class get_sentences(object):\n    \n    def __init__(self, data):\n        self.n_sent = 1\n        self.data = data\n        self.empty = False\n        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s[\"Word\"].values.tolist(),\n                                                           s[\"POS\"].values.tolist(),\n                                                           s[\"Tag\"].values.tolist())]\n        self.grouped = self.data.groupby(\"Sentence #\").apply(agg_func)\n        self.sentences = [s for s in self.grouped]\n    \n    def get_next(self):\n        try:\n            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n            self.n_sent += 1\n            return s\n        except:\n            return None","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:28.401190Z","iopub.execute_input":"2021-11-02T14:56:28.401462Z","iopub.status.idle":"2021-11-02T14:56:28.408130Z","shell.execute_reply.started":"2021-11-02T14:56:28.401409Z","shell.execute_reply":"2021-11-02T14:56:28.407047Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"getter = get_sentences(data)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:32.084939Z","iopub.execute_input":"2021-11-02T14:56:32.085212Z","iopub.status.idle":"2021-11-02T14:56:38.195952Z","shell.execute_reply.started":"2021-11-02T14:56:32.085160Z","shell.execute_reply":"2021-11-02T14:56:38.195073Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"sent = getter.get_next()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:38.197173Z","iopub.execute_input":"2021-11-02T14:56:38.197450Z","iopub.status.idle":"2021-11-02T14:56:38.208537Z","shell.execute_reply.started":"2021-11-02T14:56:38.197403Z","shell.execute_reply":"2021-11-02T14:56:38.207593Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"print(sent)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:38.209657Z","iopub.execute_input":"2021-11-02T14:56:38.210072Z","iopub.status.idle":"2021-11-02T14:56:38.222241Z","shell.execute_reply.started":"2021-11-02T14:56:38.210025Z","shell.execute_reply":"2021-11-02T14:56:38.221259Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"sentences = getter.sentences","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:41.764256Z","iopub.execute_input":"2021-11-02T14:56:41.764563Z","iopub.status.idle":"2021-11-02T14:56:41.769075Z","shell.execute_reply.started":"2021-11-02T14:56:41.764502Z","shell.execute_reply":"2021-11-02T14:56:41.768343Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"max_len = 75\nmax_len_char = 10\nword2idx = {w: i + 2 for i, w in enumerate(words)}\nword2idx[\"UNK\"] = 1\nword2idx[\"PAD\"] = 0\nidx2word = {i: w for w, i in word2idx.items()}\ntag2idx = {t: i + 1 for i, t in enumerate(tags)}\ntag2idx[\"PAD\"] = 0\nidx2tag = {i: w for w, i in tag2idx.items()}","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:43.715500Z","iopub.execute_input":"2021-11-02T14:56:43.715779Z","iopub.status.idle":"2021-11-02T14:56:43.737349Z","shell.execute_reply.started":"2021-11-02T14:56:43.715726Z","shell.execute_reply":"2021-11-02T14:56:43.736704Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"print(word2idx[\"Obama\"])\nprint(tag2idx[\"B-geo\"])","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:48.249603Z","iopub.execute_input":"2021-11-02T14:56:48.249913Z","iopub.status.idle":"2021-11-02T14:56:48.256614Z","shell.execute_reply.started":"2021-11-02T14:56:48.249857Z","shell.execute_reply":"2021-11-02T14:56:48.255591Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\nX_word = [[word2idx[w[0]] for w in s] for s in sentences]\nX_word = pad_sequences(maxlen=max_len, sequences=X_word, value=word2idx[\"PAD\"], padding='post', truncating='post')\nchars = set([w_i for w in words for w_i in w])\nn_chars = len(chars)\nprint(n_chars)\nchar2idx = {c: i + 2 for i, c in enumerate(chars)}\nchar2idx[\"UNK\"] = 1\nchar2idx[\"PAD\"] = 0\nX_char = []\nfor sentence in sentences:\n    sent_seq = []\n    for i in range(max_len):\n        word_seq = []\n        for j in range(max_len_char):\n            try:\n                if(sentence[i][0][j] in char2idx):\n                    word_seq.append(char2idx.get(sentence[i][0][j]))\n                else:\n                    word_seq.append(char2idx.get(\"UNK\"))\n            except:\n                word_seq.append(char2idx.get(\"PAD\"))\n        sent_seq.append(word_seq)\n    X_char.append(np.array(sent_seq))\ny = [[tag2idx[w[2]] for w in s] for s in sentences]\ny = pad_sequences(maxlen=max_len, sequences=y, value=tag2idx[\"PAD\"], padding='post', truncating='post')\nfrom keras.utils import to_categorical\ny = [to_categorical(i, num_classes=n_tags + 1) for i in y]\n\nfrom sklearn.model_selection import train_test_split\nX_word_tr, X_word_te, y_tr, y_te = train_test_split(X_word, y, test_size=0.1, random_state=2018)\nX_char_tr, X_char_te, _, _ = train_test_split(X_char, y, test_size=0.1, random_state=2018)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:51.233152Z","iopub.execute_input":"2021-11-02T14:56:51.233428Z","iopub.status.idle":"2021-11-02T14:57:28.218135Z","shell.execute_reply.started":"2021-11-02T14:56:51.233380Z","shell.execute_reply":"2021-11-02T14:57:28.217111Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"X_word_tr.shape, X_word_te.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:57:28.224390Z","iopub.execute_input":"2021-11-02T14:57:28.224994Z","iopub.status.idle":"2021-11-02T14:57:28.232215Z","shell.execute_reply.started":"2021-11-02T14:57:28.224940Z","shell.execute_reply":"2021-11-02T14:57:28.231376Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"len(X_char_tr), len(X_char_te[7][1])","metadata":{"execution":{"iopub.status.busy":"2021-10-30T10:54:17.677448Z","iopub.execute_input":"2021-10-30T10:54:17.677732Z","iopub.status.idle":"2021-10-30T10:54:17.685831Z","shell.execute_reply.started":"2021-10-30T10:54:17.677686Z","shell.execute_reply":"2021-10-30T10:54:17.685124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_word_tr[1]","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:57:28.234141Z","iopub.execute_input":"2021-11-02T14:57:28.235484Z","iopub.status.idle":"2021-11-02T14:57:28.245462Z","shell.execute_reply.started":"2021-11-02T14:57:28.235392Z","shell.execute_reply":"2021-11-02T14:57:28.244649Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"X_char_tr[1]","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:57:28.246717Z","iopub.execute_input":"2021-11-02T14:57:28.247148Z","iopub.status.idle":"2021-11-02T14:57:28.264897Z","shell.execute_reply.started":"2021-11-02T14:57:28.247105Z","shell.execute_reply":"2021-11-02T14:57:28.264235Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"print(n_words,n_chars,n_tags)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:58:03.602932Z","iopub.execute_input":"2021-11-02T14:58:03.603233Z","iopub.status.idle":"2021-11-02T14:58:03.607820Z","shell.execute_reply.started":"2021-11-02T14:58:03.603178Z","shell.execute_reply":"2021-11-02T14:58:03.606667Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"X_word_tr.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:58:04.793070Z","iopub.execute_input":"2021-11-02T14:58:04.793418Z","iopub.status.idle":"2021-11-02T14:58:04.803148Z","shell.execute_reply.started":"2021-11-02T14:58:04.793368Z","shell.execute_reply":"2021-11-02T14:58:04.802393Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"np.array(X_char_tr).reshape((len(X_char_tr), max_len, max_len_char)).shape","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:58:06.809321Z","iopub.execute_input":"2021-11-02T14:58:06.809618Z","iopub.status.idle":"2021-11-02T14:58:07.036355Z","shell.execute_reply.started":"2021-11-02T14:58:06.809562Z","shell.execute_reply":"2021-11-02T14:58:07.035620Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://www.github.com/keras-team/keras-contrib.git\nfrom keras_contrib.layers import CRF","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:58:21.034635Z","iopub.execute_input":"2021-11-02T14:58:21.034937Z","iopub.status.idle":"2021-11-02T14:58:30.964871Z","shell.execute_reply.started":"2021-11-02T14:58:21.034879Z","shell.execute_reply":"2021-11-02T14:58:30.964102Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"from keras.models import Model, Input\nfrom keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Conv1D\nfrom keras.layers import Bidirectional, concatenate, SpatialDropout1D, GlobalMaxPooling1D\n# input and embedding for words\nword_in = Input(shape=(max_len,),name='word_input_')\nemb_word = Embedding(input_dim=n_words + 2, output_dim=20,\n                     input_length=max_len, mask_zero=True)(word_in)\n\n# input and embeddings for characters\nchar_in = Input(shape=(max_len, max_len_char,),name='char_input')\nemb_char = TimeDistributed(Embedding(input_dim=n_chars + 2, output_dim=10,\n                           input_length=max_len_char, mask_zero=True))(char_in)\n# character LSTM to get word encodings by characters\nchar_enc = TimeDistributed(Bidirectional(LSTM(units=20, return_sequences=False,\n                                recurrent_dropout=0.5)))(emb_char)\n\n# main LSTM\nx = concatenate([emb_word, char_enc])\nx = SpatialDropout1D(0.3)(x)\nlstm = LSTM(units=50, return_sequences=True,\n                               recurrent_dropout=0.6)(x)\nout = TimeDistributed(Dense(50, activation=\"relu\"))(lstm)\n\ncrf = CRF(n_tags + 1)  # CRF layer\nout = crf(out)\n\nmodel = Model([word_in, char_in], out)\nmodel.compile(optimizer=\"adam\", loss=crf.loss_function, metrics=[crf.accuracy])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T15:28:45.088610Z","iopub.execute_input":"2021-11-02T15:28:45.088895Z","iopub.status.idle":"2021-11-02T15:28:47.022638Z","shell.execute_reply.started":"2021-11-02T15:28:45.088846Z","shell.execute_reply":"2021-11-02T15:28:47.021972Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"print(model.layers[9].weights)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:58:44.764225Z","iopub.execute_input":"2021-11-02T14:58:44.764540Z","iopub.status.idle":"2021-11-02T14:58:44.770501Z","shell.execute_reply.started":"2021-11-02T14:58:44.764456Z","shell.execute_reply":"2021-11-02T14:58:44.769463Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"np.array(y_tr).shape","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:58:48.826909Z","iopub.execute_input":"2021-11-02T14:58:48.827247Z","iopub.status.idle":"2021-11-02T14:58:49.067536Z","shell.execute_reply.started":"2021-11-02T14:58:48.827200Z","shell.execute_reply":"2021-11-02T14:58:49.066620Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"history = model.fit([X_word_tr,\n                     np.array(X_char_tr).reshape((len(X_char_tr), max_len, max_len_char))],\n                    np.array(y_tr),\n                    batch_size=32, epochs=10, validation_split=0.1, verbose=1)\nhist = pd.DataFrame(history.history)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T15:28:53.627590Z","iopub.execute_input":"2021-11-02T15:28:53.627872Z","iopub.status.idle":"2021-11-02T16:34:36.810689Z","shell.execute_reply.started":"2021-11-02T15:28:53.627819Z","shell.execute_reply":"2021-11-02T16:34:36.810000Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"code","source":"hist.columns","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:34:59.561973Z","iopub.execute_input":"2021-11-02T16:34:59.562436Z","iopub.status.idle":"2021-11-02T16:34:59.568966Z","shell.execute_reply.started":"2021-11-02T16:34:59.562323Z","shell.execute_reply":"2021-11-02T16:34:59.568272Z"},"trusted":true},"execution_count":37,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.style.use(\"ggplot\")\nplt.figure(figsize=(12, 12))\nplt.plot(hist[\"crf_viterbi_accuracy\"])\nplt.plot(hist[\"val_crf_viterbi_accuracy\"])\nplt.legend(['crf_viterbi_accuracy','val_crf_viterbi_accuracy'])\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:45:00.616314Z","iopub.execute_input":"2021-11-02T16:45:00.616611Z","iopub.status.idle":"2021-11-02T16:45:00.964916Z","shell.execute_reply.started":"2021-11-02T16:45:00.616558Z","shell.execute_reply":"2021-11-02T16:45:00.964125Z"},"trusted":true},"execution_count":47,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict([X_word_te,\n                        np.array(X_char_te).reshape((len(X_char_te),\n                                                     max_len, max_len_char))])","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:34:59.897839Z","iopub.execute_input":"2021-11-02T16:34:59.898333Z","iopub.status.idle":"2021-11-02T16:35:17.522351Z","shell.execute_reply.started":"2021-11-02T16:34:59.898285Z","shell.execute_reply":"2021-11-02T16:35:17.521599Z"},"trusted":true},"execution_count":39,"outputs":[]},{"cell_type":"code","source":"def label(pred):\n    out = []\n    for pred_i in pred:\n        out_i = []\n        for p in pred_i:\n            p_i = np.argmax(p)\n            out_i.append(idx2tag[p_i].replace(\"PAD\", \"O\"))\n        out.append(out_i)\n    return out\n    \npred_labels = label(y_pred)\ntest_labels = label(y_te)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:35:17.525845Z","iopub.execute_input":"2021-11-02T16:35:17.526068Z","iopub.status.idle":"2021-11-02T16:35:19.773592Z","shell.execute_reply.started":"2021-11-02T16:35:17.526023Z","shell.execute_reply":"2021-11-02T16:35:19.772842Z"},"trusted":true},"execution_count":40,"outputs":[]},{"cell_type":"code","source":"hist = pd.DataFrame(history.history)\n\nplt.style.use(\"ggplot\")\nplt.figure(figsize=(12,12))\nplt.plot(hist[\"loss\"])\nplt.plot(hist[\"val_loss\"])\nplt.legend(['loss','val_loss'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:37:35.603336Z","iopub.execute_input":"2021-11-02T16:37:35.603651Z","iopub.status.idle":"2021-11-02T16:37:35.945195Z","shell.execute_reply.started":"2021-11-02T16:37:35.603595Z","shell.execute_reply":"2021-11-02T16:37:35.944510Z"},"trusted":true},"execution_count":45,"outputs":[]},{"cell_type":"code","source":"import sklearn\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.preprocessing import LabelBinarizer\nfrom itertools import chain\n\ndef ner_report(y_true, y_pred):\n \n    lb = LabelBinarizer()\n    y_true_combined = lb.fit_transform(list(chain.from_iterable(y_true)))\n    y_pred_combined = lb.transform(list(chain.from_iterable(y_pred)))\n    tagset = list(sorted(set(lb.classes_)))\n    tagset = tagset[:-2]\n    print(tagset)\n    class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)}\n    \n    return classification_report(\n        y_true_combined,\n        y_pred_combined,\n        labels = [class_indices[cls] for cls in tagset],\n        target_names = tagset,\n        digits=4\n    )\n\nprint(ner_report(test_labels, pred_labels))","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:37:42.926459Z","iopub.execute_input":"2021-11-02T16:37:42.926789Z","iopub.status.idle":"2021-11-02T16:37:44.518928Z","shell.execute_reply.started":"2021-11-02T16:37:42.926737Z","shell.execute_reply":"2021-11-02T16:37:44.518181Z"},"trusted":true},"execution_count":46,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}