classification_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mô hình nhận diện tin tức giả"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, csv, sys\n",
    "import unicodedata as ud\n",
    "\n",
    "from text_utils import remove_punc, token_sylabling, is_word\n",
    "\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Update 1: pandas và sklearn có các phương thức hỗ trợ việc load dữ liệu, biểu diễn kết quả"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import model_selection\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Một số thư viện hỗ trợ đánh giá mô hình và lưu trữ mô hình"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GridSearchCV để kiếm thông số tốt nhất cho model, Pipeline để xây dựng mô hình chung"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Một số đường dẫn lưu và load file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_path = './Models'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vẽ Confusion Matrix theo dữ liệu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Biểu diễn Bag-of-word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "preproc_path = './PreprocessingData'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chuẩn bị dữ liệu: Loại bỏ dấu câu, số, ngày tháng, url... -> Chuyển thành chữ viết thường"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_preparing(text):\n",
    "    text_list = text.lower().split()\n",
    "    new_text = ''\n",
    "    for word in text_list:\n",
    "        if(is_word(word)):\n",
    "            new_text = ' '.join([new_text, word])\n",
    "    return new_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1. Data 1:  file ```preproc_vn_news_226_tlfr.csv``` - text & label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1_path = 'preproc_vn_news_226_tlfr.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data từ file ```preproc_vn_news_226_tlfr.csv``` là các file đã được tiền xử lý, các record trong file đã được xáo trộn lên theo thứ tự ngẫu nhiên"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_data1 = []\n",
    "\n",
    "data1_train_text = []\n",
    "data1_train_label = []\n",
    "\n",
    "data1_test_text = []\n",
    "data1_test_label = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(preproc_path, data1_path), 'r') as csv_file:\n",
    "    reader = csv.DictReader(csv_file)\n",
    "    news_data1 = list(reader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tách dữ liệu thành 2 phần: ```Train``` - 181 record & ```Test``` - 45 record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 181\n",
    "\n",
    "news_data1_train = news_data1[:train_size]\n",
    "news_data1_test = news_data1[train_size:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chuẩn bị dữ liệu train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for train in news_data1_train:\n",
    "    data1_train_text.append(data_preparing(train['text']))\n",
    "    data1_train_label.append(train['label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chuẩn bị dữ liệu test:\n",
    "```data1_test_label```: chứa các label của dữ liệu test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for test in news_data1_test:\n",
    "    data1_test_text.append(data_preparing(test['text']))\n",
    "    data1_test_label.append(test['label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Biểu diễn Bag-of-Word cho dữ liệu ```train```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(181, 7938)\n"
     ]
    }
   ],
   "source": [
    "data1_tfidf_vectorizer = TfidfVectorizer()\n",
    "X_data1_train_tfidf = data1_tfidf_vectorizer.fit_transform(data1_train_text)\n",
    "print(X_data1_train_tfidf.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2. Data 2:  file ```preproc_vn_news_223_tdlfr.csv``` - text & label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2_path = 'preproc_vn_news_223_tdlfr.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_data2 = pd.read_csv(os.path.join(preproc_path, data2_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_news_data2 = news_data2.drop('label', axis=1)\n",
    "y_news_data2 = news_data2['label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thay đổi giá trị trong Dataframe của Pandas tham khảo tại:\n",
    "1. [Set value for particular cell in pandas DataFrame with iloc](https://stackoverflow.com/questions/31569384/set-value-for-particular-cell-in-pandas-dataframe-with-iloc/31569794)\n",
    "2. [Set value for particular cell in pandas DataFrame using index](https://stackoverflow.com/a/13842286/5144980)\n",
    "3. [pandas.DataFrame.at - Pandas Doc](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.at.html?highlight=#pandas.DataFrame.at)\n",
    "4. [pandas.DataFrame.iat](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.iat.html?highlight=iat#pandas.DataFrame.iat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index in range(list(X_news_data2.shape)[0]):\n",
    "    news = X_news_data2.iat[index, 0]\n",
    "    X_news_data2.iat[index, 0] = data_preparing(news)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chúng ta thêm 1 columns ```text_domain``` sẽ chứa nội dung của 2 cột: ```text``` và ```domain```, khi biểu diễn bag-of-word sẽ biểu diễn các vector cho trường ```domain```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_news_data2['text_domain'] = X_news_data2['text'].map(str) + X_news_data2['domain']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trộn dữ liệu và Split Train & Test bằng ```train_test_train```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_news2_train, X_news2_test, y_news2_train, y_news2_test = train_test_split(X_news_data2, y_news_data2, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing Data2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Xử lý dữ liệu trên dataframe tham khảo tại:\n",
    "1. [How to use sklearn fit_transform with pandas and return dataframe instead of numpy array?](https://stackoverflow.com/a/35724176/5144980)\n",
    "2. [How to append TF-IDF vector into pandas dataframe ? - ResearchGate](https://www.researchgate.net/post/How_to_append_TF-IDF_vector_into_pandas_dataframe)\n",
    "3. [Pandas to Sklearn](https://medium.com/dunder-data/from-pandas-to-scikit-learn-a-new-exciting-workflow-e88e2271ef62)\n",
    "4. [Same TF-IDF Vectorizer for 2 data inputs](https://stackoverflow.com/questions/12850345/how-to-combine-two-data-frames-in-python-pandas)\n",
    "\n",
    "Chọn 1 trong các hướng tiếp cận trên"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Vector hóa__: TFIDF vectorizer dữ liệu ```text``` của training data\n",
    "1. ```data2_tfidf_vectorizer```: vectorize dữ liệu text lẫn domain\n",
    "2. ```data2_domain_tfidf_vectorizer```: vectorize dữ liệu domain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2_tfidf_vectorizer = TfidfVectorizer()\n",
    "data2_domain_tfidf_vectorizer = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dữ liệu train bao gồm ```text, domain```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(178, 8164)\n"
     ]
    }
   ],
   "source": [
    "X_data2_train_tfidf = data2_tfidf_vectorizer.fit_transform(X_news2_train['text_domain'])\n",
    "print(X_data2_train_tfidf.shape)\n",
    "# print(data2_tfidf_vectorizer.vocabulary_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dữ liệu train chỉ gồm ```domain```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(178, 32)\n"
     ]
    }
   ],
   "source": [
    "X_data2_domain_train_tfidf = data2_domain_tfidf_vectorizer.fit_transform(X_news2_train['domain'])\n",
    "print(X_data2_domain_train_tfidf.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Naive Bayes classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1. Mô hình cho dữ liệu từ file ```preproc_vn_news_226_tlfr.csv``` - text & label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.1. Mô hình 1\n",
    "Thực hiện tokenize bằng mô hình của nhóm -> Vectorize bằng TFIDF -> Thực hiện classify bằng Multinomial Naive Bayesian."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tfidfvect', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=Tru...e,\n",
       "        vocabulary=None)), ('nbclf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MultiNB_clf_1 = MultinomialNB().fit(X_data1_train_tfidf, data1_train_label)\n",
    "MultiNB_clf_1 = Pipeline([\n",
    "    ('tfidfvect', TfidfVectorizer()),\n",
    "    ('nbclf', MultinomialNB())\n",
    "])\n",
    "MultiNB_clf_1.fit(data1_train_text, data1_train_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ở đây biểu diễn test theo như mô hình TFIDFVectorize vừa tạo ở trên"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chiều của dữ liệu Test:  (45, 7938)\n"
     ]
    }
   ],
   "source": [
    "X_test = data1_tfidf_vectorizer.transform(data1_test_text)\n",
    "\n",
    "print('Chiều của dữ liệu Test: ',X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1_predict = MultiNB_clf_1.predict(data1_test_text)\n",
    "data1_predict_log_proba = MultiNB_clf_1.predict_log_proba(data1_test_text)\n",
    "data1_predict_proba = MultiNB_clf_1.predict_proba(data1_test_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Đánh giá Mô hình ```MultiNB_clf_1```:\n",
    "\n",
    "1. __Accuracy__ của mô hình"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8888888888888888"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(data1_test_label == data1_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. __Confussion Matrix__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnf_matrix = confusion_matrix(data1_test_label, data1_predict)\n",
    "np.set_printoptions(precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[25  1]\n",
      " [ 4 15]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAEYCAYAAADFzZobAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XmcHFW5//HPdxKWQMKaEMMaBIICQgjrBVlUwgWEC/hTL4gsBmRRcEEQVNQgqFwXFAXlBmSXRS8oiCirEFDWQIBAWCVsCSFhDyRAkuf3xzmDnXFmumfSNd099X3nVa90V1Wferqr5+lzTp2qUkRgZlYmbY0OwMysrznxmVnpOPGZWek48ZlZ6TjxmVnpOPGZWek48WWSBkn6k6TXJP1+McrZT9J19YytUSRtJ+nRZtmepJGSQtLAvoqpVUiaJmmn/Pibks4uYBtnSvp2vcttBLXaOD5JnwGOBj4AvAFMBr4fEbctZrn7A0cB20TE/MUOtMlJCmC9iHii0bF0RdI04JCIuCE/Hwk8BSxR730k6TzguYg4oZ7l9pWOn1Udyjsol/fhepTXbFqqxifpaODnwA+A4cCawK+APetQ/FrAY2VIerVwrao4/mybQES0xAQsD8wBPtXNOkuREuP0PP0cWCov2xF4Dvga8CIwA/hcXnYi8A7wbt7GwcB44KKKskcCAQzMzw8C/kmqdT4F7Fcx/7aK120D3A28lv/fpmLZzcBJwN9zOdcBQ7t4b+3xf70i/r2A3YDHgJeBb1asvyVwO/BqXvd0YMm8bGJ+L2/m9/vfFeUfB7wAXNg+L79mnbyNMfn5qsBsYMca9t35wNfy49Xytr+Qn6+by1WH7V0ILATm5hi/XrEPDgSeydv/Vo37f5H9kudF3v6hed+/k7f1py7eRwCHA48DrwBn8K9WUxtwAvB03j8XAMt3+O4cnOOeWDHvc8CzubzDgS2AB/J+O71i2+sANwEv5ff9W2CFiuXTgJ3y4/Hk727e73MqpvnA+LzseOBJ0nfvYWDvPP+DwDxgQX7Nq3n+ecDJFdv8PPBE3n9XAavW8lk1w9TwAGoOFHbJO21gN+t8D7gDWAUYBvwDOKkicczP6yxBShhvASt2/LJ08bz9izoQWBZ4HVg/LxsBbNjxDwxYKe/0/fPr9s3PV87Lb85fvFHAoPz8lC7eW3v838nxfx6YBVwMDAE2zF/W9+f1NwO2ztsdCUwFvtLxj76T8v+HlEAGUZGIKr7oU4FlgGuBn9S478aRkwnwmfyeL6tYdmVFDJXbm0b+Y+6wD87K8W0CvA18sIb9/95+6ewzoMMfdRfvI4CrgRVIrY1ZwC4V7+MJ4P3AYOAK4MIOcV9A+u4Mqph3JrA0sHPef3/M8a9GSqA75DLWBcbmfTOMlDx/3tlnRYfvbsU6o3PMm+bnnyL9gLWRfvzeBEZ083m99xkBHyUl4DE5pl8CE2v5rJphaqWm7srA7Oi+Kbof8L2IeDEiZpFqcvtXLH83L383Iq4h/Zqt38t4FgIbSRoUETMi4qFO1vk48HhEXBgR8yPiEuARYI+Kdc6NiMciYi7wO9KXsyvvkvoz3wUuBYYCp0XEG3n7DwEbA0TEpIi4I293GvC/wA41vKfvRsTbOZ5FRMRZpF/wO0nJ/ltVymt3C7CdpDZge+BHwLZ52Q55eU+cGBFzI+J+4H5SAoTq+78eTomIVyPiGeBv/Gt/7QecGhH/jIg5wDeAfTo0a8dHxJsdPtuTImJeRFxHSjyX5PifB24FNgWIiCci4vq8b2YBp1J9f75H0jBSUj0qIu7LZf4+IqZHxMKIuIy0b7esscj9gHMi4t6IeDu/3//I/bDtuvqsGq6VEt9LwNAq/SOrkpoa7Z7O894ro0PifIv069wjEfEm6RfycGCGpD9L+kAN8bTHtFrF8xd6EM9LEbEgP27/45lZsXxu++sljZJ0taQXJL1O6hcd2k3ZALMiYl6Vdc4CNgJ+mb/wVUXEk6QfmdHAdqSawHRJ69O7xNfVZ1Zt/9dDT7Y9kNQX3e7ZTsrruP+62p+rSLpU0vN5f15E9f1Jfu0SwP8BF0fEpRXzD5A0WdKrkl4l7deayqTD+83J/iV6/93uU62U+G4nNQX26mad6aSDFO3WzPN6401Sk67d+yoXRsS1ETGWVPN5hJQQqsXTHtPzvYypJ35Nimu9iFgO+CapH6073R7ilzSY1G/2G2C8pJV6EM8twCdJ/YzP5+cHACuSjsz3OJ5OdLf/F9mfkhbZn73YVi3bns+iiWxxtvHD/PqN8/78LNX3Z7tfkvrx3jtiLWkt0nf2SFLXywrAlIoyq8W6yPuVtCypVdYX3+3F1jKJLyJeI/VvnSFpL0nLSFpC0q6SfpRXuwQ4QdIwSUPz+hf1cpOTge0lrSlpeVJVHgBJwyX9V97Zb5NqMws6KeMaYJSkz0gaKOm/gQ1INZ6iDSH1Q87JtdEjOiyfSeqP6onTgEkRcQjwZ1L/FACSxku6uZvX3kL6I5uYn99MGj50W0UttqOextjd/r8f2FDSaElLk/rBFmdbnW37q5LWzj8QPyD1Y9ZrlMAQ8oEGSasBx9byIkmHkWrVn4mIhRWLliUlt1l5vc+RanztZgKrS1qyi6IvBj6XP8+lSO/3ztyt0vRaJvEBRMSppDF8J5B22LOkP6Y/5lVOBu4hHRV7ELg3z+vNtq4HLstlTWLRZNVGOjo8nXREawfgC52U8RKwe173JdKRyd0jYnZvYuqhY0gHEt4g/bJf1mH5eOD83Mz5dLXCJO1JOsB0eJ51NDBG0n75+Rqko9NduYX0x9ue+G4j1cAmdvmKVMs5Icd4TLUY6Wb/R8RjpIMfN5D6sjqO+/wNsEHe1h/puXNIR6Inko7yzyMl9no5kXQg4TXSj84VNb5uX1JCny5pTp6+GREPAz8ltaRmAh9i0f13E6nP+AVJ//Z9jYgbgW8Dl5NGDawD7NObN9YILTeA2ZqTpMnAx3KyN2tqTnxmVjot1dQ1s/KStIakv0maKukhSV/O88fno92T87Rb1bJc4zOzViBpBGmA9b2ShpD63vcCPg3MiYif1FqWzxk0s5YQETNIB1KIiDckTWXRcYM1c40P0MBBoSWHNDqM0tv0g2s2OgQDnn56GrNnz651jGBVA5ZbK2L+v50I9G9i7qyHSEfD202IiAmdrZvPEJlIGoJzNOkUu9dJR/W/FhGvdLctJz6gbZlVYqn1q47osIK9cvfpjQ7BgG232pxJk+6pW+Kr9e9r3uQzJkXE5tXWy+MkbyGdvnmFpOGk84aDdNGPERExrrsy3NQ1s2JJ0DagTkVpCdLYwd9GxBUAETGzYvlZ1HCCgI/qmlnx1FZ9qlaEJNJA86n5ZIb2+SMqVtubdOpdt1zjM7PiqS4t521JV9t5MA+Yh3QO+r6SRpOautOAw6oV5MRnZgVTTTW6aiLdXqKzDHpNT8ty4jOzYom69fHVixOfmRVM9Wrq1o0Tn5kVrw5N3Xpy4jOzgtVvOEu9OPGZWbGEm7pmVkJu6ppZuQgGuKlrZmUiXOMzsxJyH5+ZlYuP6ppZGbmpa2alIp+5YWZl5BqfmZWL+/jMrIzc1DWzUvE4PjMrHzd1zayMXOMzs9JxH5+ZlUodby9ZL058ZlY4ucZnZmWSrkPqxGdmZSI6vylkAznxmVnBRFubj+qaWcm4qWtmpePEZ2alIgm1OfGZWcm4xmdmpePEZ2blItzUNbPycY3PzEpFqOkSX3ONKjSzfklS1amGMtaQ9DdJUyU9JOnLef5Kkq6X9Hj+f8VqZTnxmVmxch9ftakG84GvRcQHga2BL0raADgeuDEi1gNuzM+75cRnZoWrR40vImZExL358RvAVGA1YE/g/Lza+cBe1cpyH5+ZFa7GPr6hku6peD4hIiZ0Ud5IYFPgTmB4RMyAlBwlrVJtQ058LWb14Stw9kkHMHzl5VgYwTmX/50zLrmZbx22G+M+sQ2zXpkDwHdPv4prb3u4wdGWx2GHjOMv11zNsFVWYdLkKY0Op6mImpuysyNi86rlSYOBy4GvRMTrvTlw4sTXYuYvWMjxp17B5EeeY/AyS/GPi4/jxjsfAeCXF/2Nn194Y4MjLKf9DzyIw79wJIeMO6DRoTQf1W84i6QlSEnvtxFxRZ49U9KIXNsbAbxYrRz38bWYF2a/zuRHngNgzltv88hTL7DqsBUaHJV9eLvtWWmllRodRtOq01FdAb8BpkbEqRWLrgIOzI8PBK6sVpYTXwtbc8RKjF5/de6eMg2Aw/fZnrsu+wZnfnc/VhgyqLHBmVWo01HdbYH9gY9Kmpyn3YBTgLGSHgfG5ufdKjzxSVqQA5wi6U+Sel09kTRN0tAq6/R4TE8rWnbQklzyk0M49ieX88ab8zjr97eywR7j2WqfU3hh9uuccvQnGh2i2XvqdFT3tohQRGwcEaPzdE1EvBQRH4uI9fL/L1crqy9qfHNzgBsBLwNfLHh7PR7T02oGDmzjkp98nsv+cg9X3nQ/AC++/AYLFwYRwTlX/J3NN1qrwVGaJbUkvb4+s6Ovm7q3k8bdACDpWEl3S3pA0okV8/8oaVIenX1oD7fR4zE9rebM7+7Ho0+9wC8uuum9ee8butx7j/f86CY8/OSMRoRm1qm2traqU1/qs6O6kgYAHyN1TiJpZ2A9YEvSrUiukrR9REwExkXEy5IGAXdLujwiXupQ3jXAIRExvcOmejymp5VsM/r97Lf7Vjz42PPccWmqzH739Kv49H9uzsbrr05E8PSMlznq5EsaHGm5HPDZfbn1lpuZPXs264xcnW9/50QOGndwo8NqHs11qm6fJL5BkiYDI4FJwPV5/s55ui8/H0xKhBOBL0naO89fI89fJPFFxG6LE1SuSaba5BKDF6eoPvWPyf9k0KZH/tt8j9lrrAsu8g9Nd8p4kYK5ETEaWAtYkn/18Qn4YUUn5boR8RtJOwI7Af8REZuQEuPSPdjezDyWh+7G9ETEhIjYPCI210AfATUrjOpzcKOe+qxhHRGvAV8CjsmDEK8FxuVR2EhaLTdLlwdeiYi3JH2AdDJyT/R4TI+ZFUeItrbqU1/q0x7FiLgPuB/YJyKuAy4Gbpf0IPB/wBDgr8BASQ8AJwF3dFaWpGskrdrJoh6P6TGzYknVp75UeB9fRAzu8HyPisenAad18rJduyhrZMXjTvv48kGQj/UmVjMrRrP18flcXTMrlAQDBjjxmVnJNFmFz4nPzIrnpq6ZlYpEnx+1rcaJz8wK1nx3WXPiM7PCNVnec+Izs+K5xmdmpeI+PjMrpSar8DnxmVnx3NQ1s3JxU9fMyka4qWtmpeNxfGZWQm7qmlm5NOB6e9U48ZlZoVIfX3NlPic+MyucE5+ZlY77+MysXNzHZ2Zlo1YaziJpue5eGBGv1z8cM+uPBrRQU/chIEgHZdq1Pw9gzQLjMrN+pMkqfF0nvohYoy8DMbP+Kd03t7kyX003FJe0j6Rv5serS9qs2LDMrD8Z0KaqU1+qmvgknQ58BNg/z3oLOLPIoMysf5GqT9XL0DmSXpQ0pWLeeEnPS5qcp91qiaeWGt82EXEYMA8gIl4GlqylcDMzkY/sVvlXg/OAXTqZ/7OIGJ2na2opqJbhLO9KaiMd0EDSysDCWgo3MwOoR0s2IiZKGrn4JdVW4zsDuBwYJulE4Dbgf+qxcTMrAYm2tuoTMFTSPRXToTVu4UhJD+Sm8Iq1vKBqjS8iLpA0Cdgpz/pUREzp7jVmZu0EtNV2VHd2RGzew+J/DZxEapGeBPwUGFftRbWeuTEAeDcXXtORYDOzdkWNZomImf/ahs4Crq7ldbUc1f0WcAmwKrA6cLGkb/QyTjMrmfbbS9bQ1O1F2RpR8XRvoKbWaC01vs8Cm0XEW3lD3wcmAT/saZBmVk41NnW7JekSYEdSX+BzwHeBHSWNJrVGpwGH1VJWLYnv6Q7rDQT+2YN4zazk6tHSjYh9O5n9m96U1d1FCn5GyqJvAQ9JujY/35l0ZNfMrCrRWhcpaG8rPwT8uWL+HcWFY2b9jlroslQR0asqpJlZR02W96r38UlaB/g+sAGwdPv8iBhVYFxm1k80Y1O3ljF55wHnkuLfFfgdcGmBMZlZP6Pc3O1u6ku1JL5lIuJagIh4MiJOIF2txcysJqph6ku1DGd5WykdPynpcOB5YJViwzKz/kKqzzi+eqol8X0VGAx8idTXtzw1nAtnZtau5W4vGRF35odv8K+LkZqZ1azJKnzdDmD+A/kafJ2JiE8UEpGZ9StCLdXUPb3PomiwDUetzpXX/bjRYZTePufd0+gQDHjypbfqW6BaqKkbETf2ZSBm1n8127Xsar0en5lZr4jmu72kE5+ZFW5gk1X5ak58kpaKiLeLDMbM+p+WvKG4pC0lPQg8np9vIumXhUdmZv1Gm6pPfRpPDev8AtgdeAkgIu7Hp6yZWQ/U44bi9VRLU7ctIp7uUFVdUFA8ZtbPCBjYZE3dWhLfs5K2BELSAOAo4LFiwzKz/qTJ8l5Nie8IUnN3TWAmcEOeZ2ZWldRaZ24AEBEvAvv0QSxm1k8NaLXhLPkmvf92zm5EHFpIRGbWr4jWvCzVDRWPlybdtPfZYsIxs/6oyfJeTU3dyyqfS7oQuL6wiMysfxEMaLLM15tT1tYG1qp3IGbWP6WmbqOjWFQtfXyv8K8+vjbgZeD4IoMys/6lpRJfvtfGJqT7bAAsjIguL05qZtaZljpXNye5P0TEgjw56ZlZj0hpOEu1qS/Vsrm7JI0pPBIz67fa8iDm7qa+1N09NwZGxHzgw8DnJT0JvEnqq4yIcDI0s6pa7eDGXcAYYK8+isXM+iW11HAWAUTEk30Ui5n1Q+nS842OYlHdJb5hko7uamFEnFpAPGbW39TpQqOSziFdG/TFiNgoz1sJuAwYCUwDPh0Rr1Qrq7uDGwOAwcCQLiYzs6oEDGhT1akG5wG7dJh3PHBjRKwH3EiNY4y7q/HNiIjv1VKImVl36nHUNiImShrZYfaewI758fnAzcBx1cqq2sdnZra4asx7QyVV3lV+QkRMqPKa4RExAyAiZkhapZYNdZf4PlZLAWZm3VHtFymYHRGbFx0PdNPHFxEv90UAZtb/qYapl2ZKGgGQ/3+xlhc12XVRzay/ab8QaUFnblwFHJgfHwhcWcuLnPjMrHD1qPFJugS4HVhf0nOSDgZOAcZKehwYm59X1Zvr8ZmZ9YBoq8NAvojYt4tFPT4e4cRnZoUSzde0dOIzs8I12/X4nPjMrFhqzbusmZn1mpu6ZlZKbuqaWem00oVIzcwWW2rqNlfmc+Izs8I1WUvXic/MiibkGp+ZlYmo+eosfcaJz8yKJTd1zayEnPis7hYsWMBeY7dl+IhVOfu3VzQ6nNI4cruRbL7m8rw2dz5fvuIhAPYZsypj1x/K6/PmA3DR3c8z6bnXGhlmw7mpa4U4b8IZrDPqA8x54/VGh1IqNz0+m2sefpEv77D2IvOvmjKTKx+c2aComlOzHdxotjNJrIdmTH+Ov93wVz6930GNDqV0Hn5hDnPent/oMFqCVH3qS67xtbiTT/g6x33nZN6cM6fRoVj28Q1W4SPrrcwTs97i3Duf5c13FjQ6pIZqxqZuYTU+SQskTa6YRnaz7o6Srq7Tdr8v6VlJ/T4T3HTdNaw8dBgf2mRMo0Ox7C9TX+Tw3z3IV694mFfmvsvntlqj0SE1AdX0ry8V2dSdGxGjK6ZpBW6r0p+ALftoWw016a47uPHaP7P9Zh/gy4cewO233cLRR4xrdFil9trc+SwMCOD6R2ax3rBlGx1S49XQzO3rCmGf9vFJGinpVkn35mmbTtbZQtJ9kt4vaVlJ50i6O8/bs9o2IuKO9vts9nfHnvA9/n7/E0yc9AinTbiA//jwDpz663MaHVaprThoifcebzVyRZ55ZW4Do2keBd5lrVeK7OMbJGlyfvxUROxNuvXb2IiYJ2k94BLgvfto5kT4S2DPiHhG0g+AmyJinKQVgLsk3QAsD5wdEbv1NjhJhwKHAqy6upsj1nNHf2RtNhoxhOWWHsjZ+27MpZOms9GIIay98iACePGNd/j1bU83OsyGa8Y+viIT39yIGN1h3hLA6ZJGAwuAURXLPghMAHaOiOl53s7Af0k6Jj9fGlgzIqYCvU56APkO7RMAPjR6TCxOWc1g6223Z+ttt290GKVy6t+e+rd5Nzw2uwGRtIDmynt9flT3q8BMYBNSM3texbIZpMS2KdCe+AT8v4h4tC+DNLP6Kvs4vuWBGRGxENgfGFCx7FXg48APJO2Y510LHKV8+VZJm/ZhrGZWJ22qPvVpPH27OX4FHCjpDlIz983KhRExE9gDOEPSVsBJpObxA5Km5OdIWlXSNZ1tQNKPJD0HLJNvOjy+sHdjZrVpsqMbhTV1I2JwJ/MeBzaumPWNPP9m4Ob8+Blgw4p1DuuknOl00ccXEV8Hvt7LsM2szlJea66mrs/cMLNiNaApW40Tn5kVz4nPzMrFl543s5IRbuqaWRk58ZlZ2bipa2al02Sn6jrxmVnB6njZKUnTgDdI5/rPj4jNu39F55z4zKxwdW7qfiQiFutqEE58ZlYo0XxNXd9syMwKV+MVmIdKuqdiOrSTogK4TtKkLpbXxDU+MytcjU3d2TX02W0bEdMlrQJcL+mRiJjY03hc4zOzwtXrnhvtFymOiBeBP9DL++s48ZlZ4eqR+PI9eIa0PyZdoX1Kb+JxU9fMClXHy1INB/6Qr0s8ELg4Iv7am4Kc+MysWHUaxxcR/yTdtmKxOfGZWeGabDSLE5+ZFU2oyQbyOfGZWeGaLO858ZlZsRpwL6GqnPjMrHBu6ppZ6TRZ3nPiM7PiNVnec+Izs4LJTV0zK5lmvCyVE5+ZFa7J8p4Tn5kVzzU+Mysd9/GZWek0V9pz4jOzgvXkQqN9xYnPzArnpq6ZlU5zpT0nPjPrA01W4XPiM7NiCdHWZJnPNxsys9Jxjc/MCtdkFT4nPjMrXp3uslY3TnxmVigJ2por7znxmVkfcOIzs7JxU9fMSsdNXTMrHyc+MyubZmvqKiIaHUPDSZoFPN3oOBbTUGB2o4OwfrEf1oqIYfUqTNJfSZ9LNbMjYpd6bbc7Tnz9hKR7ImLzRsdRdt4PrcGnrJlZ6TjxmVnpOPH1HxMaHYAB3g8twX18ZlY6rvGZWek48ZlZ6TjxlYya7a4vJSNpQKNjMPfx9XuStgAGAPMiYnKep/CO71OS1o2IJ/LjARGxoNExlZlrfP2YpF2Bi4FDgJMk/QIgIsI1v74jaQ/gMUkXAETEAtf8GsuJrx9SsgRwKPCdiDgEGAdsK+lccPLrK5KGAZ8FDgDWl3Q+OPk1mhNfPxTJu8AkYEGeNwvYCviQpFPb12tclOWQP/fTIuIiYDtgTGXya2hwJebE1789D3xD0soAETEf2BMYKWlUQyMrgfYadUT8Q1JbRLwDjCElvwvyOttI2rCRcZaRE18/FhHnAjcCd0oamuc9D7wLDGpkbGVQWaOOiIWSBuaa+GakZu9DwFnAG42Ksax8Pb5+KtcwFkbEMZIC+Kuk04FVgI2BVxsbYflExPyc/N7Jfa0nAztGxDONjq1sXOPrRyoPVuQaxoD8+FjgZ8CKwCbApyKi1a8/2LQkdfl3lZPfesBY4GMRMaXvIrN2HsfX4iQNj4iZFc+Xj4jXKp63RcTCrp5bMSS9DxgVERO7WL7IfrK+5Rpfi8pDVoYCd0n6TJ43APiFpG3b1+skyfmXrs66GBa0CbB2V+s66TWWa3wtLg+O/SnwrYj4vaTBETGn0XGVReVZMJJ2AZYGprSfpWHNyTW+FtRea8h/dH8CvgL8SNI+lUlP0hgPUi5WRdL7AvAdYBRwn6T3Lj8vaU9JOzQoROuEE1+L6XCe7TBJy0bENcARwA8l/XdebxngGNJRXCtI7nIYBexMOmDxMnAXcG/Fam8D0xsQnnXBTd0WJekYYGtgeeD7EXGzpJ2AX+Xn5/tk+PrreIGHfAR3EHAkMBJ4P7B7RLwr6Sjgkoho9buu9Tuu8bUgSYcBu0bEJ0l9SqdJ+mRE3EBq9n5V0nKAj97W34rtDyTtB/wAWIp0MGNrYK+c9D5NOj96mYZEad3yAOYW0F7LqKhtLA0cIulrpKbVxcCpkgZFxIWSbo6ItxoadD8kaTXgbEk/j4hrgfnAUxHxsqRvAheRfoQGARsC+3twcnNyU7fJdThq+N7Yr3zVj4uAPfKZALcAzwBH+KhuMSStCOwD7Ar8GBgOvC8iTs/LVwI2BZYFHoiIaQ0K1apwja/JVSS9I4AtJD0N3ATcDcwBviLpWeAF4AQnvfpr//GJiFckXUq64s0xpDGRL0najtTPtxRwT0TMaGC4VgMnvhaQ+5I+Q7qu242kvtnbgfOAvYFPAQf5NLT661DjHgi8FhET8jChI0k1vAeBLUj9f19sWLBWMzd1m1y+oOhxpIS3LrA/6ajhO5KGRcQsSStGxCsNDbSfk/RVUr/dysAxEfGkpM+RfnjGR8S93RZgTcU1viYj6cPAh4CngNsj4rXcvP01MDsidsrrHUNqcv3MSa/+8lVU5ufHRwG7k2rWVwO3StopIs7N/X7HSToYeMvnQbcGJ74mIuk/gZ8A9wMfBraUdDLwGCkRXitpBLA9sF+erM7yaYBbShoPLAEMIR3UGEfaD38BbpO0TUScKmkF9622Fjd1m4SkjwJ/BNaPiBn5j2+PiDg0L98D2JE0Xuxd4OsR8WCj4u2vJI0lHbE9Lg9ZQdJSpIHJE4CxETFP0iOkGvfofHFRayFOfE1C0sbAfaSxXxfnebcBVwJTSX1884AlgSVcw6i//ONzFTAmIh6TNBLYKiIuk7Qq8H3gTGA9YB3gXI/Ta01u6jaJiHhA0lbA9bmGsSrpKOEYYBvgDFIz+FcR8XbjIu3XZpOGpYyU9ARpYPiFedkbwEukO9ftQKqNO+m1KNf4mozSDcCvA16OiHUq5u8OTI6I5xoWXAlUfP4LgCMj4tL2i7fmCz8MBJaOiBcbGqgtFp+r22Qi4m5SjWIlSftXzL/aSa94+fPfHqi8563yBR/eiojXnfRan5u6TSg3e8eSrq48MN8tzfpIRDwoaWfgunzE9swFT/9gAAADZElEQVRGx2T15cTXpCLiHkmbAb7YQANExN35Ml93S3rbPz79i/v4zLohaVPSwORHGx2L1Y8Tn5mVjg9umFnpOPGZWek48ZlZ6TjxmVnpOPGZWek48ZWYpAWSJkuaIun3+ZSs3pa1o6Sr8+P/knR8N+uukG/A3dNtjM/XIaxpfod1zpP0yR5sa6SkKT2N0VqDE1+5zY2I0RGxEfAOcHjlwnyz7B5/RyLiqog4pZtVVgB6nPjM6sWJz9rdCqybazpTJf0KuBdYQ9LOkm6XdG+uGQ4GkLSLpEfy5bM+0V6QpIMktd95bLikP0i6P0/bAKcA6+Ta5o/zesdKulvSA5JOrCjrW5IelXQDsH61NyHp87mc+yVd3qEWu5OkWyU9li/6gKQBkn5cse3DFveDtObnxGftN9HZlXTTHEgJ5oKI2BR4EzgB2CkixgD3AEdLWho4C9gD2A54XxfF/wK4JSI2IV1i6yHgeODJXNs8Np8Xux6wJTAa2EzS9vmUvX1IN/T5BOmGPtVcERFb5O1NBQ6uWDaSdAGIjwNn5vdwMOkGQlvk8j8vae0atmMtzOfqltsgSZPz41uB35CuA/h0RNyR528NbAD8Pd1YjCVJd3j7AOlm2o8DSLqIdK26jj4KHAAQEQuA1/J9KirtnKf78vPBpEQ4BPhD+83RJV1Vw3vaKF+uf4VczrUVy36X74nxuKR/5vewM7BxRf/f8nnbj9WwLWtRTnzlNjciRlfOyMntzcpZwPURsW+H9UaT7itbDwJ+GBH/22EbX+nFNs4D9oqI+yUdRLpcf7uOZUXe9lHtl5mv2PbIHm7XWoibulbNHcC2ktYFkLSMpFHAI8DaktovlrpvF6+/ETgiv3aApOVIVzMeUrHOtcC4ir7D1SStAkwE9pY0SNIQUrO6miHADKXbcna8GdOnJLXlmN8PPJq3fUReH0mjJC1bw3ashbnGZ93K9+09CLgkXxIf4IR8T4pDgT9Lmg3cBmzUSRFfBiYo3X5xAXBERNwu6e95uMhfcj/fB4Hbc41zDvDZiLhX0mXAZOBpUnO8mm8Dd+b1H2TRBPsocAswHDg83zTobFLf371KG58F7FXbp2OtyldnMbPScVPXzErHic/MSseJz8xKx4nPzErHic/MSseJz8xKx4nPzErn/wNycJvL11Z3eQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes = ['Real: 0', 'Fake: 1'],\n",
    "                      title='Confusion matrix, without normalization')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "       Real       0.86      0.96      0.91        26\n",
      "       Fake       0.94      0.79      0.86        19\n",
      "\n",
      "avg / total       0.89      0.89      0.89        45\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(data1_test_label, data1_predict, target_names=['Real', 'Fake']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lưu lại model1: ```MultiNB_clf_1```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./Models/MultiNB_clf_1.sav']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1_files = 'MultiNB_clf_1.sav'\n",
    "joblib.dump(MultiNB_clf_1, os.path.join(models_path, model1_files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.2 Tìm kiếm tham số tốt hơn cho mô hình \n",
    "Tham khảo thêm tại:\n",
    "1. [Working with text data - SKLEARN](https://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html)\n",
    "2. [SVM Parameter Tuning in Scikit Learn using GridSearchCV](https://medium.com/@aneesha/svm-parameter-tuning-in-scikit-learn-using-gridsearchcv-2413c02125a0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "MultiNB_clf_2 = Pipeline([\n",
    "    ('tfidfvect', TfidfVectorizer()),\n",
    "    ('nbclf', MultinomialNB())\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Các thông số kiểm chứng"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    'tfidfvect__ngram_range': [(1, 1), (1, 2)],\n",
    "    'tfidfvect__use_idf': (True, False),\n",
    "    'nbclf__alpha': (1e-2, 1e-3),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_clf = GridSearchCV(MultiNB_clf_2, parameters, cv=5, iid=False, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "gs_clf = gs_clf.fit(data1_train_text, data1_train_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8945602745602745\n",
      "nbclf__alpha: 0.01\n",
      "tfidfvect__ngram_range: (1, 2)\n",
      "tfidfvect__use_idf: True\n"
     ]
    }
   ],
   "source": [
    "print(gs_clf.best_score_)\n",
    "\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"%s: %r\" % (param_name, gs_clf.best_params_[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3. Xây dựng mô hình trên các tham số trên"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "MultiNB_text_clf = Pipeline([\n",
    "    ('tfidfvect', TfidfVectorizer(ngram_range = (1, 2))),\n",
    "    ('nbclf', MultinomialNB(alpha = 0.01))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tfidfvect', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=Tru...,\n",
       "        vocabulary=None)), ('nbclf', MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MultiNB_text_clf.fit(data1_train_text, data1_train_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Đánh giá mô hình"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1_predited_3 = MultiNB_text_clf.predict(data1_test_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. __Accuracy__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9111111111111111"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(data1_predited_3 == data1_test_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. __Confussion Matrix__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnf_matrix2 = confusion_matrix(y_pred=data1_predited_3, y_true=data1_test_label)\n",
    "np.set_printoptions(precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[23  3]\n",
      " [ 1 18]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUcAAAEYCAYAAADPkTRJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XecXFX9//HXexMIgYRm6C0KAQWEAFIEaQKRKuAXFUSkiqDo14KFIgSx8LOgIiAiIiBS9AsoIhAiSlNKQghNQg8QEkISJLQEUj6/P87ZOJlM283cnZ3N+5nHfWRumXPOzN397Dnn3nuOIgIzM1tYR6sLYGbWGzk4mplV4OBoZlaBg6OZWQUOjmZmFTg4mplV4OCYSRoo6S+SZkr642Kkc6ikW5pZtlaRtKOkx3tLfpKGSgpJ/XuqTO1C0kRJu+fXJ0u6qIA8LpD07Wan21up3e5zlPQp4KvAe4HXgfHA9yLirsVM9zDgi8D2ETF3sQvay0kKYFhEPNXqslQjaSJwTET8La8PBZ4Flmr2OZJ0CTApIk5tZro9pfy7akJ6R+T0PtSM9NpRW9UcJX0V+BnwfWA1YF3gfGD/JiS/HvDEkhAYG+HaWXH83baJiGiLBVgBeAP4eI1jBpCC5+S8/AwYkPftAkwCvga8DEwBjsz7zgDeAebkPI4GRgKXl6Q9FAigf14/AniGVHt9Fji0ZPtdJe/bHhgDzMz/b1+y7zbgTOCfOZ1bgCFVPltn+b9RUv4DgL2BJ4BXgJNLjt8GuBt4NR97LrB03ndH/ixv5s/7yZL0vwm8BPyuc1t+z/o5jy3z+prAdGCXBs7dpcDX8uu1ct6fz+sb5HRVlt/vgPnArFzGb5Scg8OB53P+pzR4/hc6L3lb5PyPzef+nZzXX6p8jgCOA54E/gOcx39bXx3AqcBz+fxcBqxQ9rNzdC73HSXbjgReyOkdB2wNPJTP27klea8P/B2YkT/374EVS/ZPBHbPr0eSf3bzeX+jZJkLjMz7vgU8TfrZ+zdwYN7+PmA2MC+/59W8/RLguyV5fhZ4Kp+/64E1G/mu2mVpeQEaLijsmU9s/xrHfAe4B1gVWAX4F3Bm3rdLfv93gKVIQeUtYKXyH6gq650/zP2B5YDXgI3yvjWATcp/CYGV8w/GYfl9h+T1d+X9t+Ufzg2BgXn9rCqfrbP8p+XyfxaYBlwBDAY2yT/Q78nHbwVsl/MdCjwGfLnsh3eDCun/P1KQGUhJsCr5ZXgMWBYYBfy4wXN3FDngAJ/Kn/nqkn1/LilDaX4Tyb/wZefg17l8mwNvA+9r4PwvOC+VvgPKfvGrfI4AbgBWJLVapgF7lnyOp4D3AIOAa4HflZX7MtLPzsCSbRcAywAj8vn7Uy7/WqQgu3NOYwNgj3xuViEF2J9V+q4o+9ktOWZ4LvMWef3jpD9yHaQ/kG8Ca9T4vhZ8R8CHSUF6y1ymXwB3NPJdtcvSTs3qdwHTo3az91DgOxHxckRMI9UIDyvZPyfvnxMRN5L+Km7UzfLMBzaVNDAipkTEoxWO2Qd4MiJ+FxFzI+JKYAKwX8kxv42IJyJiFvAH0g9wNXNI/atzgKuAIcDPI+L1nP+jwGYAEXF/RNyT850I/ArYuYHPdHpEvJ3Ls5CI+DWpJnAv6Q/CKXXS63Q7sKOkDmAn4IfADnnfznl/V5wREbMi4kHgQVKQhPrnvxnOiohXI+J54B/893wdCpwdEc9ExBvAScDBZU3okRHxZtl3e2ZEzI6IW0jB6cpc/heBO4EtACLiqYgYnc/NNOBs6p/PBSStQgq8X4yIB3Kaf4yIyRExPyKuJp3bbRpM8lDg4ogYFxFv58/7wdwv3Knad9UW2ik4zgCG1OmvWZPUrOn0XN62II2y4PoW6a98l0TEm6S/tMcBUyT9VdJ7GyhPZ5nWKll/qQvlmRER8/Lrzl+wqSX7Z3W+X9KGkm6Q9JKk10j9tENqpA0wLSJm1znm18CmwC/yL0VdEfE06Q/RcGBHUo1isqSN6F5wrPad1Tv/zdCVvPuT+sY7vVAhvfLzV+18rirpKkkv5vN5OfXPJ/m9SwH/B1wREVeVbP+MpPGSXpX0Kum8NpQmZZ83/0GYQfd/tnuddgqOd5OaHQfUOGYy6cJKp3Xztu54k9R87LR66c6IGBURe5BqUBNIQaNeeTrL9GI3y9QVvySVa1hELA+cTOrXq6XmrQuSBpH68X4DjJS0chfKcztwEKnf88W8/hlgJdIdB10uTwW1zv9C51PSQuezG3k1kvdcFg52i5PHD/L7N8vn89PUP5+dfkHqV1xwJV7SeqSf2RNI3TwrAo+UpFmvrAt9XknLkVp3PfGz3SPaJjhGxExSf9t5kg6QtKykpSTtJemH+bArgVMlrSJpSD7+8m5mOR7YSdK6klYgNRsAkLSapI/mH4i3SbWieRXSuBHYUNKnJPWX9ElgY1LNqWiDSf2ib+Ra7fFl+6eS+se64ufA/RFxDPBXUn8ZAJJGSrqtxntvJ/0i3pHXbyPdOnVXSW24XFfLWOv8PwhsImm4pGVI/XKLk1elvL8i6d35j8j3Sf2qzbr7YTD54oiktYCvN/ImSZ8j1c4/FRHzS3YtRwqA0/JxR5Jqjp2mAmtLWrpK0lcAR+bvcwDp896bu3D6hLYJjgARcTbpHsdTSSf1BdIv3J/yId8FxpKu9j0MjMvbupPXaODqnNb9LBzQOkhXvSeTrtTtDHy+QhozgH3zsTNIV1z3jYjp3SlTF51IuvjxOqmGcHXZ/pHApblJ9Yl6iUnan3RR7Li86avAlpIOzevrkK66V3M76Re8MzjeRarJ3VH1Ham2dGou44n1ykiN8x8RT5Au2PyN1LdWfl/sb4CNc15/ousuJl1hv4N098JsUvBvljNIFz9mkv4wXdvg+w4hBf3Jkt7Iy8kR8W/gJ6QW2VTg/Sx8/v5O6sN+SdIiP68RcSvwbeAa0t0Q6wMHd+eD9VZtdxO49U6SxgO75T8IZm3PwdHMrIK2alabmfUUB0czswocHM3MKvAD8ID6DwwtPbjVxVjivX+jdVpdBANeeP45XpkxvdF7KOvqt/x6EXMXeeBqETFr2qiI2LNZ+S4uB0dASw9mwEZ172axgt34j7NbXQQD9t71g01NL+bOauj3a/b48xp9OqdHODiaWbEk6OjXhGS0DmnwjtVJ4wBcGBE/l/Qj0ngF75AGNTkyIl6t8P6JpPt+5wFzI+IDtfJzn6OZFU8d9Zf65pKGvnsfacSpL0jaGBgNbBoRm5GG7zupRhq7RsTweoERHBzNrCdI9Zc68uhX4/Lr10nD560VEbeUPKZ5D7B2M4rs4GhmBVOjNcchksaWLMdWTTENjbYFafi8UkcBN1V5WwC3SLq/Vtqd3OdoZsUSjfY5Tm+kuZsH9riGNHjzayXbTyE1vX9f5a07RMRkSasCoyVNiIiqz/a75mhmBWugSd1AsxoWjE15DfD7iLi2ZPvhpEFeDo0qz0RHxOT8/8vAddQZ2NfB0cyK14QLMpJEGj3psTxCV+f2PUlzH300It6q8t7lJA3ufE2aluKRWvm5WW1mBWvOrTykqTUOAx7Oo0BBGsT5HNI8NqNT/OSeiDhO0prARRGxN2lE9uvy/v6kUdFvrpWZg6OZFUs03GyuJdLc9JUSurHK8ZNJE+kREc/w37mGGuLgaGbFa+w+xl7FwdHMCibo15RmdY9ycDSzYgnXHM3MKmpCn2NPc3A0s4I17Wp1j3JwNLPiuVltZlamC0/A9CYOjmZWPNcczczKuc/RzKwyN6vNzMr4Pkczs0rcrDYzq6wNa47tV2Izaz9NGOxW0jqS/iHpMUmPSvrfvH1lSaMlPZn/X6nK+w/PxzyZB8etycHRzIrVOTVrvaW+arMPfgu4NSKGAbfm9bIiaGXgdGBb0gjgp1cLop0cHM2scJLqLvVUm30Q2B+4NB92KXBAhbd/BBgdEa9ExH9I07nuWSs/9zmaWaHSWLcN3cozRNLYkvULI+LCimkuPPvgahExBVIAzRNolVsLeKFkfVLeVpWDo5kVS1Qev3tR3Zp9sMHAW+mgihNxdXKz2swKJjo6OuouDaVUefbBqZLWyPvXAF6u8NZJwDol62sDk2vl5eBoZoVrRp9jtdkHgeuBzqvPhwN/rvD2UcAISSvlCzEj8raqHBzNrHDNCI78d/bBD0san5e9gbOAPSQ9CeyR15H0AUkXAUTEK8CZwJi8fCdvq8p9jmZWKEmoo9DZBwF2q3D8WOCYkvWLgYsbzc/B0cwK12DNsFdxcDSzwjk4mpmVE01pVvc0B0czK5xrjmZmZUTDV6N7FQdHMyucg6OZWTn3OZqZVeaao5lZBQ6OVri1V1uRi878DKu9a3nmR3DxNf/kvCtv47TP78O+O2/G/AimvfI6x55+OVOmzWx1cZcYs2fP5n/22Y133n6befPmsvdHP8aJJ53W6mL1CqI5T8j0NAfHNjN33ny+dfa1jJ8wiUHLDuBfV3yTW++dwE8vvZXvnP9XAD5/yM6cdOxefOl7V7W4tEuOAQMG8Ic/j2K5QYOYM2cOB+61K7vu/hG22nrbVhet9eSao/WAl6a/xkvTXwPgjbfeZsKzL7HmKisy4ZmXFhyz7MABRNQcqs6aTBLLDRoEwNw5c5g7Z05bBoSitON34eDYxtZdY2WGb7Q2Yx6ZCMDIL+zHoftuw8w3ZrHnsee0tnBLoHnz5rHXLtsx8dmnOfzo49jyA9u0uki9Rjs2qwsfskzSvDy00COS/iJpxcVIa6KkIXWOaWgmsna33MClufLHx/D1H1/D62/OBmDkeX9h2F7f5qqbxnLcJ3dqcQmXPP369eOWO8cw5tFnGD9uLBP+/Wiri9RrNGk8x4slvSzpkZJtV5cMXzZR0vgq750o6eF83NhKx5TrifEcZ0XE8IjYFHgF+ELB+dWdiazd9e/fwZU//ixX3zSWP//9wUX2/+GmMRyw2/AWlMwAVlhhRT74oZ247daaY6kuMRoJjA02uy+hbFKsiPhkji/DSSOEX1vpjdmu+di6UzFAzw92ezclk9pI+rqkMZIeknRGyfY/Sbo/z017bBfzaGQmsrZ2wemH8vizL3HO5X9fsG39dVdZ8HqfnTfjiYlTW1G0JdaM6dOYOfNVAGbNmsVdt/2dDYZt1OJS9R7NmCYhIu4gVbAWkUcJ/wRwZbPK3GN9jpL6kQak/E1eHwEMI80hK+B6STvlL+CoiHhF0kBgjKRrImJGWXo3AsdERPk8EI3MRNa2th/+Hg7dd1sefuJF7rkqVYpPP/d6jjhge4attyrz5wfPT3nFV6p72NSXXuIrnz+aefPmEfPns++BB7H7nvu0uli9R/FdjjsCUyPiySr7A7hFUgC/qjarYameCI4Dcz/AUOB+0nyxkOZwGAE8kNcHkYLlHcCXJB2Yt6+Tty8UHCNi78UpVK6RplrpUoMWJ6ke9a/xzzBwixMW2T7qrn+3oDTWaeNN38+oO+5rdTF6rWZPzVrBIdSuNe4QEZNzZWm0pAm5IlZVTwTHWRExXNIKwA2kPsdzSH9LfhARvyo9WNIuwO7AByPiLUm3Act0Ib+pktbItcZqM5GRv/QLATqWXdX3vZgVpfH7HBuamnWR5KX+wMeAraod09nCjIiXJV1HarHWDI491ucYETOBLwEn5ukVRwFH5TlokbRWjuorAP/JgfG9wHZdzKqRmcjMrIcI0dFRf1kMuwMTImJSxfyl5SQN7nxNarE+UunYUj16QSYiHgAeBA6OiFuAK4C7JT0M/B8wGLgZ6C/pIdJsYfdUSkvSjZLWrLCr4kxkZtY6Uv2lfhq6knRRdyNJkyQdnXcdTFmTWtKa+boEwGrAXZIeBO4D/hoRN9fLr/BmdUQMKlvfr+T1z4GfV3jbXlXSGlryumKfY75ws8hMZGbWOs14QiYiDqmy/YgK2yYDe+fXzwCbdzU/PyFjZoWSoF+/9ntCxsHRzArXho9WOziaWfE88ISZWRmJxb0a3RIOjmZWMM8+aGZWURvGRgdHMyuea45mZmXc52hmVkUbVhwdHM2seG5Wm5mVc7PazGxRws1qM7MKfJ+jmVlF7dis7ukJtsxsSdPAWI4NjudYaWrWkZJeLJmeteJQhpL2lPS4pKckNTQjqYOjmRUq9TkWMzVr9tPO6Vkj4sbynXlyv/NI48RuDBwiaeN6mTk4mlnhmhEca03NWsc2wFMR8UxEvANcRZrCuSYHRzMrXINzyAyRNLZkaXTO+hMkPZSb3StV2L8W8ELJ+qS8rXaZG8zczKx7Gu9znB4RHyhZGpmW9ZfA+sBwYArwk8olWETdGUd9tdrMCqUCb+WJiKkL8pF+TZr+udwkYJ2S9bWByfXSrhocJS1fp1Cv1UvczAygX0G38nTOUZ9XD6TylKtjgGGS3g28SJqt8FP10q5Vc3yUVPUs/VSd6wGsW7/oZmbNeUImT826C6lvchJwOrCLpOGkmDQR+Fw+dk3goojYOyLmSjoBGAX0Ay6OiEfr5Vc1OEbEOtX2mZk1KvUpFjY162+qHLtgata8fiOwyG0+tTR0QUbSwZJOzq/XlrRVVzIxsyVbvw7VXXqbusFR0rnArsBhedNbwAVFFsrM+pZmPCHT0xq5Wr19RGwp6QGAiHhF0tIFl8vM+giRrli3m0aC4xxJHeT7giS9C5hfaKnMrE/pha3muhoJjucB1wCrSDoD+ARwRqGlMrO+Q2rLUXnqBseIuEzS/cDuedPHI6LSvURmZosQ0NEbOxXraPQJmX7AHFLT2o8cmlmXtGFsbOhq9SnAlcCapMdurpB0UtEFM7O+oXNq1gYGnuhVGqk5fhrYKiLeApD0PeB+4AdFFszM+o6+2qx+ruy4/sAzxRTHzPqi9guNtQee+Cmpj/Et4FFJo/L6COCunimembU7UdzAE0WqVXPsvCL9KPDXku33FFccM+tzGp8GoVepNfBExQe6zcy6qg1jY/0+R0nrA98jTUyzTOf2iNiwwHKZWR/RrGa1pIuBfYGXI2LTvO1HwH7AO8DTwJER8WqF904EXgfmAXMj4gP18mvknsVLgN+SPuNewB9IE9SYmTWkwNkHRwObRsRmwBNArdsMd80zFNYNjNBYcFw2IkYBRMTTEXEqaZQeM7OGqIGlnkqzD0bELRExN6/eQ7oXuykauZXnbaWw/rSk40jDjK/arAKYWd8mNXyf4xBJY0vWL2xwkq1ORwFXV9kXwC2SAvhVI+k2Ehy/AgwCvkTqe1whF8LMrCENPgEzvdEmb7n8JN9c4PdVDtkhIiZLWhUYLWlCrolW1cjAE/fml6/z3wFvzcwaVuTVakmHky7U7BYRFadczdMmEBEvS7oO2AboXnDMCVSd2zUiPtZAuc1sCSdU2OODkvYEvgns3PmIc4VjlgM6IuL1/HoE8J16adeqOZ7bncK2oy3ety7/vHeJ+bi91rAv/7nVRTBg6oszm5ugGm5W106m8uyDJwEDSE1lgHsi4rjS2QeB1YDr8v7+wBURcXO9/GrdBH7rYn4WMzOgOeMcdnf2wYh4Bti8q/k1Op6jmVm3iOZMzdrTHBzNrHD923CI7IaDo6QBEfF2kYUxs74nTb3afjXHRkYC30bSw8CTeX1zSb8ovGRm1md0qP7S2zRS2T2HdA/RDICIeBA/PmhmXZBqj7WX3qaRZnVHRDxXVi2eV1B5zKyPEdC/N0a/OhoJji9I2gYISf2AL5JGvzAza0gbxsaGguPxpKb1usBU4G95m5lZXVJxT8gUqZFnq18GDu6BsphZH9WvL97KI+nXVHjGOiKOLaREZtaniL47NevfSl4vAxwIvFBMccysL2rD2NhQs3qhwSMl/Y40NLmZWX2Cfm0YHbvz+OC7gfWaXRAz65tSs7rVpei6Rp6Q+Y+kV/LyKqnWeHLxRTOzvqIZT8hIuljSy5IeKdm2sqTRkp7M/69U5b2H52OezIPj1i9zncKINNTPKnlZKSLeExF/aCRxMzModPbBbwG3RsQw4Na8Xp73yqSxH7cljQB+erUgWqpmcMxDjl8XEfPyUnVkcDOzSqR0K0+9pZ5Ksw8C+wOX5teXAgdUeOtHgNER8UpE/IfU+i0Psoto5O6j+yRt2cBxZmYVdeQbwWst3bRaREwByP9Xmhl1LRa+w2ZS3lZTrTlk+uf5YD8EfFbS08CbpP7ViAgHTDOrqwsXZBZ3atZaRShXtxVc62r1fcCWVK6mmpk1SI3eytOdqVmnSlojIqZIWgN4ucIxk0hzz3RaG7itXsK1gqMAIuLpxstpZrawNE1CYclfDxwOnJX/rzRL2yjg+yUXYUaQJuaqqVZwXEXSV6vtjIiz6yVuZkaTBrOtMvvgWcAfJB0NPA98PB/7AeC4iDgmIl6RdCYwJif1nYgov7CziFrBsR8wiMrtdTOzhgjo14ToWGX2QYDdKhw7FjimZP1i4OKu5FcrOE6JiLoTX5uZ1dPXBp5ov09jZr1SG8bGmsFxkaqqmVlXqa8NPNFIh6WZWSPaLzR2b1QeM7OG9eXBbs3MFkv7hUYHRzMrnOhowwEdHRzNrFCisRFuehsHRzMrXIPjNfYqDo5mViz5goyZ2SLcrDYzq8LNajOzCtrwYrWDo5kVKzWr2y86tmNXgJm1Gan+Uj8NbSRpfMnymqQvlx2zi6SZJcec1t0yu+ZoZgUTakLNMSIeB4YDSOoHvAhcV+HQOyNi38XNz8HRzAolChmVZzfg6Yh4rtkJd3Kz2syK1UCTOsfOIZLGlizH1kj1YODKKvs+KOlBSTdJ2qS7xXbN0cwK12DFsaHZByUtDXyUypNkjQPWi4g3JO0N/AkY1oWiLuDg2OY+d8xR3HTjDayy6qrcP/6RVhdnifLjQ4ez26arM+P1t9n9+/8AYOO1lucHB2/OgKX6MW9+cMrVDzL+uVdbXNLWKqBZvRcwLiKmlu+IiNdKXt8o6XxJQyJielczcbO6zR12+BH8+YabW12MJdIf73mBw867e6FtpxywCT+96XH2POs2fnzDY5x8QLdbdX2KGvjXBYdQpUktaXXlO84lbUOKcTO6U2bXHNvch3bciecmTmx1MZZI9z49g7VXHrjQtgAGL5N+rZYfuBRTZ85uQcl6n2ZVHCUtC+wBfK5k23EAEXEBcBBwvKS5wCzg4IiI7uTl4GjWRCP/72Eu/8IHOfXATekQHPCTO1tdpJZrZrM6It4C3lW27YKS1+cC5zYjr8Ka1ZLmld2wObTGsbtIuqFJ+X5P0guS3mhGemZdcdiO7+aMax9h22/fwhnXPMKPDt2i1UXqBRppVPe+J2iK7HOcFRHDS5aJBeZV6i/ANj2Ul9lCDtp2HW4aPwWAGx6YzPD1VmxxiXqBxm/l6VV69IKMpKGS7pQ0Li/bVzhma0kPSHqPpOUkXSxpTN62f708IuKeiJhSzCcwq23qzNlsNyy1+nbYcAjPTnuzxSXqHdTA0tsU2ec4UNL4/PrZiDgQeBnYIyJmSxpGuuK04L6mHCx/AewfEc9L+j7w94g4StKKwH2S/gasAFwUEXt3t3D5BtNjAdZZd93uJtNyn/n0Idx5+21Mnz6d9YeuzbdPO4Mjjjq61cVaIpx7xFZsN2wIKw9amvvOHMFPbpzAN68Yz8iD3k//DvH23Pl868rx9RPq4wp6QqZwRQbHWRExvGzbUsC5koYD84ANS/a9D7gQGBERk/O2EcBHJZ2Y15cB1o2Ix4BuB0aAiLgw58dWW32gW1ezeoPLLq/2kIAV7YRL7q+4fZ8f3t7DJWkD7Rcbe/xq9VeAqcDmpCZ96X0OU0jBbwugMzgK+J/8wLmZtaneeMGlnp6+CXwFYEpEzAcOA/qV7HsV2Af4vqRd8rZRwBdLbur0pT+zNtSh+ktv09PB8XzgcEn3kJrUC/VW58eB9gPOk7QtcCapKf6QpEfyOpLWlHRjpQwk/VDSJGBZSZMkjSzs05hZY9rwikxhzeqIGFRh25PAZiWbTsrbbwNuy6+fB0qfufocZXKfZMU+x4j4BvCNbhbbzJosxb5eGP3q8BMyZlasXtpsrsfB0cyK5+BoZlaudz4eWI+Do5kVSrRns9rjOZpZ8Zp0tVrSREkP58FsxlbYL0nnSHpK0kOStuxukV1zNLPCNblZvWuNkb33Ik2LMAzYFvhl/r/LXHM0s8L14Kg8+wOXRXIPsKKkNbqTkIOjmRWrubMPBnCLpPur7F8LeKFkfVLe1mVuVptZ4RpsVjcy++AOETFZ0qrAaEkTIuKOhbJaVLcGlnHN0cwKJZrXrO4csSsiXgauY9GBrScB65Ssr81/B7LpEgdHMytcM4JjHvx6cOdr0pCG5fMRXw98Jl+13g6Y2d3Br92sNrPCNelq9WrAdXmQrv7AFRFxc9nsgzeSxl14CngLOLK7mTk4mlnhmnE1OiKeIY0FW769dPbBAL6w+Lk5OJpZD2jDWRIcHM2sWB6yzMyskl469Wo9Do5mVrg2jI0OjmZWNKE2rDo6OJpZ4dowNjo4mlmxeun8WXU5OJpZ4dysNjOroA1jo4OjmRWvDWOjg6OZFUxuVpuZLaJzyLJ24+BoZoVrw9jo8RzNrHhNGs9xHUn/kPSYpEcl/W+FY3aRNDPPTjhe0mndLbNrjmZWuCb1Oc4FvhYR4/Kgt/dLGh0R/y477s6I2HdxM3PN0cwK14xpqyNiSkSMy69fBx6jm5NnNcLB0cwK1UiTuqsVS0lDgS2Aeyvs/qCkByXdJGmT7pbbzWozK1yDzeohksaWrF8YERdWSGsQcA3w5Yh4rWz3OGC9iHhD0t7An4Bh3Smzg6OZFa7BimHdqVklLUUKjL+PiGvL95cGy4i4UdL5koZExPSuldjNajPrAU26Wi3gN8BjEXF2lWNWz8chaRtSjJvRnTK75mhmhRKiozlXq3cADgMeljQ+bzsZWBcWTLR1EHC8pLnALODgPOlWlzk4mllbiIi7qNNCj4hzgXObkZ+Do5kVzo8PmplV4NkHzczKSNDRfrHRwdHMeoCDo5nZotysNjOrwM1qM7NKHBzNzBbVjs1qdfPm8T5F0jTguVaXYzENAbr8/Kg1XV84D+tFxCrNSkzSzaTvpZ7pEbFns/JdXA6OfYSksfUe2rfi+Tz0HR54wszSI/XpAAAJOUlEQVSsAgdHM7MKHBz7jkUGBbWW8HnoI9znaGZWgWuOZmYVODiamVXg4LiEUZMmELbukdSv1WWwxrjPsY+TtDXQD5gdEePzNnV36HjrHkkbRMRT+XW/iJjX6jJZba459mGS9gKuAI4BzpR0DkBEhGuQPUfSfsATki4DiIh5rkH2fg6OfZCSpYBjgdMi4hjgKGAHSb8FB8ieImkV4NPAZ4CNJF0KDpDtwMGxD4pkDnA/MC9vmwZsC7xf0tmdx7WulEuG/L3/PCIuB3YEtiwNkC0tnNXk4Ni3vQicJOldABExF9gfGCppw5aWbAnQWTOPiH9J6oiId4AtSQHysnzM9pI2aWU5rTIHxz4sIn4L3ArcK2lI3vYiMAcY2MqyLQlKa+YRMV9S/1yj34rUxH4U+DXweqvKaNV5PMc+KtdU5kfEiZICuFnSucCqwGbAq60t4ZInIubmAPlO7vv9LrBLRDzf6rLZolxz7ENKL7Dkmkq//PrrwE+BlYDNgY9HRLuPX9lrSar6e5UD5DBgD2C3iHik50pmXeH7HNucpNUiYmrJ+goRMbNkvSMi5ldbt2JIWh3YMCLuqLJ/ofNkvY9rjm0q364zBLhP0qfytn7AOZJ26DyuQiD0X8Mmq3JL1ObAu6sd68DY+7nm2ObyDcY/AU6JiD9KGhQRb7S6XEuK0qeNJO0JLAM80vk0jLUv1xzbUGftI/9i/gX4MvBDSQeXBkZJW/pG72KVBMbPA6cBGwIPSFowVYKk/SXt3KIiWjc5OLaZsueiV5G0XETcCBwP/EDSJ/NxywInkq5OW0Fy98aGwAjSRZZXgPuAcSWHvQ1MbkHxbDG4Wd2mJJ0IbAesAHwvIm6TtDtwfl6/1AMcNF/5oB35yvRA4ARgKPAeYN+ImCPpi8CVEdHusxEukVxzbEOSPgfsFREHkfq4fi7poIj4G6mJ/RVJywO+Kt18K3W+kHQo8H1gAOkCzHbAATkwfoL0PPuyLSmlLTbfBN4GOmsrJbWWZYBjJH2N1Iy7Ajhb0sCI+J2k2yLirZYWug+StBZwkaSfRcQoYC7wbES8Iulk4HLSH6qBwCbAYb7Bu325Wd3LlV0NXXBvXB7t5XJgv/zExe3A88DxvlpdDEkrAQcDewE/AlYDVo+Ic/P+lYEtgOWAhyJiYouKak3gmmMvVxIYjwe2lvQc8HdgDPAG8GVJLwAvAac6MDZf5x+oiPiPpKtIIx2dSLpndIakHUn9jgOAsRExpYXFtSZxcGwDuW/rU6RxAW8l9RXfDVwCHAh8HDjCjwQ2X1nNvT8wMyIuzLdInUCqKT4MbE3qj/xCywprTeVmdS+XB639JikobgAcRroa+o6kVSJimqSVIuI/LS1oHyfpK6R+xHcBJ0bE05KOJP1xGhkR42omYG3HNcdeRtKHgPcDzwJ3R8TM3JT+JTA9InbPx51Iat791IGx+fLoOXPz6y8C+5Jq6DcAd0raPSJ+m/shvynpaOAtP7fedzg49iKSPgL8GHgQ+BCwjaTvAk+QguUoSWsAOwGH5sWaLD+SuY2kkcBSwGDShZijSOfhJuAuSdtHxNmSVnRfb9/jZnUvIenDwJ+AjSJiSv4F3S8ijs379wN2Id1PNwf4RkQ83Kry9lWS9iBdif5mvl0HSQNIN3dfCOwREbMlTSDV3IfnAWytj3Fw7CUkbQY8QLo37oq87S7gz8BjpD7H2cDSwFKuqTRf/gN1PbBlRDwhaSiwbURcLWlN4HvABcAwYH3gt76Pse9ys7qXiIiHJG0LjM41lTVJVz+3BLYHziM1uc+PiLdbV9I+bTrplpyhkp4i3Vz/u7zvdWAGaUbHnUm1egfGPsw1x15G0tbALcArEbF+yfZ9gfERMallhVsClHz/84ATIuKqzgGC82Ae/YFlIuLllhbUCudnq3uZiBhDqpmsLOmwku03ODAWL3//OwGlc0orD+LxVkS85sC4ZHCzuhfKTew9SKN898+zCFoPiYiHJY0AbslXoi9odZms5zk49lIRMVbSVoAHkGiBiBiTh4AbI+lt/4Fa8rjP0awGSVuQbu5+vNVlsZ7l4GhmVoEvyJiZVeDgaGZWgYOjmVkFDo5mZhU4OJqZVeDguASTNE/SeEmPSPpjfjyuu2ntIumG/Pqjkr5V49gVJX2+G3mMzONYNrS97JhLJB3UhbyGSnqkq2W0vsPBcck2KyKGR8SmwDvAcaU784T1Xf4ZiYjrI+KsGoesCHQ5OJr1JAdH63QnsEGuMT0m6XxgHLCOpBGS7pY0LtcwBwFI2lPShDy02sc6E5J0hKTOGflWk3SdpAfzsj1wFrB+rrX+KB/3dUljJD0k6YyStE6R9LikvwEb1fsQkj6b03lQ0jVlteHdJd0p6Yk8kAeS+kn6UUnen1vcL9L6BgdH65w4ai/SRFGQgtBlEbEF8CZwKrB7RGwJjAW+KmkZ4NfAfsCOwOpVkj8HuD0iNicNv/Yo8C3g6Vxr/Xp+jnkYsA0wHNhK0k758cmDSZNYfYw0iVU910bE1jm/x4CjS/YNJQ3qsQ9wQf4MR5Mmzdo6p/9ZSe9uIB/r4/xs9ZJtoKTx+fWdwG9I40g+FxH35O3bARsD/0wT7rE0aebD95ImtH8SQNLlpLEOy30Y+AxARMwDZuZ5V0qNyMsDeX0QKVgOBq6LiLdyHtc38Jk2zVNLrJjTGVWy7w95jpcnJT2TP8MIYLOS/sgVct5PNJCX9WEOjku2WRExvHRDDoBvlm4CRkfEIWXHDSfN29wMAn4QEb8qy+PL3cjjEuCAiHhQ0hGkqSU6lacVOe8vdk6JUJL30C7ma32Mm9VWzz3ADpI2AJC0rKQNgQnAuyV1Dsh7SJX33wocn9/bT9LypFG1B5ccMwo4qqQvcy1JqwJ3AAdKGihpMKkJX89gYIrSlLblE5B9XFJHLvN7gMdz3sfn45G0oaTlGsjH+jjXHK2mPC/2EcCVefoGgFPzHCvHAn+VNB24C9i0QhL/C1yoNHXpPOD4iLhb0j/zrTI35X7H9wF355rrG8CnI2KcpKuB8cBzpKZ/Pd8G7s3HP8zCQfhx4HZgNeC4PFHWRaS+yHFKmU8DDmjs27G+zKPymJlV4Ga1mVkFDo5mZhU4OJqZVeDgaGZWgYOjmVkFDo5mZhU4OJqZVfD/AX+uUDnJEzQMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix2, classes = ['Real: 0', 'Fake: 1'],\n",
    "                      title='Confusion matrix, without normalization')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "       Real       0.96      0.88      0.92        26\n",
      "       Fake       0.86      0.95      0.90        19\n",
      "\n",
      "avg / total       0.92      0.91      0.91        45\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_true=data1_test_label, y_pred=data1_predited_3, target_names=['Real', 'Fake']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lưu lại model1: ```MultiNB_clf_1```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./Models/MultiNB_clf_2.sav']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1_files = 'MultiNB_clf_2.sav'\n",
    "joblib.dump(MultiNB_text_clf, os.path.join(models_path, model1_files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2. Mô hình cho dữ liệu từ file ```preproc_vn_news_223_tdlfr.csv``` - text, domain & label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2.1. Mô hình & đánh giá\n",
    "Nếu mô hình ở phần 1 dễ dàng đạt được do việc classify trên 1  tham số chính là text, ở đây chúng ta phải classify trên 2 feature: ```text``` và ```domain```\n",
    "\n",
    "Tham khảo:\n",
    "1. [How can I use sklearn.naive_bayes with (multiple) categorical features?](https://stackoverflow.com/a/38622421/5144980)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "MultiNB_text_domain_clf_2 = MultinomialNB().fit(X_data2_train_tfidf, y_news2_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)\n"
     ]
    }
   ],
   "source": [
    "print(MultiNB_text_domain_clf_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "MultiNB_domain_clf_2 = MultinomialNB().fit(X_data2_domain_train_tfidf, y_news2_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2 kết quả dự đoán trên 2 mô hình"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_data2_tokenizer = data2_tfidf_vectorizer.transform(X_news2_test['text_domain'])\n",
    "y_predicted_text_domain = MultiNB_text_domain_clf_2.predict(X_data2_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_data2_domain_tokenizer = data2_domain_tfidf_vectorizer.transform(X_news2_test['domain'])\n",
    "y_predicted_domain = MultiNB_domain_clf_2.predict(X_data2_domain_tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Đánh giá mô hình trên dataset Data2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. __Accuracy__:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mô hình ```Text & Domain```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8666666666666667"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(y_predicted_text_domain == y_news2_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mô hình ```Domain```: Overfit vì dữ liệu quá ít"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9333333333333333"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(y_predicted_domain == y_news2_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. __Confusion Matrix__\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mô hình ```Text & Domain```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnf_matrix_data2_dt = confusion_matrix(y_predicted_text_domain, y_news2_test)\n",
    "np.set_printoptions(precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[26  4]\n",
      " [ 2 13]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAEYCAYAAADFzZobAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XmcHFW5//HPdxKWQEJYEjAgGNmRLQQJyhoVuOCFC6goiyBrhHtBBVERuRIEhZ8iCIJiAFll0YsgILIFQwDZAoTNsBO2hEDYAwGyPL8/zhnsjDPTPZNep77vvOqV7qrqU093TT99zqmqU4oIzMyKpK3RAZiZ1ZsTn5kVjhOfmRWOE5+ZFY4Tn5kVjhOfmRWOE18maYCkayS9JelPC1HOXpJurGZsjSJpS0mPN8v2JA2XFJL61yumViFpqqRt8uOjJZ1Tg22cJel/q11uI6jVzuOTtCdwBLA28A4wGfhpRNy+kOXuDRwGbBYRcxc60CYnKYA1IuKpRsfSFUlTgQMj4ub8fDjwLLBItfeRpPOBFyPimGqWWy8dP6sqlLdvLm+LapTXbFqqxifpCOBXwM+AFYBVgN8AO1eh+E8ATxQh6VXCtara8WfbBCKiJSZgMDAL2K2bdRYjJcZpefoVsFheNhp4Efgu8AowHdgvLzsO+BCYk7dxADAWuLik7OFAAP3z832BZ0i1zmeBvUrm317yus2Ae4G38v+blSybABwP3JHLuREY0sV7a4//+yXx7wJ8EXgCeB04umT9UcCdwJt53TOARfOyifm9vJvf79dKyv8B8DJwUfu8/JrV8jZG5ucrAjOB0RXsuwuA7+bHK+Vt/3d+vnouVx22dxEwH5idY/x+yT74BvB83v6PKtz/C+yXPC/y9sfkff9h3tY1XbyPAA4GngTeAM7kX62mNuAY4Lm8fy4EBnf42zkgxz2xZN5+wAu5vIOBTYCH8n47o2TbqwG3AK/l9/0HYOmS5VOBbfLjseS/3bzfZ5VMc4GxedlRwNOkv71/Arvm+esA7wPz8mvezPPPB04o2eZBwFN5/10NrFjJZ9UMU8MDqDhQ2D7vtP7drPMT4C5geWAo8A/g+JLEMTevswgpYbwHLNPxj6WL5+1/qP2BJYG3gbXysmHAuh2/YMCyeafvnV+3R36+XF4+If/hrQkMyM9P6uK9tcf/4xz/QcCrwCXAIGDd/Me6al5/Y+AzebvDgSnAdzp+6Tsp//+REsgAShJRyR/6FGAJ4Abg5Ar33f7kZALsmd/z5SXL/lISQ+n2ppK/zB32wdk5vg2BD4B1Ktj/H+2Xzj4DOnypu3gfAVwLLE1qbbwKbF/yPp4CVgUGAn8GLuoQ94Wkv50BJfPOAhYHtsv776oc/0qkBLp1LmN1YNu8b4aSkuevOvus6PC3W7LOiBzzRvn5bqQfsDbSj9+7wLBuPq+PPiPg86QEPDLH9GtgYiWfVTNMrdTUXQ6YGd03RfcCfhIRr0TEq6Sa3N4ly+fk5XMi4jrSr9lavYxnPrCepAERMT0iHu1knf8EnoyIiyJibkRcCjwG7FSyznkR8UREzAb+SPrj7MocUn/mHOAyYAhwWkS8k7f/KLABQETcFxF35e1OBX4HbF3Bezo2Ij7I8SwgIs4m/YLfTUr2PypTXrtbgS0ltQFbAT8HNs/Lts7Le+K4iJgdEQ8CD5ISIJTf/9VwUkS8GRHPA3/nX/trL+CUiHgmImYBPwR279CsHRsR73b4bI+PiPcj4kZS4rk0x/8ScBuwEUBEPBURN+V98ypwCuX350ckDSUl1cMi4oFc5p8iYlpEzI+Iy0n7dlSFRe4F/D4i7o+ID/L7/Wzuh23X1WfVcK2U+F4DhpTpH1mR1NRo91ye91EZHRLne6Rf5x6JiHdJv5AHA9Ml/VXS2hXE0x7TSiXPX+5BPK9FxLz8uP3LM6Nk+ez210taU9K1kl6W9DapX3RIN2UDvBoR75dZ52xgPeDX+Q++rIh4mvQjMwLYklQTmCZpLXqX+Lr6zMrt/2roybb7k/qi273QSXkd919X+3N5SZdJeinvz4spvz/Jr10E+D/gkoi4rGT+PpImS3pT0puk/VpRmXR4vznZv0bv/7brqpUS352kpsAu3awzjXSQot0qeV5vvEtq0rX7WOnCiLghIrYl1XweIyWEcvG0x/RSL2Pqid+S4lojIpYCjib1o3Wn20P8kgaS+s3OBcZKWrYH8dwKfIXUz/hSfr4PsAzpyHyP4+lEd/t/gf0paYH92YttVbLtuSyYyBZmGyfm12+Q9+fXKb8/2/2a1I/30RFrSZ8g/c0eSup6WRp4pKTMcrEu8H4lLUlqldXjb3uhtUzii4i3SP1bZ0raRdISkhaRtIOkn+fVLgWOkTRU0pC8/sW93ORkYCtJq0gaTKrKAyBpBUn/lXf2B6TazLxOyrgOWFPSnpL6S/oa8ClSjafWBpH6IWfl2ughHZbPIPVH9cRpwH0RcSDwV1L/FACSxkqa0M1rbyV9ySbm5xNIpw/dXlKL7ainMXa3/x8E1pU0QtLipH6whdlWZ9s+XNIn8w/Ez0j9mNU6S2AQ+UCDpJWA71XyIknfJNWq94yI+SWLliQlt1fzevuRanztZgAfl7RoF0VfAuyXP8/FSO/37tyt0vRaJvEBRMQppHP4jiHtsBdIX6ar8ionAJNIR8UeBu7P83qzrZuAy3NZ97FgsmojHR2eRjqitTXw352U8RqwY173NdKRyR0jYmZvYuqhI0kHEt4h/bJf3mH5WOCC3Mz5arnCJO1MOsB0cJ51BDBS0l75+cqko9NduZX05W1PfLeTamATu3xFquUck2M8slyMdLP/I+IJ0sGPm0l9WR3P+zwX+FTe1lX03O9JR6Inko7yv09K7NVyHOlAwlukH50/V/i6PUgJfZqkWXk6OiL+CfyS1JKaAazPgvvvFlKf8cuS/u3vNSLGA/8LXEE6a2A1YPfevLFGaLkTmK05SZoMfCEne7Om5sRnZoXTUk1dM7NqcOIzs8Jx4jOzwvHF0oD6DwgtOqjRYRTe+mut3OgQDHjh+ed4/bWZlZ4jWFa/pT4RMfffLgT6NzH71RsiYvtqbbc7TnyAFh3EYmuVPaPDauz6Cac0OgQDth/92aqWF3NnV/T9en/ymZVeNbLQnPjMrLYkaOvX6CgW4MRnZrWn5jqc4MRnZrWnqnUZVkVzpWEz64OUanzlpnKlSCtL+rukKZIelfTtPH9sHrVmcp6+WK4s1/jMrLZEtfr45pJG8r5f0iDgPkk35WWnRsTJlRbkxGdmNaaqNHUjYjppQAQi4h1JU1hw/L+KualrZrVXhabuAsWlkZ43Io0GDnCopIck/V7SMuVe78RnZjWWT2cpN6UR1ieVTGM6LS2Nd3gF6R4yb5MG3V2NNML3dNJwW91yU9fMaktU2tSdGRGf7raoNIz+FcAfIuLPABExo2T52VQw0K9rfGZWe9U5qivSgLFT8qDE7fOHlay2K2kI/W65xmdmNSboV5WjupuT7pr3cB74FtK9ZPaQNII0lP5U4JvlCnLiM7PaElW5ciMibqfzGyxd19OynPjMrPaa7MoNJz4zqzEPUmBmReRBCsysUFSdKzeqyYnPzGrPNT4zKxb38ZlZEbmpa2aFUqXz+KrJic/MasxNXTMrItf4zKxw3MdnZoXi20uaWRHJNT4zK5I0DqkTn5kVieh8MKkGcuIzsxoTbW0+qmtmBeOmrpkVjhOfmRWKJNTmxGdmBeMan5kVjhOfmRWLcFPXzIrHNT4zKxQhJz4zKx4nPjMrFvfxmVkRucZnZoXjxGcL5eMrLM05x+/DCsstxfwIfn/FHZx56QQADtl9aw7+2lbMnTef6297hB+d9pfGBlsw8+bNY/vRn2XYiity4eVXNTqcpiF85YYtpLnz5nPUKX9m8mMvMnCJxfjHJT9g/N2Psfyyg9hx9Pps8tUT+XDOXIYuM7DRoRbOOb/9NWustTaz3nm70aE0FzVfja+5xoqxsl6e+TaTH3sRgFnvfcBjz77MikOXZsxuW3LyeTfx4Zy5ALz6xqxGhlk40156kfE3/o09996v0aE0JUllp3py4mthqwxblhFrfZx7H5nK6p9Yns03Wo2JFx7Jjed8m40/tUqjwyuUY394JMf85MSmG3euWahNZad6qvlekjRP0mRJj0i6RtLSC1HWVElDyqyzrKSbJD2Z/1+mt9trZksOWJRLTz6Q7518Be+8+z79+7WxzFJLsNU+J3P0qVdx8c/3b3SIhXHT9X9lyNChbDBiZKNDaVrVqPFJWlnS3yVNkfSopG/n+T3+ztfj52l2RIyIiPWA14H/qfH2jgLGR8QawPj8vE/p37+NS08+iMv/Nom/3PIgAC/NeJOrxqfHkx59jvnzgyHu56uLe+++kxv/9ldGrb8mhxywN7dPnMChY/ZtdFhNo5KkV2FTdy7w3YhYB/gM8D+SPkUvvvP1rpffCazU/kTS9yTdK+khSceVzL9K0n05q4/p4TZ2Bi7Ijy8AdlnoqJvMWcfuxePPvszpF9/y0bxrJjzE6FFrArD6Ksuz6CL9mel+vro4+tgTuO+fz3DPw0/w23MvYoutRnPGuPMbHVZTaWtrKzuVExHTI+L+/PgdYAopn/T4O1+3o7qS+gFfAM7Nz7cD1gBGkW5FcrWkrSJiIrB/RLwuaQBwr6QrIuK1DuVdBxwYEdM6bGqFiJgO6YOStHxt31l9bTZiVfbacVMefuIl7ros/bAde8bVXHDVnfxu7F5M+tPRfDhnHgf++KIGR2pWospdeJKGAxsBd9OL73w9Et8ASZOB4cB9wE15/nZ5eiA/H0hKhBOBb0naNc9fOc9fIPFFxBcXJqhck0y1yUVap0n4j8nPMGCjQztdtv8xF9Y5Gutosy23ZrMtt250GE2nwqbsEEmTSp6Pi4hxnZQ1ELgC+E5EvN2bI8L1SHyzI2KEpMHAtaQ+vtNJvwEnRsTvSleWNBrYBvhsRLwnaQKweA+2N0PSsJz5hwGvdLZS/kDHAbQtsXz08D2ZWaUqP49vZkR8utuipEVISe8PEfHnPLui73ypuvXxRcRbwLeAI3PwNwD75+yNpJVyFXUw8EZOemuTOjF74mrgG/nxNwBfvmDWQEK0tZWfypaTsue5wJSIOKVkUY+/83U9uBERDwAPArtHxI3AJcCdkh4G/g8YBFwP9Jf0EHA8cFdnZUm6TtKKnSw6CdhW0pPAtvm5mTWQVH6qwObA3sDn8ylykyV9kV5852ve1I2IgR2e71Ty+DTgtE5etkMXZQ0vedxpH18+CPKF3sRqZrVRjSszIuJ2uj5M0qPvvK/VNbOakqBfv+a6VteJz8xqrsnGKHDiM7Paa7bRWZz4zKymJCo6altPTnxmVmO+y5qZFVCT5T0nPjOrPdf4zKxQ3MdnZoXUZBU+Jz4zqz03dc2sWNzUNbOiEW7qmlnh+Dw+MysgN3XNrFgqH2+vbpz4zKymUh9fc2U+Jz4zqzknPjMrHPfxmVmxuI/PzIpGrXQ6i6SlunthRLxd/XDMrC/q10JN3UeBYMG7GrU/D2CVGsZlZn1Ik1X4uk58EbFyPQMxs74p3Te3uTJfRTcUl7S7pKPz449L2ri2YZlZX9KvTWWneiqb+CSdAXyOdAdzgPeAs2oZlJn1LVL5qZ4qOaq7WUSMlPQAQES8LmnRGsdlZn2ESEd2m0kliW+OpDbSAQ0kLQfMr2lUZtanNNlB3YoS35nAFcBQSccBXwWOq2lUZtZ3SK135UZEXCjpPmCbPGu3iHiktmGZWV8hoK3JjupWeuVGP2AOqblb0ZFgM7N2TZb3Kjqq+yPgUmBF4OPAJZJ+WOvAzKxvaL+9ZLmpniqp8X0d2Dgi3gOQ9FPgPuDEWgZmZn1HszV1K2m2PseCCbI/8ExtwjGzvkgVTGXLkH4v6RVJj5TMGyvpJUmT8/TFSuLpbpCCU0l9eu8Bj0q6IT/fDri9ksLNzETVBik4HzgDuLDD/FMj4uSeFNRdU7c9qz4K/LVk/l092YCZFZyqMyxVREyUNHyhC6L7QQrOrcYGzMwqzHtDJE0qeT4uIsZV8LpDJe0DTAK+GxFvlHtB2YMbklYDfgp8Cli8fX5ErFlBQGZWcD1o6s6MiE/3sPjfAseTuuGOB34J7F/uRZUc3DgfOI8U/w7AH4HLehicmRWYcnO3u6k3ImJGRMyLiPnA2cCoSl5XSeJbIiJuyBt5OiKOIY3WYmZWkWoc1e20XGlYydNd+dexiW5Vch7fB0rp+GlJBwMvAcv3PEQzKyKpOufxSboUGE3qC3wROBYYLWkEqak7FfhmJWVVkvgOBwYC3yL19Q2mgja0mVm7alyZERF7dDK7VwdhKxmk4O788B3+NRipmVnFmuzCjW5PYL6SPAZfZyLiSzWJyMz6FKGmu2StuxrfGXWLosE2WmcV7ri7MG+3aZ1w8xONDsGAV979oLoFqjpN3Wrq7gTm8fUMxMz6rmYby67S8fjMzHpFNN/tJZ34zKzm+jdZla/ixCdpsYiocuPfzPq6lryhuKRRkh4GnszPN5T065pHZmZ9RpvKT3WNp4J1Tgd2BF4DiIgH8SVrZtYDrXhD8baIeK5DVXVejeIxsz5GQP8ma+pWkvhekDQKCEn9gMMAn3BlZhVrsrxXUeI7hNTcXQWYAdyc55mZlSW11pUbAETEK8DudYjFzPqofq12Oouks+nkmt2IGFOTiMysTxHNd3vJSpq6N5c8Xpw02N8LtQnHzPqiJst7FTV1Ly99Luki4KaaRWRmfYugX5Nlvt5csvZJ4BPVDsTM+qbU1G10FAuqpI/vDf7Vx9cGvA4cVcugzKxvaanEl++1sSHpPhsA8yOiy8FJzcw601LX6uYkd2W+fds8Jz0z6ykpnc5SbqqnSjZ3j6SRNY/EzPqstnwSc3dTPXV3z43+ETEX2AI4SNLTwLukvsqICCdDMyur1Q5u3AOMBHapUyxm1ieppU5nEUBEPF2nWMysD0pDzzc6igV1l/iGSjqiq4URcUoN4jGzvqYBA42W013i6wcMJNf8zMx6Q0C/Jst83SW+6RHxk7pFYmZ9VisNUtBckZpZy2qyvNdt4vtC3aIwsz5LrTRIQUS8Xs9AzKzvaq605xuKm1mNtepApGZmC6W50l5l1+qamS0E0dZWfipbivR7Sa9IeqRk3rKSbpL0ZP5/mUoicuIzs5oSKdGUmypwPrB9h3lHAeMjYg1gPBWOFerEZ2Y1J6nsVE5ETCQNhFxqZ+CC/PgCKhxbwH18ZlZbqvjgxhBJk0qej4uIcWVes0JETAeIiOmSlq9kQ058ZlZT7U3dCsyMiE/XNJjMTV0zq7lqNHW7MEPSsLyNYcArlbzIic/Maq5N5adeuhr4Rn78DeAvlbzITV0zq6nU1F34M/kkXQqMJvUFvggcC5wE/FHSAcDzwG6VlOXEZ2Y1V40LNyJijy4W9XhcASc+M6sxoSa7dsOJz8xqSrTQ6CxmZlWh1hqPz8ysKpz4rGpeeOEFDtxvH2bMeJm2tjb2P2AMh37r240OqzCuOeWHPHXPBJZcejnGnHUtABMu/BVP3jke2tpYcvBy7PTdExm03AoNjrSxmrGp6/P4Wlj//v056ee/ZPLDU7j19rv43VlnMuWf/2x0WIWx4bZfYvcTzllg3me/fCAH/fYaDjrzL6yx6Whuu+TMBkXXXFTBv3py4mthw4YNY6ORIwEYNGgQa6+9DtOmvdTgqIpjlfU3YcCgwQvMW2zJgR89/vD92U13NLNRpPJTPbmp20c8N3Uqkyc/wCajNm10KIX39/NP5eHxV7H4koPY66QLGx1OwxWqqStpnqTJJdPwbtYdLenaKm33p5JekDSrGuW1glmzZrHHV7/ML375K5ZaaqlGh1N4n9v3cL510a2s+7mdmHTNxY0OpwlU0tDtO03d2RExomSaWsNtlboGGFWnbTXcnDlz2OOrX+Zre+zFLrt+qdHhWIn1Ru/I43fc2OgwGq+CZm69K4R17eOTNFzSbZLuz9NmnayziaQHJK0qack83PS9ed7O5bYREXe1j8/V10UEBx90AGutvQ7fPvyIRodjwOsvTf3o8RN33cJyH1+1ccE0EVUw1VMt+/gGSJqcHz8bEbuShozZNiLel7QGcCnw0fhbORH+Gtg5Ip6X9DPglojYX9LSwD2SbgYGA+dExBd7G5ykMcAYgJVXWaW3xTTUP+64g0v+cBHrrbc+m248AoDjTvgZ2+/Q64/FeuDKk47guYfuYfbbb3D617diq70P46l7J/L6i88iiaWWX4kdDjuu0WE2XDP28dUy8c2OiBEd5i0CnCFpBDAPWLNk2TrAOGC7iJiW520H/JekI/PzxYFVImIKsFDf7jyy6ziAjTf+dCxMWY2y+RZbMHtOS4beJ+x61Cn/Nm/Ef1Q0OEjxNFfeq/tR3cOBGcCGpGb2+yXLppMS20ZAe+IT8OWIeLyeQZpZdTXbaT31Po9vMDA9IuYDewP9Spa9Cfwn8DNJo/O8G4DDlIdnlbRRHWM1syqp4UCkvYunvpvjN8A3JN1Faua+W7owImYAOwFnStoUOJ7UPH4o30vzeABJK0q6rrMNSPp5HqRwCUkvShpbs3djZpVpsqMbNWvqRsTATuY9CWxQMuuHef4EYEJ+/Dywbsk63+yknGl00ccXEd8Hvt/LsM2sylJea66mrq/cMLPaakBTthwnPjOrPSc+MysWDz1vZgUj3NQ1syJy4jOzonFT18wKp8ku1XXiM7Ma813WzKyI3NQ1s0IRrvGZWQE58ZlZ4bipa2aF4xqfmRWOE5+ZFYqHpTKz4qnieXySpgLvkO7ZMzciPt39KzrnxGdmNVfl+t7nImLmwhTgxGdmNSbUZJ189b7nhpkVkFR+AoZImlQyjemkqABulHRfF8sr4hqfmdVUD+4lNLOCPrvNI2KapOWBmyQ9FhETexqTa3xmVnOSyk6VyDcaIyJeAa4ERvUmHic+M6u5Cpu6ZcrQkpIGtT8GtgMe6U08buqaWc1V6dDGCsCVuXbYH7gkIq7vTUFOfGZWW6IqR3Uj4hlgw4UPyInPzGrMw1KZWSE1Wd5z4jOz2nONz8wKp9mu3HDiM7Oaa66058RnZjVW6Xl69eTEZ2Y156aumRVOc6U9Jz4zq4Mmq/A58ZlZbQnR1mSZz4MUmFnhuMZnZjXXZBU+Jz4zqz3fZc3MCkWCtubKe058ZlYHTnxmVjRu6ppZ4bipa2bF48RnZkXTbE1dRUSjY2g4Sa8CzzU6joU0BJjZ6CCsT+yHT0TE0GoVJul60udSzsyI2L5a2+2OE18fIWlSBTdjthrzfmgNvmTNzArHic/MCseJr+8Y1+gADPB+aAnu4zOzwnGNz8wKx4nPzArHia9g1Gx3fSkYSf0aHYO5j6/Pk7QJ0A94PyIm53kK7/i6krR6RDyVH/eLiHmNjqnIXOPrwyTtAFwCHAgcL+l0gIgI1/zqR9JOwBOSLgSIiHmu+TWWE18fpGQRYAzw44g4ENgf2FzSeeDkVy+ShgJfB/YB1pJ0ATj5NZoTXx8UyRzgPmBenvcqsCmwvqRT2tdrXJTFkD/30yLiYmBLYGRp8mtocAXmxNe3vQT8UNJyABExF9gZGC5pzYZGVgDtNeqI+Iektoj4EBhJSn4X5nU2k7RuI+MsIie+PiwizgPGA3dLGpLnvQTMAQY0MrYiKK1RR8R8Sf1zTXxjUrP3UeBs4J1GxVhUHo+vj8o1jPkRcaSkAK6XdAawPLAB8GZjIyyeiJibk9+Hua/1BGB0RDzf6NiKxjW+PqT0YEWuYfTLj78HnAosA2wI7BYRrT7+YNOS1OX3Kie/NYBtgS9ExCP1i8za+Ty+FidphYiYUfJ8cES8VfK8LSLmd/XcakPSx4A1I2JiF8sX2E9WX67xtah8ysoQ4B5Je+Z5/YDTJW3evl4nSc6/dFXWxWlBGwKf7GpdJ73Gco2vxeWTY38J/Cgi/iRpYETManRcRVF6FYyk7YHFgUfar9Kw5uQaXwtqrzXkL901wHeAn0vavTTpSRrpk5RrqyTp/TfwY2BN4AFJHw0/L2lnSVs3KETrhBNfi+lwne1QSUtGxHXAIcCJkr6W11sCOJJ0FNdqJHc5rAlsRzpg8TpwD3B/yWofANMaEJ51wU3dFiXpSOAzwGDgpxExQdI2wG/y8wt8MXz1dRzgIR/BHQAcCgwHVgV2jIg5kg4DLo2IVr/rWp/jGl8LkvRNYIeI+AqpT+k0SV+JiJtJzd7DJS0F+Oht9S3T/kDSXsDPgMVIBzM+A+ySk95XSddHL9GQKK1bPoG5BbTXMkpqG4sDB0r6LqlpdQlwiqQBEXGRpAkR8V5Dg+6DJK0EnCPpVxFxAzAXeDYiXpd0NHAx6UdoALAusLdPTm5Obuo2uQ5HDT869yuP+nExsFO+EuBW4HngEB/VrQ1JywC7AzsAvwBWAD4WEWfk5csCGwFLAg9FxNQGhWpluMbX5EqS3iHAJpKeA24B7gVmAd+R9ALwMnCMk171tf/4RMQbki4jjXhzJOmcyNckbUnq51sMmBQR0xsYrlXAia8F5L6kPUnjuo0n9c3eCZwP7ArsBuzry9Cqr0ONuz/wVkSMy6cJHUqq4T0MbELq//ufhgVrFXNTt8nlAUV/QEp4qwN7k44afihpaES8KmmZiHijoYH2cZIOJ/XbLQccGRFPS9qP9MMzNiLu77YAayqu8TUZSVsA6wPPAndGxFu5eftbYGZEbJPXO5LU5DrVSa/68igqc/Pjw4AdSTXra4HbJG0TEeflfr8fSDoAeM/XQbcGJ74mIuk/gJOBB4EtgFGSTgCeICXCGyQNA7YC9sqTVVm+DHCUpLHAIsAg0kGN/Un74W/A7ZI2i4hTJC3tvtXW4qZuk5D0eeAqYK2ImJ6/fDtFxJi8fCdgNOl8sTnA9yPi4UbF21dJ2pZ0xPYH+ZQVJC1GOjF5HLBtRLwv6TFSjXtEHlzUWogTX5OQtAHwAOncr0vyvNuBvwBTSH187wOLAou4hlF9+cfnamBkRDwhaTiwaURcLmlF4KfAWcAawGrAeT5PrzW5qdskIuIhSZsCN+Uaxoqko4Qjgc2AM0nN4N9ExAeNi7RPm0k6LWW3iYszAAAD70lEQVS4pKdIJ4ZflJe9A7xGunPd1qTauJNei3KNr8ko3QD8RuD1iFitZP6OwOSIeLFhwRVAyec/Dzg0Ii5rH7w1D/zQH1g8Il5paKC2UHytbpOJiHtJNYplJe1dMv9aJ73ay5//VkDpPW+VB3x4LyLedtJrfW7qNqHc7N2WNLpy/3y3NKuTiHhY0nbAjfmI7VmNjsmqy4mvSUXEJEkbAx5soAEi4t48zNe9kj7wj0/f4j4+s25I2oh0YvLjjY7FqseJz8wKxwc3zKxwnPjMrHCc+MyscJz4zKxwnPjMrHCc+ApM0jxJkyU9IulP+ZKs3pY1WtK1+fF/STqqm3WXzjfg7uk2xuZxCCua32Gd8yV9pQfbGi7pkZ7GaK3Bia/YZkfEiIhYD/gQOLh0Yb5Zdo//RiLi6og4qZtVlgZ6nPjMqsWJz9rdBqyeazpTJP0GuB9YWdJ2ku6UdH+uGQ4EkLS9pMfy8Flfai9I0r6S2u88toKkKyU9mKfNgJOA1XJt8xd5ve9JulfSQ5KOKynrR5Iel3QzsFa5NyHpoFzOg5Ku6FCL3UbSbZKeyIM+IKmfpF+UbPubC/tBWvNz4rP2m+jsQLppDqQEc2FEbAS8CxwDbBMRI4FJwBGSFgfOBnYCtgQ+1kXxpwO3RsSGpCG2HgWOAp7Otc3v5eti1wBGASOAjSVtlS/Z2510Q58vkW7oU86fI2KTvL0pwAEly4aTBoD4T+Cs/B4OIN1AaJNc/kGSPlnBdqyF+VrdYhsgaXJ+fBtwLmkcwOci4q48/zPAp4A70o3FWJR0h7e1STfTfhJA0sWkseo6+jywD0BEzAPeyvepKLVdnh7IzweSEuEg4Mr2m6NLurqC97ReHq5/6VzODSXL/pjvifGkpGfye9gO2KCk/29w3vYTFWzLWpQTX7HNjogRpTNycnu3dBZwU0Ts0WG9EaT7ylaDgBMj4ncdtvGdXmzjfGCXiHhQ0r6k4frbdSwr8rYPax9mvmTbw3u4XWshbupaOXcBm0taHUDSEpLWBB4DPimpfbDUPbp4/XjgkPzafpKWIo1mPKhknRuA/Uv6DleStDwwEdhV0gBJg0jN6nIGAdOVbsvZ8WZMu0lqyzGvCjyet31IXh9Ja0pasoLtWAtzjc+6le/buy9waR4SH+CYfE+KMcBfJc0EbgfW66SIbwPjlG6/OA84JCLulHRHPl3kb7mfbx3gzlzjnAV8PSLul3Q5MBl4jtQcL+d/gbvz+g+zYIJ9HLgVWAE4ON806BxS39/9Sht/Fdilsk/HWpVHZzGzwnFT18wKx4nPzArHic/MCseJz8wKx4nPzArHic/MCseJz8wK5/8D9yZ+uoISZxoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix_data2_dt, classes = ['Real: 0', 'Fake: 1'],\n",
    "                      title='Confusion matrix, without normalization')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "       Real       0.93      0.87      0.90        30\n",
      "       Fake       0.76      0.87      0.81        15\n",
      "\n",
      "avg / total       0.87      0.87      0.87        45\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_predicted_text_domain, y_news2_test, target_names=['Real', 'Fake']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lưu lại model1: ```MultiNB_clf_textdomain_1```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./Models/MultiNB_clf_textdomain_1.sav']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1_files = 'MultiNB_clf_textdomain_1.sav'\n",
    "joblib.dump(MultiNB_text_domain_clf_2, os.path.join(models_path, model1_files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2.2 Tìm kiếm tham số tốt hơn cho mô hình \n",
    "Tham khảo thêm tại:\n",
    "1. [Working with text data - SKLEARN](https://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html)\n",
    "2. [SVM Parameter Tuning in Scikit Learn using GridSearchCV](https://medium.com/@aneesha/svm-parameter-tuning-in-scikit-learn-using-gridsearchcv-2413c02125a0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "MultiNB_domain_clf_2 = Pipeline([\n",
    "    ('tfidfvect', TfidfVectorizer()),\n",
    "    ('nbclf', MultinomialNB())\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Các thông số kiểm chứng"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    'tfidfvect__ngram_range': [(1, 1), (1, 2)],\n",
    "    'tfidfvect__use_idf': (True, False),\n",
    "    'nbclf__alpha': (1e-2, 1e-3),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_clf = GridSearchCV(MultiNB_domain_clf_2, parameters, cv=5, iid=False, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "gs_clf = gs_clf.fit(X_news2_train['text_domain'], y_news2_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8985714285714286\n",
      "nbclf__alpha: 0.01\n",
      "tfidfvect__ngram_range: (1, 1)\n",
      "tfidfvect__use_idf: False\n"
     ]
    }
   ],
   "source": [
    "print(gs_clf.best_score_)\n",
    "\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"%s: %r\" % (param_name, gs_clf.best_params_[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.3. Xây dựng mô hình trên các tham số trên"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "MultiNB_text_domain_clf = Pipeline([\n",
    "    ('tfidfvect', TfidfVectorizer(ngram_range = (1, 2))),\n",
    "    ('nbclf', MultinomialNB(alpha = 0.001))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tfidfvect', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=Tru...\n",
       "        vocabulary=None)), ('nbclf', MultinomialNB(alpha=0.001, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MultiNB_text_domain_clf.fit(X_news2_train['text_domain'], y_news2_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Đánh giá mô hình"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predicted_text_domain3 = MultiNB_text_domain_clf.predict(X_news2_test['text_domain'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. __Accuracy__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8666666666666667"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(y_predicted_text_domain3 == y_news2_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. __Confussion Matrix__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnf_matrix3 = confusion_matrix(y_pred=y_predicted_text_domain3, y_true=y_news2_test)\n",
    "np.set_printoptions(precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[24  4]\n",
      " [ 2 15]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUcAAAEYCAYAAADPkTRJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XecXFX9//HXexNKgECAQKSHXqSEAAEBBQQiIAhYKSJVBEUFvyj1K6BYfqgoAsoXBWlSVARRSogoTWkJhCahCiQkEEIJCQmQhM/vj3MmTCYzO7ObmZ2d2fczj/vI3DLnnJm7+9lzzr33HEUEZmY2v45mF8DMrDdycDQzK8PB0cysDAdHM7MyHBzNzMpwcDQzK8PBMZM0QNJfJU2T9MeFSOdASbfWs2zNIumjkp7sLflJGiopJPXvqTK1CknPS9olvz5Z0m8bkMcFkv633un2Vmq1+xwlHQB8C9gAmA6MA34QEXcvZLoHAV8Hto2IOQtd0F5OUgDrRsQzzS5LJZKeB46IiL/n9aHAf4FF6n2OJF0CTIyIU+uZbk8p/a7qkN4hOb3t65FeK2qpmqOkbwG/AH4IDAFWB34F7F2H5NcAnuoLgbEWrp01jr/bFhERLbEAywAzgM91csxipOA5KS+/ABbL+3YEJgL/A0wBJgOH5n1nAO8Bs3MehwOnA1cUpT0UCKB/Xj8EeI5Ue/0vcGDR9ruL3rct8AAwLf+/bdG+24HvA//K6dwKDK7w2Qrl/05R+fcB9gCeAl4HTi46fgRwD/BmPvY8YNG87878Wd7On/cLRemfALwMXF7Ylt+zds5jeF5fGZgK7FjDubsU+J/8epWc91fz+jo5XZXkdznwPjArl/E7RefgYODFnP8pNZ7/+c5L3hY5/yPzuX8v5/XXCp8jgKOAp4E3gPP5oPXVAZwKvJDPz2XAMiU/O4fnct9ZtO1QYEJO7yhgK+CRfN7OK8p7beAfwGv5c/8eGFS0/3lgl/z6dPLPbj7vM4qWOcDped+JwLOkn73/APvm7RsC7wBz83vezNsvAc4syvPLwDP5/N0ArFzLd9UqS9MLUHNBYbd8Yvt3csz3gHuBFYEVgH8D38/7dszv/x6wCCmozASWLf2BqrBe+GHuDywJvAWsn/etBHy49JcQWC7/YByU37d/Xl8+7789/3CuBwzI6z+u8NkK5f9uLv+XgVeBK4GBwIfzD/Ra+fgtgG1yvkOBJ4BjS3541ymT/v8jBZkBFAWrol+GJ4AlgFHAT2s8d4eRAw5wQP7M1xTt+0tRGYrze578C19yDn6Ty7cZ8C6wYQ3nf955KfcdUPKLX+FzBPA3YBCp1fIqsFvR53gGWAtYCvgzcHlJuS8j/ewMKNp2AbA4MDKfv+tz+VchBdkdchrrALvmc7MCKcD+otx3RcnPbtExw3KZN8/rnyP9kesg/YF8G1ipk+9r3ncEfJwUpIfnMp0L3FnLd9UqSys1q5cHpkbnzd4Dge9FxJSIeJVUIzyoaP/svH92RNxE+qu4fjfL8z6wsaQBETE5Ih4vc8wngacj4vKImBMRVwHjgb2KjvldRDwVEbOAP5B+gCuZTepfnQ1cDQwGzomI6Tn/x4FNASJibETcm/N9Hvg/YIcaPtNpEfFuLs98IuI3pJrAfaQ/CKdUSa/gDuCjkjqAjwFnAdvlfTvk/V1xRkTMioiHgYdJQRKqn/96+HFEvBkRLwL/5IPzdSBwdkQ8FxEzgJOA/Uqa0KdHxNsl3+33I+KdiLiVFJyuyuV/CbgL2BwgIp6JiNH53LwKnE318zmPpBVIgffrEfFQTvOPETEpIt6PiGtI53ZEjUkeCFwcEQ9GxLv5834k9wsXVPquWkIrBcfXgMFV+mtWJjVrCl7I2+alURJcZ5L+yndJRLxN+kt7FDBZ0o2SNqihPIUyrVK0/nIXyvNaRMzNrwu/YK8U7Z9VeL+k9ST9TdLLkt4i9dMO7iRtgFcj4p0qx/wG2Bg4N/9SVBURz5L+EA0DPkqqUUyStD7dC46VvrNq578eupJ3f1LfeMGEMumVnr9K53NFSVdLeimfzyuofj7J710E+BNwZURcXbT9S5LGSXpT0puk81pTmpR83vwH4TW6/7Pd67RScLyH1OzYp5NjJpEurBSsnrd1x9uk5mPBh4p3RsSoiNiVVIMaTwoa1cpTKNNL3SxTV/yaVK51I2Jp4GRSv15nOr11QdJSpH68i4DTJS3XhfLcAXyW1O/5Ul7/ErAs6Y6DLpenjM7O/3znU9J857MbedWS9xzmD3YLk8eP8vs3zefzi1Q/nwXnkvoV512Jl7QG6Wf2GFI3zyDgsaI0q5V1vs8raUlS664nfrZ7RMsEx4iYRupvO1/SPpKWkLSIpN0lnZUPuwo4VdIKkgbn46/oZpbjgI9JWl3SMqRmAwCShkj6VP6BeJdUK5pbJo2bgPUkHSCpv6QvABuRak6NNpDULzoj12qPLtn/Cql/rCvOAcZGxBHAjaT+MgAknS7p9k7eewfpF/HOvH476dapu4tqw6W6WsbOzv/DwIclDZO0OKlfbmHyKpf3cZLWzH9EfkjqV63X3Q8DyRdHJK0CfLuWN0n6Cql2fkBEvF+0a0lSAHw1H3coqeZY8AqwqqRFKyR9JXBo/j4XI33e+3IXTltomeAIEBFnk+5xPJV0UieQfuGuz4ecCYwhXe17FHgwb+tOXqOBa3JaY5k/oHWQrnpPIl2p2wH4apk0XgP2zMe+RrriumdETO1OmbroeNLFj+mkGsI1JftPBy7NTarPV0tM0t6ki2JH5U3fAoZLOjCvr0a66l7JHaRf8EJwvJtUk7uz4jtSbenUXMbjq5WRTs5/RDxFumDzd1LfWul9sRcBG+W8rqfrLiZdYb+TdPfCO6TgXy9nkC5+TCP9Yfpzje/bnxT0J0makZeTI+I/wM9ILbJXgE2Y//z9g9SH/bKkBX5eI+I24H+Ba0l3Q6wN7NedD9ZbtdxN4NY7SRoH7Jz/IJi1PAdHM7MyWqpZbWbWUxwczczKcHA0MyvDD8AD6j8gtOjAZhejz9tk/dWaXQQDJrz4Aq+/NrXWeyir6rf0GhFzFnjgagEx69VREbFbvfJdWA6OgBYdyGLrV72bxRrspn+e3ewiGLDHTh+pa3oxZ1ZNv1/vjDu/1qdzeoSDo5k1lgQd/Zpdii5zcDSzxlPrXd5wcDSzxlPdujB7jIOjmTWYXHM0M1uAcJ+jmdmC5Ga1mVlZblabmZXyrTxmZgsSLdmsbr26rpm1HnVUX6olIa0m6Z+SnpD0uKRv5u0/kTRe0iOSrpM0qML7n5f0aJ43Z0y1/BwczazBBP36VV+qm0Oa/3xD0rTDX5O0ETAa2DgiNiXN4X5SJ2nsFBHDImLLapk5OJpZY4m61BzzFMgP5tfTSXOorxIRtxbN1XMvsGo9iu3gaGaNJ1Vf0tTLY4qWIysnp6GkOb3vK9l1GHBzhbcFcKuksZ2lXeALMmbWYDVfrZ5aS3M3z+54LXBsRLxVtP0UUtP79xXeul1ETJK0IjBa0viIqDjBm2uOZtZ4dWhWA0hahBQYfx8Rfy7afjBpps8Do8LEWBExKf8/BbgOGNFZXg6OZtZYtTSpa7jVR5JIU+g+kadpLmzfDTgB+FREzKzw3iUlDSy8BkYCj3WWn5vVZtZ49XlCZjvgIODRPBUwwMnAL4HFSE1lgHsj4ihJKwO/jYg9gCHAdXl/f+DKiLils8wcHM2swerzhExE3J0SW8BNFY6fBOyRXz8HbNaV/BwczazxWvAJGQdHM2uswn2OLcbB0cwazANPmJmV55qjmVkZ7nM0MyvhqVnNzMqTa45mZvNLY906OJqZzU+Uv3W7l3NwNLMGEx0dvlptZrYAN6vNzMpwcDQzKyEJdbRecGy9jgAzazmSqi41pFFp9sHlJI2W9HT+f9kK7z84H/N0Hhy3Uw6OZtZw9QiOVJ598ETgtohYF7gtr5fmvxxwGrA1aQTw0yoF0QIHRzNrLIE6VHWpptLsg8DewKX5sEuBfcq8/RPA6Ih4PSLeIE3nultn+bnP0cwarsaa4WBJY4rWL4yICyukN5QPZh8cEhGTIQXQPIFWqVWACUXrE/O2ihwczayhRM3N5m7NPlhj2uUOKjsRV4Gb1WbWcHXqc6w0++ArklbK+1cCppR560RgtaL1VYFJneXl4GhmjVWnPsdKsw8CNwCFq88HA38p8/ZRwEhJy+YLMSPztoocHM2s4epUcyzMPvhxSePysgfwY2BXSU8Du+Z1JG0p6bcAEfE68H3ggbx8L2+ryH2OZtZw9XhCppPZBwF2LnP8GOCIovWLgYtrzc81xxaz6pBB3HLhN3jo2lMZ+6dT+Nr+O863/9iDdmbWQ+ex/KAlm1PAPmzu3Ll84mMjOPgL5e4k6btE9SZ1b3yCxjXHFjNn7vucePafGTd+IkstsRj/vvIEbrtvPOOfe5lVhwzi49tswIuTO20tWINcdMG5rLPeBsyY/lazi9K7qDWfrXbNscW8PPUtxo2fCMCMme8y/r8vs/IKgwA46/jPcMo51xPR6R0K1gCTXprIbbfezAFfOrTZRemV6nW1uic5OLaw1VdajmHrr8oDjz3PJ3fYhElT3uTRp15qdrH6pNNPPp5TzvgRasFxC3tCKzarG34mJc3NV5Uek/RXSYMWIq3nJQ2uckxND6G3uiUHLMpVPz2Cb//0WubMncsJh3+C7/36xmYXq0/6+y03MnjwCmw6bHizi9JrueZY3qyIGBYRGwOvA19rcH5VH0Jvdf37d3DVT7/MNTeP4S//eJi1Vl2BNVZZnvuvOYnxN57BKisO4p4rT2DI8gObXdQ+4YH77uHWW25km03X42uHH8S/7rqdrx95SLOL1WvUEhh7Y3Ds6Qsy9wCbFlYkfRv4PLAYcF1EnJa3X0+6m31x4JxKz1dWsDewY359KXA7cMLCFrw3ueC0A3nyvy/zyyv+AcDjz0xijZ1Pmrd//I1nsN2BZ/Ham283q4h9ykmnnclJp50JwL/vvoP/O/fnnHvhJc0tVC/TitMk9FiJJfUj3Yt0Q14fCaxLGj5oGLCFpI/lww+LiC2ALYFvSFq+THo3SVq5TFbzPYQOlHsIvWVtO2wtDtxza3bYaj3uvfpE7r36RD6x/UbNLpZZ51TD0sv0RM1xgKRxwFBgLGmoIEiP74wEHsrrS5GC5Z2kgLhv3r5a3v5acaIRscfCFErSkcCRACyy1MIk1aP+Pe45Bmx+TKfHbPDJ03qoNFZq2+13YNvtd2h2MXqd3thsrqbH+hyBNYBF+aDPUcCPcn/ksIhYJyIukrQjsAvwkYjYjBQ8F+9CfrU8hE5EXBgRW0bEluo/oHufzMyqky/IdCoipgHfAI7PI2uMAg7Lww8haZU8DtsywBsRMVPSBqQRf7uilofQzayHCNHRUX3pbXq0lzQiHgIeBvaLiFuBK4F7JD0K/AkYCNwC9Jf0COlB8XvLpdVJn2PZh9DNrHmk6ktv0/A+x4hYqmR9r6LX5wDnlHnb7hXSGlr0umyfY0S8RpmH0M2seXpjs7kaP1ttZg0lQb9+Cx8cJV0M7AlMyfdNI+kaYP18yCDgzXyNo/S9zwPTgbnAnFpGHHdwNLOGq1PF8RLgPOCywoaI+MIHeehnwLRO3r9TREytNTMHRzNruDqN53hnnlirXPoiPVDy8YXOKGu929bNrKVI1Hq1erCkMUXLkV3I5qPAKxHxdIX9AdwqaWyt6brmaGYNVt/ZByvYH7iqk/3bRcSkfLvgaEnjI+LOzhJ0zdHMGq6Rt/JI6g98Grim0jERMSn/PwW4jvTYcqccHM2s4Rr8hMwuwPiImFgh7yUlDSy8Jj22/Fi1RB0czayhutDnWCUdXUUa2Wt9SRMlHZ537UdJk1rSypJuyqtDgLslPQzcD9wYEbdUy899jmbWcPW4lSci9q+w/ZAy2yYBe+TXzwGbdTU/B0czazg/IWNmVio3q1uNg6OZNZTonQNLVOPgaGYN1jvHa6zGwdHMGs7NajOzUr10vMZqHBzNrKFSn2PrRUcHRzNrOAdHM7My3OdoZlbKfY5mZgtSu93KI2npzt4YEW/Vvzhm1o76tVmz+nHS6LnFn6qwHsDqDSyXmbWRFqw4Vg6OEbFaTxbEzNpTGsy29aJjTeM5StpP0sn59aqStmhsscysnfTrUNWlGkkXS5oi6bGibadLeknSuLyUnc9e0m6SnpT0jKQTaylz1eAo6TxgJ+CgvGkmcEEtiZuZQd2mSbgE2K3M9p9HxLC83FS6U1I/4Hxgd2AjYH9JG1XLrJaa47YR8RXgHYCIeB1YtIb3mZmlJ2Rq+FdNnhDr9W4UYQTwTEQ8FxHvAVcDe1d7Uy3BcbakDtJFGCQtD7zfjQKaWR/VoeoL3Z+a9RhJj+Rm97Jl9q8CTChan5i3dV7mGjI+H7gWWEHSGcDdwP+r4X1mZqDq88fkJ2imRsSWRcuFNaT+a2BtYBgwGfhZuRKU2RbVEq56E3hEXCZpLGmGL4DPRUTVmbvMzCBFpo4GXa2OiFfm5SP9BvhbmcMmAsV336wKTKqWdq2zD/YDZgPvdeE9ZmZA4+atlrRS0eq+lJ9y9QFgXUlrSlqUNFvhDdXSruVq9SmkaQ9XJkXcKyWdVEvBzcwaPDXrWZIelfQI6a6a4/Kx86ZmjYg5wDHAKOAJ4A8R8Xi1/Gp5tvqLwBYRMTNn+gNgLPCjGt5rZlaXZnWFqVkvqnDsvKlZ8/pNwAK3+XSmluD4Qslx/YHnupKJmfVtrfd8TOcDT/ycdEVnJvC4pFF5fSTpirWZWVWi/QaeKHRsPg7cWLT93sYVx8zajtpsyLKIKNuWNzPrqhaMjdX7HCWtDfyA9Ezi4oXtEbFeA8tlZm2iVZvVtdyzeAnwO9Jn3B34A+nZRDOzmig3rTtbeptaguMSETEKICKejYhTSfcTmZnVRDUsvU0tt/K8qxTWn5V0FPASsGJji2Vm7UJq3OODjVRLcDwOWAr4BqnvcRngsEYWyszaS1tOzRoR9+WX0/lgwFszs5q1YMWx05vAr6OTYX0i4tMNKZGZtRWhtmtWn9djpWiyzTdcnX/d12c+bq/1uYsfaHYRDHjh9Zn1TVBt1qyOiNt6siBm1r5acZzDViyzmbUQUZ/7HCvMPvgTSePzNAnXSRpU4b3P56HNxkkaU0u5HRzNrOH6d1RfanAJC84+OBrYOCI2BZ4COhtrdqc8Q+GWtWRWc3CUtFitx5qZFaSRvhe+5lhu9sGIuDUPZgtpUJxV61XuWkYCHyHpUeDpvL6ZpHPrVQAza38Nnn2w4DDg5gr7ArhV0tha063lJvBfAnsC1wNExMOS/PigmdWsxjt5ptba5F0wfZ0CzAF+X+GQ7SJikqQVgdGSxueaaEW1BMeOiHihpNo7t6YSm1mfJ6B/A+9zlHQwqQK3c0SUvTc7T5tAREzJ93CPADoNjrX0OU6QNAIISf0kHUvq+DQzq0kDZx/cDTgB+FRhnqsyxywpaWDhNWk2g6rTS9cSHI8GvgWsDrwCbJO3mZlVJaUnZKotNaRTbvbB84CBpKbyOEkX5GPnzT4IDAHulvQwcD9wY0TcUi2/Wp6tnkKa59XMrFv61eGmwe7OPhgRzwGbdTW/WkYC/w1lnrGOiK5eSTKzPki075Blfy96vTiwLzChMcUxs3bUgrGxpmb1NcXrki4n3ZVuZladoF8LRsdaao6l1gTWqHdBzKw9pWZ1s0vRdbX0Ob7BB32OHaTHd05sZKHMrL20XXDMc8dsRpo3BuD9SjdZmplV0htnF6ym0wvsORBeFxFz8+LAaGZdIqVbeaotvU0tRbpf0vCGl8TM2lY9bgLvaZ3NIdM/DwW0PfBlSc8Cb5P6VyMiHDDNrKp2vCBzPzAc2KeHymJmbUltdyuPACLi2R4qi5m1oTRNQrNL0XWdBccVJH2r0s6IOLsB5TGzdqP2a1b3A5Yi1yDNzLpDQL8WjI6dBcfJEfG9HiuJmbWtelyNlnQxaVDbKRGxcd62HHANMBR4Hvh8RLxR5r0HA6fm1TMj4tKqZe6sLF0quZlZBXUa7PYSFpx98ETgtohYF7iNMk/v5QB6GrA1aQTw0yQtWy2zzoLjzjUV18ysE8oDT1Rbqik3+yCwN1CoBV5K+btrPgGMjojXc61yNAsG2QVUbFZHRGkhzMy6pcZm6GBJY4rWL4yIC6u8Z0hETAaIiMl5Aq1SqzD/MIsT87ZOdWdUHjOzmnVhsNtuzz5YQxFKVX0Uuhc+0Whm7UY1LN30iqSVAPL/U8ocMxFYrWh9VWBStYQdHM2swURHR/Wlm24ADs6vDwb+UuaYUcBIScvmCzEj87ZOOTiaWUOJFGiqLVXTKT/74I+BXSU9Deya15G0paTfwrzrJ98HHsjL92q5puI+RzNruHqM51hh9kEoc2dNRIwBjihavxi4uCv5OTiaWWOpfWcfNDPrtkKzutU4OJpZw7XiNAkOjmbWcC047oSDo5k1VmpWt150dHA0s4ZrwVa1g6OZNZqQa45mZvMTtN0cMmZmC6/28Rp7FQdHM2s4B0frURMmTOCIQ7/EK6+8TEdHB4cdfiTHfOObzS5Wn/GNHYay1eqDmDZrNsf86XEA9t9iZT6xwQpMmzUHgMsemMjYCdOaWcymc7Paelz//v358Vk/Y/Phw5k+fTrbbr0FO++yKxtutFGzi9Yn3PbkVG58bArH7bTmfNv/8ugrXPfIy00qVe/UihdkWvGpHstWWmklNh8+HICBAweywQYbMmnSS00uVd/x+MszmP7unGYXoyXUaQ6ZHuWaY5t44fnnGTfuIbYasXWzi9LnffLDK7LTusvzzNS3ueieCbz93txmF6mpWrVZ3bCao6S5ksYVLUM7OXZHSX+rU74/kDRB0ox6pNcKZsyYwf6f/ww/+dkvWHrppZtdnD7t5v9M4cirH+Gb1z7OGzNnc/hHVqv+pranmv5VTUVavySmvCXp2JJjdpQ0reiY73a31I2sOc6KiGENTL+SvwLnAU83Ie8eN3v2bPb//Gf4wv4Hss++n252cfq8N2d90Mwe9cSrfHe3dZtYml6iTs3miHgSGAYgqR/wEnBdmUPviog9Fza/Hu1zlDRU0l2SHszLtmWO2UrSQ5LWkrSkpIslPZC37V0tj4i4tzAbWbuLCI768uGsv8GGfPO4bzW7OAYsO2CRea8/suayvPDGrCaWpvdowBwyOwPPRsQLdStkiUbWHAdIGpdf/zci9iVNfrNrRLwjaV3gKmDebGM5WJ4L7B0RL0r6IfCPiDhM0iDgfkl/B5YBfhsRe3S3cJKOBI4EWG311bubTFP9+1//4srfX87GG2/C1lukSvoZZ/6Q3Xbv9tdiXXD8x9dik5UHsvTi/fndAZtx5diX2GTlgay5/BJEwJQZ73L+nQ373W0ZXehz7MrUrPuR4kc5H5H0MGkSreMj4vGaC1ukp5vViwDnSRoGzAXWK9q3IXAhMDIiCjODjQQ+Jen4vL44sHpEPAEsVATIX/qFAFtssWXVaRp7o+22355Zs1uy6G3hp/94boFto5+c2oSStIDaqoY1Tc0qaVHgU8BJZXY/CKwRETMk7QFcD3Srb6Onr1YfB7wCbEZq0r9TtG8yKfhtzgfTJgr4TO5rMLMWVef7HHcHHoyIV0p3RMRbRa9vkvQrSYMjost/tXr6PsdlgMkR8T5wENCvaN+bwCeBH0raMW8bBXxdeRhhSZv3YFnNrE46VH3pgv2p0KSW9KGieDGCFONe61aZu/OmhfAr4GBJ95Ka1G8X78x/CfYCzpe0NWk6xUWARyQ9lteRtLKkm8plIOksSROBJfL0jac37NOYWW3qdEVG0hKkKVj/XLTtKElH5dXPAo/lPsdfAvtFRLf6nhrWrI6IpcpsexrYtGjTSXn77cDt+fWLwIeLjvlKmXQmUaHPMSK+A3ynm8U2szpLsa8+zeqImAksX7LtgqLX55Fu5VtofkLGzBqr683mXsHB0cwaz8HRzKyUp0kwM1uAcLPazKw8B0czswW5WW1mVkYLDufo4GhmDdZLR/quxsHRzBrOzWozsxLCNUczs7IcHM3MynCz2sysDNcczczKqFdwlPQ8MJ00k8Cc0pHD81iO55BG7ZoJHBIRD3YnLwdHM2uoeg5Zlu3Uycjeu5OmRVgX2Br4df6/y3p6sFsz62vyfY7VljrZG7gsknuBQZJW6k5CDo5m1nA1DgQ+WNKYouXIMkkFcKuksRX2rwJMKFqfmLd1mZvVZtZgQrVVDWuZfXC7iJgkaUVgtKTxEXHnfJktqFvTJLjmaGYNV69mdWHa5oiYAlwHjCg5ZCKwWtH6qnwwm2mXODiaWUPV0qSuJTZKWlLSwMJr0rz2j5UcdgPwJSXbANMiYnJ3yu1mtZk1XI3N6mqGANfltPoDV0bELYWZB/NEWzeRbuN5hnQrz6HdzczB0cwarh6xMSKeAzYrs7149sEAvrbwuTk4mlkPaMEHZBwczazBVLdmdY9ycDSzhvKQZWZmFbRgbHRwNLPGc83RzKwM9zmamZXReqHRwdHMGqzOo+70GAdHM2s4N6vNzMpovdDo4GhmPaAFK44OjmbWWEJ0tGB09JBlZmZlODiaWcPVY7BbSatJ+qekJyQ9LumbZY7ZUdI0SePy8t3ultnNajNruDrNPjgH+J+IeDAPejtW0uiI+E/JcXdFxJ4Lm5mDo5k1lAQd9RnPcTIwOb+eLukJ0uRZpcGxLtysNrPGq22ehFpmH0zJSUOBzYH7yuz+iKSHJd0s6cPdLbJrjmbWcDU2q2uZfRBJSwHXAsdGxFslux8E1oiIGZL2AK4H1u1qecE1RzPrAR2qvtRC0iKkwPj7iPhz6f6IeCsiZuTXNwGLSBrcrTJ3501mZl1Sh+kHlZ5BvAh4IiLOrnDMh/JxSBpBinGvdafIblabWcPV6Wr1dsBBwKOSxuVtJwOrw7yJtj4LHC1pDjAL2C9PutVl6ub72oqkV4EXml2OhTQYmNrsQlhbnIc1ImKFeiUm6RbS91LN1IjYrV75LiwHxzYhaUwtndnWWD4P7cN9jmZmZTg4mpmV4eDYPi5sdgEM8HloG+5zNDMrwzVHM7MyHBzNzMpwcOxjCk/fR2EbAAAJBklEQVQPWHNI6tfsMlht3OfY5iRtBfQD3omIcXmbuvvUgHWPpHUi4pn8ul9EzG12maxzrjm2MUm7A1cCRwDfl/RLgIgI1yB7jqS9gKckXQYQEXNdg+z9HBzbkJJFgCOB70bEEcBhwHaSfgcOkD1F0grAF4EvAetLuhQcIFuBg2MbimQ2MBaYm7e9CmwNbCLp7MJxzStl35C/93Mi4grgo8Dw4gDZ1MJZpxwc29tLwEmSlgeIiDnA3sBQSes1tWR9QKFmHhH/ltQREe8Bw0kB8rJ8zLYLM1q1NY6DYxuLiN8BtwH3FQb8jIiXgNnAgGaWrS8orplHxPuS+uca/RakJvbjwG+A6c0qo1Xm8RzbVK6pvB8Rx0sK4BZJ5wErApsCbza3hH1PRMzJAfK93Pd7JrBjRLzY7LLZglxzbCPFF1hyTaVffv1t4OfAssBmwOciotXHr+y1JFX8vcoBcl1gV2DniHis50pmXeH7HFucpCER8UrR+jIRMa1ovSMi3q+0bo0h6UPAehFxZ4X9850n631cc2xR+XadwcD9kg7I2/oBv5S0XeG4MoHQfw3rrMItUZsBa1Y61oGx93PNscXlG4x/BpwSEX+UtFRh9jVrvOKnjSTtBiwOPFZ4GsZal2uOLahodjVFxF+BY4GzJO1XHBglDfeN3o1VFBi/CnwXWA94SNK8qRIk7S1phyYV0brJwbHFlDwXvYKkJfP8vEcDP5L0hXzcEsDxpKvT1iC5e2M9YCTpIsvrwP2kyeUL3gUmNaF4thDcrG5Rko4HtgGWAX4QEbdL2gX4VV6/1AMc1F/poB35yvQA4BhgKLAWsGdEzJb0deCqiGj12Qj7JNccW5CkrwC7R8RnSX1c50j6bET8ndTEPk7S0oCvStffsoUXkg4EfggsRroAsw2wTw6Mnyc9z75EU0ppC803gbeAQm2lqNayOHCEpP8hNeOuBM6WNCAiLpd0e0TMbGqh25CkVYDfSvpFRIwC5gD/jYjXJZ0MXEH6QzUA+DBwkG/wbl1uVvdyJVdD590bl0d7uQLYKz9xcQfwInC0r1Y3hqRlgf2A3YGfAEOAD0XEeXn/csDmwJLAIxHxfJOKanXgmmMvVxQYjwa2kvQC8A/gAWAGcKykCcDLwKkOjPVX+AMVEW9Iupo00tHxpHtGX5P0UVK/42LAmIiY3MTiWp04OLaA3Ld1AGlcwNtIfcX3AJcA+wKfAw7xI4H1V1Jz7w9Mi4gL8y1Sx5Bqio8CW5H6I7/WtMJaXblZ3cvlQWtPIAXFdYCDSFdD35O0QkS8KmnZiHijqQVtc5KOI/UjLg8cHxHPSjqU9Mfp9Ih4sNMErOW45tjLSNoe2AT4L3BPREzLTelfA1MjYpd83PGk5t3PHRjrL4+eMye//jqwJ6mG/jfgLkm7RMTvcj/kCZIOB2b6ufX24eDYi0j6BPBT4GFge2CEpDOBp0jBcpSklYCPAQfmxeosP5I5QtLpwCLAQNKFmMNI5+Fm4G5J20bE2ZIGua+3/bhZ3UtI+jhwPbB+REzOv6B7RcSRef9ewI6k++lmA9+JiEebVd52JWlX0pXoE/LtOkhajHRz94XArhHxjqTxpJr7sDyArbUZB8deQtKmwEOke+OuzNvuBv4CPEHqc3wHWBRYxDWV+st/oG4AhkfEU5KGAltHxDWSVgZ+AFwArAusDfzO9zG2Lzere4mIeETS1sDoXFNZmXT1cziwLXA+qcn9q4h4t3klbWtTSbfkDJX0DOnm+svzvunAa6QZHXcg1eodGNuYa469jKStgFuB1yNi7aLtewLjImJi0wrXBxR9/3OBYyLi6sIAwXkwj/7A4hExpakFtYbzs9W9TEQ8QKqZLCfpoKLtf3NgbLz8/X8MKJ5TWnkQj5kR8ZYDY9/gZnUvlJvYu5JG+e6fZxG0HhIRj0oaCdyar0Rf0OwyWc9zcOylImKMpC0ADyDRBBHxQB4C7gFJ7/oPVN/jPkezTkjanHRz95PNLov1LAdHM7MyfEHGzKwMB0czszIcHM3MynBwNDMrw8HRzKwMB8c+TNJcSeMkPSbpj/nxuO6mtaOkv+XXn5J0YifHDpL01W7kcXoex7Km7SXHXCLps13Ia6ikx7paRmsfDo5926yIGBYRGwPvAUcV78wT1nf5ZyQiboiIH3dyyCCgy8HRrCc5OFrBXcA6ucb0hKRfAQ8Cq0kaKekeSQ/mGuZSAJJ2kzQ+D6326UJCkg6RVJiRb4ik6yQ9nJdtgR8Da+da60/ycd+W9ICkRySdUZTWKZKelPR3YP1qH0LSl3M6D0u6tqQ2vIukuyQ9lQfyQFI/ST8pyvsrC/tFWntwcLTCxFG7kyaKghSELouIzYG3gVOBXSJiODAG+JakxYHfAHsBHwU+VCH5XwJ3RMRmpOHXHgdOBJ7NtdZv5+eY1wVGAMOALSR9LD8+uR9pEqtPkyaxqubPEbFVzu8J4PCifUNJg3p8Erggf4bDSZNmbZXT/7KkNWvIx9qcn63u2wZIGpdf3wVcRBpH8oWIuDdv3wbYCPhXmnCPRUkzH25AmtD+aQBJV5DGOiz1ceBLABExF5iW510pNjIvD+X1pUjBciBwXUTMzHncUMNn2jhPLTEopzOqaN8f8hwvT0t6Ln+GkcCmRf2Ry+S8n6ohL2tjDo5926yIGFa8IQfAt4s3AaMjYv+S44aR5m2uBwE/ioj/K8nj2G7kcQmwT0Q8LOkQ0tQSBaVpRc7764UpEYryHtrFfK3NuFlt1dwLbCdpHQBJS0haDxgPrCmpMCDv/hXefxtwdH5vP0lLk0bVHlh0zCjgsKK+zFUkrQjcCewraYCkgaQmfDUDgclKU9qWTkD2OUkducxrAU/mvI/OxyNpPUlL1pCPtTnXHK1TeV7sQ4Cr8vQNAKfmOVaOBG6UNBW4G9i4TBLfBC5Umrp0LnB0RNwj6V/5Vpmbc7/jhsA9ueY6A/hiRDwo6RpgHPACqelfzf8C9+XjH2X+IPwkcAcwBDgqT5T1W1Jf5INKmb8K7FPbt2PtzKPymJmV4Wa1mVkZDo5mZmU4OJqZleHgaGZWhoOjmVkZDo5mZmU4OJqZlfH/ATLoA60ENJ0WAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix3, classes = ['Real: 0', 'Fake: 1'],\n",
    "                      title='Confusion matrix, without normalization')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "       Real       0.92      0.86      0.89        28\n",
      "       Fake       0.79      0.88      0.83        17\n",
      "\n",
      "avg / total       0.87      0.87      0.87        45\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_true=y_news2_test, y_pred=y_predicted_text_domain3, target_names=['Real', 'Fake']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lưu lại model1: ```MultiNB_clf_textdomain_2```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./Models/MultiNB_clf_textdomain_2.sav']"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1_files = 'MultiNB_clf_textdomain_2.sav'\n",
    "joblib.dump(MultiNB_text_domain_clf, os.path.join(models_path, model1_files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. SVM classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1. Mô hình cho dữ liệu từ file ```preproc_vn_news_226_tlfr.csv``` - text & label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mô hình 1: \n",
    "Thực hiện tokenize bằng mô hình của nhóm -> Vectorize bằng TFIDF -> Thực hiện classify bằng SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm1 = Pipeline([\n",
    "    ('tfidfvect', TfidfVectorizer(ngram_range=(1, 2))),\n",
    "    ('svm', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                           alpha=1e-3, random_state=42,\n",
    "                           max_iter=5, tol=None))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tfidfvect', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=Tru...ty='l2', power_t=0.5, random_state=42, shuffle=True,\n",
       "       tol=None, verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm1.fit(data1_train_text, data1_train_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_predict1 = svm1.predict(data1_test_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. __Accuracy__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8666666666666667\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(svm_predict1 == data1_test_label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lưu lại model1: ```SVM_clf_1```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./Models/SVM_clf_1.sav']"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1_files = 'SVM_clf_1.sav'\n",
    "joblib.dump(svm1, os.path.join(models_path, model1_files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2. Mô hình cho dữ liệu từ file ```preproc_vn_news_223_tdlfr.csv``` - text, domain & label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm2 = Pipeline([\n",
    "    ('tfidfvect', TfidfVectorizer(ngram_range=(1, 2))),\n",
    "    ('svm', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                           alpha=1e-3, random_state=42,\n",
    "                           max_iter=5, tol=None))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('tfidfvect', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=Tru...ty='l2', power_t=0.5, random_state=42, shuffle=True,\n",
       "       tol=None, verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm2.fit(X_news2_train['text_domain'], y_news2_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_predict2 = svm2.predict(X_news2_test['text_domain'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. __Accuracy__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9111111111111111\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(svm_predict2 == y_news2_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. __Confussion Matrix__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnf_matrix4 = confusion_matrix(y_pred=svm_predict2, y_true=y_news2_test)\n",
    "np.set_printoptions(precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[26  2]\n",
      " [ 2 15]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAEYCAYAAADFzZobAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XmcXFWZ//HPt9MsgYQ1EAMSwxYIomSB4AQERoEBBwZQQRZB2WEGRBAVERUGF34ICArKgMoqiw6IiCirEEBAAoRtwk7YEkLCmkACWZ7fH+c0VtrurupOVddyv++87itV994696m6VU+fc+695yoiMDMrkrZ6B2Bm1t+c+MyscJz4zKxwnPjMrHCc+MyscJz4zKxwnPgySQMl/VHSW5J+twTl7CPpxmrGVi+SPinpiUbZnqQRkkJSe3/F1CwkTZW0bX58vKRf1mAb50r6TrXLrQc123l8kvYGjgE2BGYDk4EfRMSdS1juvsCRwISIWLDEgTY4SQGsHxFP1zuW7kiaChwUETfn5yOA54Clqr2PJF0IvBQRJ1Sz3P7S+bOqQnlfzuVtWY3yGk1T1fgkHQOcCfwQGAoMB34O7FKF4j8CPFmEpFcJ16pqx59tA4iIppiAFYE5wO49rLMMKTFOy9OZwDJ52TbAS8DXgFeB6cD+edlJwPvA/LyNA4ETgUtLyh4BBNCen38ZeJZU63wO2Kdk/p0lr5sA3Ae8lf+fULLsNuBk4K5czo3AkG7eW0f83yiJf1fgM8CTwOvA8SXrjwfuBt7M654NLJ2XTczv5Z38fr9QUv43gVeASzrm5desm7cxNj9fA5gFbFPBvrsI+Fp+vGbe9n/m5+vlctVpe5cAi4C5OcZvlOyDLwEv5O1/u8L9v9h+yfMib/+QvO/fz9v6YzfvI4DDgKeAN4Bz+EerqQ04AXg+75+LgRU7fXcOzHFPLJm3P/BiLu8wYDPg4bzfzi7Z9rrArcBr+X3/BlipZPlUYNv8+ETydzfv9zkl0wLgxLzsOOAZ0nfv/4Dd8vxRwDxgYX7Nm3n+hcD3S7Z5MPB03n/XAmtU8lk1wlT3ACoOFHbIO629h3X+G7gHWB1YDfgbcHJJ4liQ11mKlDDeBVbu/GXp5nnHF7UdWB54G9ggLxsGfLTzDwxYJe/0ffPr9srPV83Lb8tfvJHAwPz8lG7eW0f8383xHwzMBC4DBgMfzV/WdfL644BP5O2OAKYAX+38o++i/P9HSiADKUlEJV/0KcBywA3AaRXuuwPIyQTYO7/nK0uW/aEkhtLtTSX/mDvtg/NzfJsA7wGjKtj/H+yXrj4DOv2ou3kfAVwHrERqbcwEdih5H08D6wCDgKuBSzrFfTHpuzOwZN65wLLA9nn/XZPjX5OUQLfOZawHbJf3zWqk5HlmV58Vnb67JeuMzjGPyc93J/0BayP98XsHGNbD5/XBZwR8ipSAx+aYfgZMrOSzaoSpmZq6qwKzouem6D7Af0fEqxExk1ST27dk+fy8fH5EXE/6a7ZBH+NZBGwsaWBETI+Ix7pY59+BpyLikohYEBGXA48DO5esc0FEPBkRc4Hfkr6c3ZlP6s+cD1wBDAHOiojZefuPAR8HiIj7I+KevN2pwP8AW1fwnr4XEe/leBYTEeeT/oLfS0r23y5TXofbgU9KagO2Ak4FtsjLts7Le+OkiJgbEQ8BD5ESIJTf/9VwSkS8GREvAH/lH/trH+CMiHg2IuYA3wL27NSsPTEi3un02Z4cEfMi4kZS4rk8x/8ycAcwBiAino6Im/K+mQmcQfn9+QFJq5GS6pER8WAu83cRMS0iFkXElaR9O77CIvcBfh0RD0TEe/n9/kvuh+3Q3WdVd82U+F4DhpTpH1mD1NTo8Hye90EZnRLnu6S/zr0SEe+Q/kIeBkyX9CdJG1YQT0dMa5Y8f6UX8bwWEQvz444fz4yS5XM7Xi9ppKTrJL0i6W1Sv+iQHsoGmBkR88qscz6wMfCz/IUvKyKeIf2RGQ18klQTmCZpA/qW+Lr7zMrt/2rozbbbSX3RHV7sorzO+6+7/bm6pCskvZz356WU35/k1y4F/C9wWURcUTJ/P0mTJb0p6U3Sfq2oTDq935zsX6Pv3+1+1UyJ725SU2DXHtaZRjpI0WF4ntcX75CadB0+VLowIm6IiO1INZ/HSQmhXDwdMb3cx5h64xekuNaPiBWA40n9aD3p8RC/pEGkfrNfASdKWqUX8dwOfJ7Uz/hyfr4fsDLpyHyv4+lCT/t/sf0pabH92YdtVbLtBSyeyJZkGz/Kr/943p9fpPz+7PAzUj/eB0esJX2E9J09gtT1shLwaEmZ5WJd7P1KWp7UKuuP7/YSa5rEFxFvkfq3zpG0q6TlJC0laUdJp+bVLgdOkLSapCF5/Uv7uMnJwFaShktakVSVB0DSUEn/kXf2e6TazMIuyrgeGClpb0ntkr4AbESq8dTaYFI/5JxcGz280/IZpP6o3jgLuD8iDgL+ROqfAkDSiZJu6+G1t5N+ZBPz89tIpw/dWVKL7ay3Mfa0/x8CPipptKRlSf1gS7KtrrZ9tKS18x+IH5L6Mat1lsBg8oEGSWsCX6/kRZIOJdWq946IRSWLliclt5l5vf1JNb4OM4APS1q6m6IvA/bPn+cypPd7b+5WaXhNk/gAIuIM0jl8J5B22IukH9M1eZXvA5NIR8UeAR7I8/qyrZuAK3NZ97N4smojHR2eRjqitTXwn12U8RqwU173NdKRyZ0iYlZfYuqlY0kHEmaT/rJf2Wn5icBFuZmzR7nCJO1COsB0WJ51DDBW0j75+Vqko9PduZ304+1IfHeSamATu31FquWckGM8tlyM9LD/I+JJ0sGPm0l9WZ3P+/wVsFHe1jX03q9JR6Inko7yzyMl9mo5iXQg4S3SH52rK3zdXqSEPk3SnDwdHxH/B5xOaknNAD7G4vvvVlKf8SuS/un7GhG3AN8BriKdNbAusGdf3lg9NN0JzNaYJE0GPp2TvVlDc+Izs8JpqqaumVk1OPGZWeE48ZlZ4fhiaUDtA0NLD653GIU3ZtTweodgwPPPT2XWrFmVniNY1oAVPhKx4J8uBPonMXfmDRGxQ7W22xMnPkBLD2aZDcqe0WE1dte9Z9c7BAO22HzTqpYXC+ZW9PuaN/mcSq8aWWJOfGZWWxK0Dah3FItx4jOz2lNjHU5w4jOz2lPVugyrorHSsJm1IKUaX7mpXCnSWpL+KmmKpMckHZXnn5hHrZmcp8+UK8s1PjOrLVGtPr4FpJG8H5A0GLhf0k152U8i4rRKC3LiM7MaU1WauhExnTQgAhExW9IUFh//r2Ju6ppZ7VWhqbtYcWmk5zGk0cABjpD0sKRfS1q53Oud+MysxvLpLOWmNML6pJLpkC5LS+MdXkW6h8zbpEF31yWN8D2dNNxWj9zUNbPaEpU2dWdFRI9nT+dh9K8CfhMRVwNExIyS5edTwUC/rvGZWe1V56iuSAPGTsmDEnfMH1ay2m6kIfR75BqfmdWYYEBVjupuQbpr3iN54FtI95LZS9Jo0lD6U4FDyxXkxGdmtSWqcuVGRNxJ1zdYur63ZTnxmVntNdiVG058ZlZjHqTAzIrIgxSYWaGoOlduVJMTn5nVnmt8ZlYs7uMzsyJyU9fMCqVK5/FVkxOfmdWYm7pmVkSu8ZlZ4biPz8wKxbeXNLMikmt8ZlYkaRxSJz4zKxLR9WBSdeTEZ2Y1JtrafFTXzArGTV0zKxwnPjMrFEmozYnPzArGNT4zKxwnPjMrFuGmrpkVj2t8ZlYoQk58ZlY8TnxmVizu4zOzInKNz8wKx4nPlsiHh67EL0/ej6GrrsCiCH591V2cc/ltABy+59Yc9oWtWLBwEX+541G+fdYf6htsgbz44osctP9+zJjxCm1tbRxw4CEc8ZWj6h1WQxC+csOW0IKFizjujKuZ/PhLDFpuGf522Te55d7HWX2Vwey0zcfYbI8f8f78Bay28qB6h1oo7e3tnHLq6YwZO5bZs2czYfNxfHrb7Ri10Ub1Dq3+5BqfLaFXZr3NK7PeBmDOu+/x+HOvsMZqK3HAZydw2gU38f78BQDMfGNOPcMsnGHDhjFs2DAABg8ezIYbjmLatJed+LJGS3yNNUiW9crwYasweoMPc9+jU1nvI6uzxZh1mXjxsdz4y6MYt9HweodXWM9PncrkyQ+y2fjN6x1Kw1Cbyk79qeaJT9JCSZMlPSrpj5JWWoKypkoaUmadVSTdJOmp/P/Kfd1eI1t+4NJcftpBfP20q5j9zjzaB7Sx8grLsdV+p3H8T67h0lMPqHeIhTRnzhz22uNz/Pj0M1lhhRXqHU7DkFR2qqCMtST9VdIUSY9JOirP7/Vvvj9qfHMjYnREbAy8DvxXjbd3HHBLRKwP3JKft5T29jYuP+1grvzzJP5w60MAvDzjTa65JT2e9NjzLFoUDHE/X7+aP38+e+3xOb6w1z7suttn6x1Ow6gk6VXYFF4AfC0iRgGfAP5L0kb04Tff303du4E1O55I+rqk+yQ9LOmkkvnXSLo/Z/VDermNXYCL8uOLgF2XOOoGc+739uGJ517hp5fe+sG8P972MNuMHwnAesNXZ+ml2pnlfr5+ExEcdvCBbLDhKI46+ph6h9Nw2trayk7lRMT0iHggP54NTCHlk17/5vst8UkaAHwauDY/3x5YHxgPjAbGSdoqr35ARIwDNgW+ImnVLsq7XtIaXWxqaERMh/RBAatX/c3U0YTR67DPTpuz9WYjueeK47jniuP4ty034qJr7mbtNVdl0u+O5+JT9ueg715S71AL5W933cVlv7mE2/96K5uPG83m40bzlz9fX++wGocqmHpTnDQCGAPcSx9+8/1xVHegpMnACOB+4KY8f/s8PZifDyIlwomkZLdbnr9Wnv9aaaER8ZklCSrXJFNtcqnmaRL+bfKzDBxzRJfLDjjh4n6OxjpsseWWzJ0f9Q6jYVXYlB0iaVLJ8/Mi4rwuyhoEXAV8NSLe7ssR4/5IfHMjYrSkFYHrSH18PyXl+B9FxP+UrixpG2Bb4F8i4l1JtwHL9mJ7MyQNi4jpkoYBr3a1Uv5AzwNoW251f2PNaqXy8/hmRcSmPRYlLUVKer+JiKvz7Ip+86X6rakbEW8BXwGOzcHfAByQszeS1pS0OrAi8EZOehuSOjF741rgS/nxlwBfvmBWR0K0tZWfypaTsuevgCkRcUbJol7/5vv14EZEPAg8BOwZETcClwF3S3oE+F9gMPAXoF3Sw8DJwD1dldVDH98pwHaSngK2y8/NrI6k8lMFtgD2BT6VT5GbLOkz9OE3X/OmbkQM6vR855LHZwFndfGyHbspa0TJ4y77+CLiNdJBFDNrENW4ciMi7qT7wyC9+s37kjUzqykJBgxorEvWnPjMrOYa7FJdJz4zq71GG6TAic/MakqioqO2/cmJz8xqzHdZM7MCarC858RnZrXnGp+ZFYr7+MyskBqswufEZ2a156aumRWLm7pmVjTCTV0zKxyfx2dmBeSmrpkVS+Xj7fUbJz4zq6nUx9dYmc+Jz8xqzonPzArHfXxmVizu4zOzolEznc4iaYWeXhgRb1c/HDNrRQOaqKn7GBAsflejjucBDK9hXGbWQhqswtd94ouItfozEDNrTem+uY2V+Sq6obikPSUdnx9/WNK42oZlZq1kQJvKTv2pbOKTdDbwr6Q7mAO8C5xby6DMrLVI5af+VMlR3QkRMVbSgwAR8bqkpWscl5m1CJGO7DaSShLffEltpAMaSFoVWFTTqMyspTTYQd2KEt85wFXAapJOAvYATqppVGbWOqTmu3IjIi6WdD+wbZ61e0Q8WtuwzKxVCGhrsKO6lV65MQCYT2ruVnQk2MysQ4PlvYqO6n4buBxYA/gwcJmkb9U6MDNrDR23lyw39adKanxfBMZFxLsAkn4A3A/8qJaBmVnraLSmbiXN1udZPEG2A8/WJhwza0WqYCpbhvRrSa9KerRk3omSXpY0OU+fqSSengYp+AmpT+9d4DFJN+Tn2wN3VlK4mZmo2iAFFwJnAxd3mv+TiDitNwX11NTtyKqPAX8qmX9PbzZgZgWn6gxLFRETJY1Y4oLoeZCCX1VjA2ZmFea9IZImlTw/LyLOq+B1R0jaD5gEfC0i3ij3grIHNyStC/wA2AhYtmN+RIysICAzK7heNHVnRcSmvSz+F8DJpG64k4HTgQPKvaiSgxsXAheQ4t8R+C1wRS+DM7MCU27u9jT1RUTMiIiFEbEIOB8YX8nrKkl8y0XEDXkjz0TECaTRWszMKlKNo7pdlisNK3m6G/84NtGjSs7je08pHT8j6TDgZWD13odoZkUkVec8PkmXA9uQ+gJfAr4HbCNpNKmpOxU4tJKyKkl8RwODgK+Q+vpWpII2tJlZh2pcmRERe3Uxu08HYSsZpODe/HA2/xiM1MysYg124UaPJzD/njwGX1ci4rM1icjMWopQw12y1lON7+x+i6LOxowazl33FubtNqxDrnyo3iEYMPWNudUtUNVp6lZTTycw39KfgZhZ62q0sewqHY/PzKxPROPdXtKJz8xqrr3BqnwVJz5Jy0TEe7UMxsxaT1PeUFzSeEmPAE/l55tI+lnNIzOzltGm8lO/xlPBOj8FdgJeA4iIh/Ala2bWC814Q/G2iHi+U1V1YY3iMbMWI6C9wZq6lSS+FyWNB0LSAOBI4MnahmVmraTB8l5Fie9wUnN3ODADuDnPMzMrS2quKzcAiIhXgT37IRYza1EDmu10Fknn08U1uxFxSE0iMrOWIhrv9pKVNHVvLnm8LGmwvxdrE46ZtaIGy3sVNXWvLH0u6RLgpppFZGatRTCgwTJfXy5ZWxv4SLUDMbPWlJq69Y5icZX08b3BP/r42oDXgeNqGZSZtZamSnz5XhubkO6zAbAoIrodnNTMrCtNda1uTnK/z7dvW+ikZ2a9JaXTWcpN/amSzf1d0tiaR2JmLastn8Tc09SferrnRntELAC2BA6W9AzwDqmvMiLCydDMymq2gxt/B8YCu/ZTLGbWktRUp7MIICKe6adYzKwFpaHn6x3F4npKfKtJOqa7hRFxRg3iMbNWU4eBRsvpKfENAAaRa35mZn0hYECDZb6eEt/0iPjvfovEzFpWMw1S0FiRmlnTarC812Pi+3S/RWFmLUvNNEhBRLzen4GYWetqrLTnG4qbWY0160CkZmZLpLHSXmXX6pqZLQHR1lZ+KluK9GtJr0p6tGTeKpJukvRU/n/lSiJy4jOzmhIp0ZSbKnAhsEOneccBt0TE+sAtVDhWqBOfmdWcpLJTORExkTQQcqldgIvy44uocGwB9/GZWW2p4oMbQyRNKnl+XkScV+Y1QyNiOkBETJe0eiUbcuIzs5rqaOpWYFZEbFrTYDI3dc2s5qrR1O3GDEnD8jaGAa9W8iInPjOruTaVn/roWuBL+fGXgD9U8iI3dc2splJTd8nP5JN0ObANqS/wJeB7wCnAbyUdCLwA7F5JWU58ZlZz1bhwIyL26mZRr8cVcOIzsxoTarBrN5z4zKymRBONzmJmVhVqrvH4zMyqwonPqubFF1/koP33Y8aMV2hra+OAAw/hiK8cVe+wCuOgT6zF6DUH8/a8BRz/pycB2O1jQ9l6vVWZPW8BAL97aDoPT5tdzzDrzk1dq6r29nZOOfV0xowdy+zZs5mw+Tg+ve12jNpoo3qHVgh3PPs6Nz0xi0MnrLXY/Bsen8mfp8ysU1SNqdEObvgE5iY2bNgwxowdC8DgwYPZcMNRTJv2cp2jKo4nXn2Hd95fUO8wmoJUfupPrvG1iOenTmXy5AfZbPzm9Q6l8LYdOYQt1l6Zqa/P5bIHpvHu+wvrHVJdNWJTt2Y1PkkLJU0umUb0sO42kq6r0nZ/IOlFSXOqUV4zmDNnDnvt8Tl+fPqZrLDCCvUOp9Bueeo1jr12Ct+5/knenDufvceuUe+QGoAq+tefatnUnRsRo0umqTXcVqk/AuP7aVt1N3/+fPba43N8Ya992HW3z9Y7nMJ7e94CIiCA255+jXVWHVjvkOqvgmZuf1cI+7WPT9IISXdIeiBPE7pYZzNJD0paR9Lyebjp+/K8XcptIyLu6Rifq9VFBIcdfCAbbDiKo44+pt7hGLDisv/oPRq31oq89Oa8OkbTOFTB1J9q2cc3UNLk/Pi5iNiNNGTMdhExT9L6wOXAB+Nv5UT4M2CXiHhB0g+BWyPiAEkrAX+XdDOwIvDLiPhMX4OTdAhwCMBaw4f3tZi6+ttdd3HZby5h440/xubjRgNw0vd/yA479vljsV44fIvhjBo6iEHLtHPmbqO4+uEZjBq6PMNXHkgEzHrnfS6496V6h1l3jdjHV8vENzciRneatxRwtqTRwEJgZMmyUcB5wPYRMS3P2x74D0nH5ufLAsMjYgqwRL/uPLLreQDjxm0aS1JWvWyx5ZbMnd+UobeEX9z1wj/Nm/iMb0fdpcbKe/1+VPdoYAawCamZXdoOmE5KbGOAjsQn4HMR8UR/Bmlm1VX08/hWBKZHxCJgX2BAybI3gX8HfihpmzzvBuBI5eFZJY3px1jNrEpqOBBp3+Lp383xc+BLku4hNXPfKV0YETOAnYFzJG0OnExqHj+c76V5MoCkNSRd39UGJJ2aBylcTtJLkk6s2bsxs8o02NGNmjV1I2JQF/OeAj5eMutbef5twG358QvAR0vWObSLcqbRTR9fRHwD+EYfwzazKkt5rbGaur5yw8xqqw5N2XKc+Mys9pz4zKxYPPS8mRWMcFPXzIrIic/MisZNXTMrnAa7VNeJz8xqzHdZM7MiclPXzApFuMZnZgXkxGdmheOmrpkVjmt8ZlY4TnxmVigelsrMiqeK5/FJmgrMJt2zZ0FEbNrzK7rmxGdmNVfl+t6/RsSsJSnAic/MakyowTr5+vueG2ZWQFL5CRgiaVLJdEgXRQVwo6T7u1leEdf4zKymenEvoVkV9NltERHTJK0O3CTp8YiY2NuYXOMzs5qTVHaqRL7RGBHxKvB7YHxf4nHiM7Oaq7CpW6YMLS9pcMdjYHvg0b7E46aumdVclQ5tDAV+n2uH7cBlEfGXvhTkxGdmtSWqclQ3Ip4FNlnygJz4zKzGPCyVmRVSg+U9Jz4zqz3X+MyscBrtyg0nPjOrucZKe058ZlZjlZ6n15+c+Mys5tzUNbPCaay058RnZv2gwSp8TnxmVltCtDVY5vMgBWZWOK7xmVnNNViFz4nPzGrPd1kzs0KRoK2x8p4Tn5n1Ayc+MysaN3XNrHDc1DWz4nHiM7OiabSmriKi3jHUnaSZwPP1jmMJDQFm1TsIa4n98JGIWK1ahUn6C+lzKWdWROxQre32xImvRUiaVMHNmK3GvB+agy9ZM7PCceIzs8Jx4msd59U7AAO8H5qC+/jMrHBc4zOzwnHiM7PCceIrGDXaXV8KRtKAesdg7uNreZI2AwYA8yJicp6n8I7vV5LWi4in8+MBEbGw3jEVmWt8LUzSjsBlwEHAyZJ+ChAR4Zpf/5G0M/CkpIsBImKha3715cTXgpQsBRwCfDciDgIOALaQdAE4+fUXSasBXwT2AzaQdBE4+dWbE18LimQ+cD+wMM+bCWwOfEzSGR3r1S/KYsif+1kRcSnwSWBsafKra3AF5sTX2l4GviVpVYCIWADsAoyQNLKukRVAR406Iv4mqS0i3gfGkpLfxXmdCZI+Ws84i8iJr4VFxAXALcC9kobkeS8D84GB9YytCEpr1BGxSFJ7romPIzV7HwPOB2bXK8ai8nh8LSrXMBZFxLGSAviLpLOB1YGPA2/WN8LiiYgFOfm9n/tavw9sExEv1Du2onGNr4WUHqzINYwB+fHXgZ8AKwObALtHRLOPP9iwJHX7u8rJb31gO+DTEfFo/0VmHXweX5OTNDQiZpQ8XzEi3ip53hYRi7p7brUh6UPAyIiY2M3yxfaT9S/X+JpUPmVlCPB3SXvneQOAn0raomO9LpKc/9JVWTenBW0CrN3duk569eUaX5PLJ8eeDnw7In4naVBEzKl3XEVRehWMpB2AZYFHO67SsMbkGl8T6qg15B/dH4GvAqdK2rM06Uka65OUa6sk6f0n8F1gJPCgpA+Gn5e0i6St6xSidcGJr8l0us52NUnLR8T1wOHAjyR9Ia+3HHAs6Siu1UjuchgJbE86YPE68HfggZLV3gOm1SE864abuk1K0rHAJ4AVgR9ExG2StgV+np9f5Ivhq6/zAA/5CO5A4AhgBLAOsFNEzJd0JHB5RDT7Xddajmt8TUjSocCOEfF5Up/SWZI+HxE3k5q9R0taAfDR2+pbueOBpH2AHwLLkA5mfALYNSe9PUjXRy9XlyitRz6BuQl01DJKahvLAgdJ+hqpaXUZcIakgRFxiaTbIuLdugbdgiStCfxS0pkRcQOwAHguIl6XdDxwKemP0EDgo8C+Pjm5Mbmp2+A6HTX84NyvPOrHpcDO+UqA24EXgMN9VLc2JK0M7AnsCPwYGAp8KCLOzstXAcYAywMPR8TUOoVqZbjG1+BKkt7hwGaSngduBe4D5gBflfQi8ApwgpNe9XX88YmINyRdQRrx5ljSOZGvSfokqZ9vGWBSREyvY7hWASe+JpD7kvYmjet2C6lv9m7gQmA3YHfgy74Mrfo61bjbgbci4rx8mtARpBreI8BmpP6//6pbsFYxN3UbXB5Q9JukhLcesC/pqOH7klaLiJmSVo6IN+oaaIuTdDSp325V4NiIeEbS/qQ/PCdGxAM9FmANxTW+BiNpS+BjwHPA3RHxVm7e/gKYFRHb5vWOJTW5fuKkV315FJUF+fGRwE6kmvV1wB2Sto2IC3K/3zclHQi86+ugm4MTXwOR9G/AacBDwJbAeEnfB54kJcIbJA0DtgL2yZNVWb4McLykE4GlgMGkgxoHkPbDn4E7JU2IiDMkreS+1ebipm6DkPQp4Bpgg4iYnn98O0fEIXn5zsA2pPPF5gPfiIhH6hVvq5K0HemI7TfzKStIWoZ0YvJ5wHYRMU/S46Qa9+g8uKg1ESe+BiHp48CDpHO/Lsvz7gT+AEwh9fHNA5YGlnINo/ryH59rgbER8aSkEcDmEXGlpDWAHwDnAusD6wIX+Dy95uSmboOIiIclbQ7clGsYa5A+AVvKAAAEBUlEQVSOEo4FJgDnkJrBP4+I9+oXaUubRTotZYSkp0knhl+Sl80GXiPduW5rUm3cSa9JucbXYJRuAH4j8HpErFsyfydgckS8VLfgCqDk818IHBERV3QM3poHfmgHlo2IV+saqC0RX6vbYCLiPlKNYhVJ+5bMv85Jr/by578VUHrPW+UBH96NiLed9Jqfm7oNKDd7tyONrtye75Zm/SQiHpG0PXBjPmJ7br1jsupy4mtQETFJ0jjAgw3UQUTcl4f5uk/Se/7j01rcx2fWA0ljSCcmP1HvWKx6nPjMrHB8cMPMCseJz8wKx4nPzArHic/MCseJz8wKx4mvwCQtlDRZ0qOSfpcvyeprWdtIui4//g9Jx/Ww7kr5Bty93caJeRzCiuZ3WudCSZ/vxbZGSHq0tzFac3DiK7a5ETE6IjYG3gcOK12Yb5bd6+9IRFwbEaf0sMpKQK8Tn1m1OPFZhzuA9XJNZ4qknwMPAGtJ2l7S3ZIeyDXDQQCSdpD0eB4+67MdBUn6sqSOO48NlfR7SQ/laQJwCrBurm3+OK/3dUn3SXpY0kklZX1b0hOSbgY2KPcmJB2cy3lI0lWdarHbSrpD0pN50AckDZD045JtH7qkH6Q1Pic+67iJzo6km+ZASjAXR8QY4B3gBGDbiBgLTAKOkbQscD6wM/BJ4EPdFP9T4PaI2IQ0xNZjwHHAM7m2+fV8Xez6wHhgNDBO0lb5kr09STf0+Szphj7lXB0Rm+XtTQEOLFk2gjQAxL8D5+b3cCDpBkKb5fIPlrR2BduxJuZrdYttoKTJ+fEdwK9I4wA+HxH35PmfADYC7ko3FmNp0h3eNiTdTPspAEmXksaq6+xTwH4AEbEQeCvfp6LU9nl6MD8fREqEg4Hfd9wcXdK1FbynjfNw/Svlcm4oWfbbfE+MpyQ9m9/D9sDHS/r/VszbfrKCbVmTcuIrtrkRMbp0Rk5u75TOAm6KiL06rTeadF/ZahDwo4j4n07b+GoftnEhsGtEPCTpy6Th+jt0Livyto/sGGa+ZNsjerldayJu6lo59wBbSFoPQNJykkYCjwNrS+oYLHWvbl5/C3B4fu0ASSuQRjMeXLLODcABJX2Ha0paHZgI7CZpoKTBpGZ1OYOB6Uq35ex8M6bdJbXlmNcBnsjbPjyvj6SRkpavYDvWxFzjsx7l+/Z+Gbg8D4kPcEK+J8UhwJ8kzQLuBDbuooijgPOUbr+4EDg8Iu6WdFc+XeTPuZ9vFHB3rnHOAb4YEQ9IuhKYDDxPao6X8x3g3rz+IyyeYJ8AbgeGAoflmwb9ktT394DSxmcCu1b26Viz8ugsZlY4buqaWeE48ZlZ4TjxmVnhOPGZWeE48ZlZ4TjxmVnhOPGZWeH8fyO5hnaLoZLnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix4, classes = ['Real: 0', 'Fake: 1'],\n",
    "                      title='Confusion matrix, without normalization')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "       Real       0.93      0.93      0.93        28\n",
      "       Fake       0.88      0.88      0.88        17\n",
      "\n",
      "avg / total       0.91      0.91      0.91        45\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_true=y_news2_test, y_pred=svm_predict2, target_names=['Real', 'Fake']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lưu lại model1: ```SVM_clf_2```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./Models/SVM_clf_2.sav']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1_files = 'SVM_clf_2.sav'\n",
    "joblib.dump(svm2, os.path.join(models_path, model1_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}