Analyzing Documents.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyzing Documents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importamos librerías"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import Bonche\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn import metrics\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defino el problema a resolver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PROBLEMA = \"problema1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtengo los datos tanto de prueba como validación"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "getdata = Bonche.ObtenerDatos(PROBLEMA)\n",
    "clusters = getdata.ObtenerClusters(PROBLEMA)\n",
    "X_train = getdata.Data_train(PROBLEMA)\n",
    "X_valid = getdata.Data_valid(PROBLEMA)\n",
    "Y_train = getdata.TargetTrain(PROBLEMA)\n",
    "Y_valid = getdata.TargetValid(PROBLEMA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imprimo forma del problema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Problema:  problema1\n",
      "Num Clusters:  35\n",
      "Num text for train:  40\n",
      "Num text for valid:  10\n",
      "Num ground truth for train 40\n",
      "Num ground truth for valid 10\n"
     ]
    }
   ],
   "source": [
    "print(\"Problema: \", PROBLEMA)\n",
    "print(\"Num Clusters: \", clusters)\n",
    "print(\"Num text for train: \", len(X_train))\n",
    "print(\"Num text for valid: \", len(X_valid))\n",
    "print(\"Num ground truth for train\", len(Y_train))\n",
    "print(\"Num ground truth for valid\", len(Y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inicializo el vectorizador"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(analyzer='char_wb', ngram_range=(3,3), max_df=0.5, max_features=100, norm='l2', min_df=1, stop_words='english', use_idf=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cargo mis datos de entrenamiento al vectorizador"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = vectorizer.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inicializo K-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "km = KMeans(n_clusters=clusters, init='k-means++', max_iter=100, n_init=1, verbose=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cargo mis datos vectorizados a k-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=100,\n",
       "    n_clusters=35, n_init=1, n_jobs=1, precompute_distances='auto',\n",
       "    random_state=None, tol=0.0001, verbose=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "km.fit(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imprimo métricas de rendimiento"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Homogeneity: 0.942\n",
      "Completeness: 0.866\n",
      "V-measure: 0.902\n",
      "Silhouette Coefficient: 0.042\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nHomogeneity: %0.3f\" % metrics.homogeneity_score(Y_train, km.labels_))\n",
    "print(\"Completeness: %0.3f\" % metrics.completeness_score(Y_train, km.labels_))\n",
    "print(\"V-measure: %0.3f\" % metrics.v_measure_score(Y_train, km.labels_))\n",
    "print(\"Silhouette Coefficient: %0.3f\\n\" %metrics.silhouette_score(X, km.labels_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtengo elementos para mostrar datos de importancia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "order_centroids = km.cluster_centers_.argsort()[:, ::-1]\n",
    "terms = vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imprimo elemento relevantes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0:  cot  sco  tis  epe  sal  oni  emb  vot  sts  rli \n",
      "Cluster 1:  iba  ats  alk  nit  fri  t's  ela   ru  it'   mp \n",
      "Cluster 2:  n’t  ’t   ’s   bla  rli  mmi  ela  tia  epe  iat \n",
      "Cluster 3:  iba  nit  lls  ism  ail  bal  vot  pas  nfe  muc \n",
      "Cluster 4:  ’s   orn  loc  tua  ’t   n’t  201  tch  bat  bor \n",
      "Cluster 5:  gg   egg  n't  't   ib   t's   ru   ed   au  nit \n",
      "Cluster 6:  oal  coa  ib   muc  tia  rva  sts  nt,  vis  tru \n",
      "Cluster 7:  egg  tax  gg   ib   ibe  vie  sal  ax    fl  iat \n",
      "Cluster 8:  ibe  ocr  moc  coa  ats  oal  rva  iam  rsh  soc \n",
      "Cluster 9:  bla  ark  tia  n't  acc  't   cot  orn  epe  t's \n",
      "Cluster 10:  bat   mp  ben  lls  ocr  roo  moc  't   ker  n't \n",
      "Cluster 11:  tua  vis  nfe  lve  spi  r's  't   n't  ora  tis \n",
      "Cluster 12:  ib   vot  't   n't  dy    ru  ats  egg  nom  loc \n",
      "Cluster 13:  iam  rli  xt   tis  ’t   n’t  cot  usi  sco  sts \n",
      "Cluster 14:  mmi  t's  riv  acc   uk  icy  vie  tia  bat  sha \n",
      "Cluster 15:   uk   mp  ’s   roc  mmi  cro  vot  uth  bat  roo \n",
      "Cluster 16:  pas  spi  loc  lls  oca  ail  bor  oni  pit  tia \n",
      "Cluster 17:  tax  ax   ib   icy  bor  orn  ink  egg  coa  oal \n",
      "Cluster 18:  cer  oca  uth  emb  ism  loc  fri  bot  oni  bla \n",
      "Cluster 19:  nk   rva  ink  t's  oci  soc  ibe  n't  vot  't  \n",
      "Cluster 20:  roo  201   fl  tax  fri  sco  xt   tis  't   n't \n",
      "Cluster 21:  mmi  vie  n't  't   it'  bor  tch  ink  t's  roc \n",
      "Cluster 22:  egg  gg   ism  ib   pit  ibe  rke  oal  coa  ail \n",
      "Cluster 23:  ’s   bor  ’t   n’t  orn  tax  usi  rke   au  loc \n",
      "Cluster 24:  ’s   tch  ’t   n’t  vie  tit  dy   ark   au  ail \n",
      "Cluster 25:  ’s    mp  ker  roc  vie  alk  tua  sha  nt,  mmi \n",
      "Cluster 26:  egg  alk  ib   gg   acc  fri  tch  uth  ben  nt, \n",
      "Cluster 27:  lls  bal  iba  rsh   ed  coa  oal  bro  sha  it' \n",
      "Cluster 28:  egg  gg   nom   ec  ono  r's  vis  loc  coa  oal \n",
      "Cluster 29:  ib   nfe  sts  vis   ed  coa  oal  bro  iba  vie \n",
      "Cluster 30:  n't  cro  't   it'  ib   rke  gg   nic  t's   ni \n",
      "Cluster 31:  ’s   cot  sco  usi  vot  n’t  ’t   lls  iba  oti \n",
      "Cluster 32:   uk  ’s    ni  ara  ’t   n’t  pit  spi  mmi  tax \n",
      "Cluster 33:  ib   coa  oal  nfe  t's  fri  oni  tis  tia  oti \n",
      "Cluster 34:  iba   ed  bro  r's  bla  nit   ec  ono  nic  emb \n"
     ]
    }
   ],
   "source": [
    "for i in range(clusters):\n",
    "\tprint(\"Cluster %d: \"%i, end='')\n",
    "\tfor ind in order_centroids[i, :10]:\n",
    "\t\tprint(' %s ' %terms[ind], end='')\n",
    "\tprint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}