cluster_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fca180e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import clustering_metrics as cm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import json\n",
    "import nltk\n",
    "import util\n",
    "import imp\n",
    "import os\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e719d941",
   "metadata": {},
   "source": [
    "# Load data\n",
    "- here we use the first extensive clustering in clustering.ipynb to later filter the best clustering algorith /configuration combinations\n",
    "- for some metrics we need unclustered papers thats why we only use clusterings of papers until 2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b3746e3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>publisher</th>\n",
       "      <th>address</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>editor</th>\n",
       "      <th>title</th>\n",
       "      <th>ENTRYTYPE</th>\n",
       "      <th>ID</th>\n",
       "      <th>pages</th>\n",
       "      <th>...</th>\n",
       "      <th>note</th>\n",
       "      <th>pdf</th>\n",
       "      <th>abstract</th>\n",
       "      <th>semantic_scholar</th>\n",
       "      <th>semantic_scholar_authorIds</th>\n",
       "      <th>semantic_scholar_keywords</th>\n",
       "      <th>cso_syntactic</th>\n",
       "      <th>cso_semantic</th>\n",
       "      <th>cso_union</th>\n",
       "      <th>cso_enhanced</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.aclweb.org/anthology/2020.acl-main.1</td>\n",
       "      <td>Association for Computational Linguistics</td>\n",
       "      <td>Online</td>\n",
       "      <td>2020</td>\n",
       "      <td>July</td>\n",
       "      <td></td>\n",
       "      <td>Learning to Understand Child-directed and Adul...</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>gelderloos-etal-2020-learning</td>\n",
       "      <td>1--6</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>2020.acl-main.1.pdf</td>\n",
       "      <td>Speech directed to children differs from adult...</td>\n",
       "      <td>2020.acl-main.1.json</td>\n",
       "      <td>['7805500', '2756960', '103538973']</td>\n",
       "      <td>[1017215, 1588157]</td>\n",
       "      <td>[linguistics, acoustics, language acquisition,...</td>\n",
       "      <td>[speech signals, synthetic speech, linguistics...</td>\n",
       "      <td>[linguistics, automatic speech recognition, ac...</td>\n",
       "      <td>[speech recognition, signal processing, educat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.aclweb.org/anthology/2020.acl-main.2</td>\n",
       "      <td>Association for Computational Linguistics</td>\n",
       "      <td>Online</td>\n",
       "      <td>2020</td>\n",
       "      <td>July</td>\n",
       "      <td></td>\n",
       "      <td>Predicting Depression in Screening Interviews ...</td>\n",
       "      <td>inproceedings</td>\n",
       "      <td>rinaldi-etal-2020-predicting</td>\n",
       "      <td>7--18</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>2020.acl-main.2.pdf</td>\n",
       "      <td>Despite the pervasiveness of clinical depressi...</td>\n",
       "      <td>2020.acl-main.2.json</td>\n",
       "      <td>['19320780', '2457504', '37202877']</td>\n",
       "      <td>[8505]</td>\n",
       "      <td>[linguistics, pattern languages, psycholinguis...</td>\n",
       "      <td>[latent variable, latent factor, linguistics, ...</td>\n",
       "      <td>[latent factor, linguistics, dialogue, pattern...</td>\n",
       "      <td>[matrix factorizations, argumentation, speech ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                url  \\\n",
       "0  https://www.aclweb.org/anthology/2020.acl-main.1   \n",
       "1  https://www.aclweb.org/anthology/2020.acl-main.2   \n",
       "\n",
       "                                   publisher address  year month editor  \\\n",
       "0  Association for Computational Linguistics  Online  2020  July          \n",
       "1  Association for Computational Linguistics  Online  2020  July          \n",
       "\n",
       "                                               title      ENTRYTYPE  \\\n",
       "0  Learning to Understand Child-directed and Adul...  inproceedings   \n",
       "1  Predicting Depression in Screening Interviews ...  inproceedings   \n",
       "\n",
       "                              ID  pages  ... note                  pdf  \\\n",
       "0  gelderloos-etal-2020-learning   1--6  ...       2020.acl-main.1.pdf   \n",
       "1   rinaldi-etal-2020-predicting  7--18  ...       2020.acl-main.2.pdf   \n",
       "\n",
       "                                            abstract      semantic_scholar  \\\n",
       "0  Speech directed to children differs from adult...  2020.acl-main.1.json   \n",
       "1  Despite the pervasiveness of clinical depressi...  2020.acl-main.2.json   \n",
       "\n",
       "            semantic_scholar_authorIds semantic_scholar_keywords  \\\n",
       "0  ['7805500', '2756960', '103538973']        [1017215, 1588157]   \n",
       "1  ['19320780', '2457504', '37202877']                    [8505]   \n",
       "\n",
       "                                       cso_syntactic  \\\n",
       "0  [linguistics, acoustics, language acquisition,...   \n",
       "1  [linguistics, pattern languages, psycholinguis...   \n",
       "\n",
       "                                        cso_semantic  \\\n",
       "0  [speech signals, synthetic speech, linguistics...   \n",
       "1  [latent variable, latent factor, linguistics, ...   \n",
       "\n",
       "                                           cso_union  \\\n",
       "0  [linguistics, automatic speech recognition, ac...   \n",
       "1  [latent factor, linguistics, dialogue, pattern...   \n",
       "\n",
       "                                        cso_enhanced  \n",
       "0  [speech recognition, signal processing, educat...  \n",
       "1  [matrix factorizations, argumentation, speech ...  \n",
       "\n",
       "[2 rows x 28 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load anthology dataframe\n",
    "df = pd.read_csv(\"data/anthology_conferences.csv\", sep=\"|\", keep_default_na=False,\n",
    "                 converters={\"semantic_scholar_keywords\":lambda x: x.strip(\"[]\").replace(\"'\", \"\").split(\", \"),\n",
    "                            \"cso_syntactic\":lambda x: x.strip(\"[]\").replace(\"'\", \"\").split(\", \"),\n",
    "                            \"cso_semantic\":lambda x: x.strip(\"[]\").replace(\"'\", \"\").split(\", \"),\n",
    "                            \"cso_union\":lambda x: x.strip(\"[]\").replace(\"'\", \"\").split(\", \"),\n",
    "                            \"cso_enhanced\":lambda x: x.strip(\"[]\").replace(\"'\", \"\").split(\", \")})\n",
    "df[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fa7b7699",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load clustering results up to 2019\n",
    "with open(\"data/clusters/paraphrase-mpnet-base-v2_titles_abstracts_2019.json\") as jf:\n",
    "    config2clusters_2019 = json.load(jf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "83a8f5f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load embeddings\n",
    "texts, embeddings = util.load_embeddings(\"paraphrase-mpnet-base-v2_titles_abstracts.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fddb146a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seperate dataframe in papers that were used for clustering and in those not used\n",
    "last_year = 2019\n",
    "df_clustered = df[df[\"year\"] <= last_year].reset_index()\n",
    "df_not_clustered = df[df[\"year\"] > last_year].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "94ea4305",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load mappping from semantic scholar topic ids to their topics\n",
    "with open(\"data/semantic_scholar/topicId_mapping.json\") as jf:\n",
    "    id2topic_sem_scholar = json.load(jf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a10c9476",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abstracting and indexing',\n",
       " 'subject headings',\n",
       " 'parse trees',\n",
       " 'part of speech',\n",
       " 'pos tagging',\n",
       " 'pos taggers',\n",
       " 'part of speech tagging',\n",
       " 'natural language text',\n",
       " 'natural language understanding',\n",
       " 'lexical resources',\n",
       " 'wordnet',\n",
       " 'topic model',\n",
       " 'hierarchical dirichlet process',\n",
       " 'text processing',\n",
       " 'word processing',\n",
       " 'electronic document',\n",
       " 'style sheets',\n",
       " 'electronic documents',\n",
       " 'text mining',\n",
       " 'text mining techniques',\n",
       " 'text document',\n",
       " 'textual data',\n",
       " 'text data',\n",
       " 'text representation',\n",
       " 'text summarization',\n",
       " 'automatic text summarization',\n",
       " 'automatic summarization',\n",
       " 'sentence extraction',\n",
       " 'term frequency',\n",
       " 'inverse document frequency',\n",
       " 'document frequency',\n",
       " 'document classification',\n",
       " 'textual entailment',\n",
       " 'sentiment classification',\n",
       " 'text clustering',\n",
       " 'text classification',\n",
       " 'text classifiers',\n",
       " 'reuters 21578',\n",
       " 'term weighting',\n",
       " 'text classification methods',\n",
       " 'document categorization',\n",
       " 'text feature',\n",
       " 'training documents',\n",
       " 'text categorization',\n",
       " 'text recognition',\n",
       " 'text detection',\n",
       " 'text region',\n",
       " 'text localization',\n",
       " 'scene text',\n",
       " 'natural scene images',\n",
       " 'text lines',\n",
       " 'free texts',\n",
       " 'document representation',\n",
       " 'text entry',\n",
       " 'text input',\n",
       " 'information extraction',\n",
       " 'relation extraction',\n",
       " 'information extraction systems',\n",
       " 'web information extraction',\n",
       " 'extracting information',\n",
       " 'information extraction techniques',\n",
       " 'dom tree',\n",
       " 'speech transmission',\n",
       " 'computer aided language translation',\n",
       " 'data abstraction',\n",
       " 'translation systems',\n",
       " 'telluric prospecting',\n",
       " 'speech translation',\n",
       " 'translation models',\n",
       " 'smt systems',\n",
       " 'phrase based statistical machine translation',\n",
       " 'statistical machine translation system',\n",
       " 'translation quality',\n",
       " 'statistical machine translation',\n",
       " 'parallel corpora',\n",
       " 'bilingual corpora',\n",
       " 'word alignment',\n",
       " 'language pairs',\n",
       " 'speech transcriptions',\n",
       " 'target language',\n",
       " 'source language',\n",
       " 'speech to speech translation',\n",
       " 'automatic evaluation',\n",
       " 'sentiment analysis',\n",
       " 'opinion mining',\n",
       " 'product reviews',\n",
       " 'product feature',\n",
       " 'online products',\n",
       " 'customer review',\n",
       " 'semantic orientation',\n",
       " 'polarity classification',\n",
       " 'computational grammars',\n",
       " 'grammatical evolution',\n",
       " 'grammar induction',\n",
       " 'formal grammars',\n",
       " 'grammatical inferences',\n",
       " 'grammatical inference',\n",
       " 'natural language generation',\n",
       " 'word segmentation',\n",
       " 'chinese word segmentation',\n",
       " 'word sense disambiguation',\n",
       " 'word sense',\n",
       " 'n gram language models',\n",
       " 'syntactic structure',\n",
       " 'syntactic analysis',\n",
       " 'syntactic features',\n",
       " 'syntactic information',\n",
       " 'syntactic parsing',\n",
       " 'parsing',\n",
       " 'treebanks',\n",
       " 'cross language information retrieval',\n",
       " 'query translation',\n",
       " 'query translations',\n",
       " 'computing with words',\n",
       " 'computing with word %28cww%29',\n",
       " 'dialogue systems',\n",
       " 'dialogue management',\n",
       " 'dialogue manager',\n",
       " 'spoken dialogue',\n",
       " 'spoken language understanding',\n",
       " 'spoken dialogue system',\n",
       " 'spoken dialogue systems',\n",
       " 'semantic relatedness',\n",
       " 'semantic distance',\n",
       " 'word similarity',\n",
       " 'concept similarity',\n",
       " 'sentence similarity',\n",
       " 'semantic similarity',\n",
       " 'semantic similarity measures',\n",
       " 'named entity recognition',\n",
       " 'name recognition',\n",
       " 'named entities',\n",
       " 'named entity',\n",
       " 'machine translation systems',\n",
       " 'translation process',\n",
       " 'automatic translation',\n",
       " 'bilingual dictionary',\n",
       " 'human evaluation',\n",
       " 'parallel text',\n",
       " 'english sentences',\n",
       " 'machine translations',\n",
       " 'machine translation',\n",
       " 'question answering',\n",
       " 'qa system',\n",
       " 'question classification',\n",
       " 'answer extraction',\n",
       " 'natural language questions',\n",
       " 'passage retrieval',\n",
       " 'complex questions',\n",
       " 'question answering systems',\n",
       " 'question answering system',\n",
       " 'pseudo relevance feedback',\n",
       " 'information retrieval technology',\n",
       " 'n gram models',\n",
       " 'statistical language models',\n",
       " 'statistical language modeling',\n",
       " 'sequence labeling',\n",
       " 'parsing algorithm',\n",
       " 'maximum entropy models',\n",
       " 'maximum entropy model',\n",
       " 'dependency parser',\n",
       " 'dependency trees',\n",
       " 'dependency parsing',\n",
       " 'relevance models',\n",
       " 'lexical database',\n",
       " 'synsets',\n",
       " 'lexical semantics',\n",
       " 'word embedding',\n",
       " 'word embeddings']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load list of cso nlp classes\n",
    "with open(\"data/cso_nlp_hierarchie.json\") as jf:\n",
    "    cso_nlp_hier = json.load(jf)\n",
    "\n",
    "# Get only nlp topics\n",
    "# !!! not used - can be used to show and use only nlp topics later\n",
    "cso_nlp_classes = []\n",
    "def get_nlp_topics_cso(dict_to_search):\n",
    "    for cso_topic in dict_to_search:\n",
    "        cso_topic_cleaned = cso_topic.replace(\"-\", \" \")\n",
    "        cso_topic_cleaned = cso_topic_cleaned.replace(\"_\", \" \")\n",
    "        \n",
    "        if cso_topic_cleaned not in cso_nlp_classes:\n",
    "            cso_nlp_classes.append(cso_topic_cleaned)\n",
    "            \n",
    "        get_nlp_topics_cso(dict_to_search[cso_topic])\n",
    "        \n",
    "get_nlp_topics_cso(cso_nlp_hier['nlp'])\n",
    "cso_nlp_classes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb6969a0",
   "metadata": {},
   "source": [
    "# Evaluation with and without classification\n",
    "In this section, we learn a classifier on the clusters and predict the cluster for new, unclustered embeddings (i.e. paper of not used years)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fd95ae8",
   "metadata": {},
   "source": [
    "## 1. Select a good classifier\n",
    "- Load one clustering result\n",
    "- Try different sklearn classifiers\n",
    "- Training set: 0.8 of clustered embeddings\n",
    "- Testing set: 0.2 of clustered embeddings\n",
    "- Try all classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9a23c28a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create classifiers\n",
    "names = [\"Nearest Neighbors 5\", \"Nearest Neighbors 25\", \"Nearest Neighbors 50\", \"Nearest Neighbors 75\", \"Nearest Neighbors 100\", \n",
    "         \"Nearest Neighbors 200\", \"Nearest Neighbors 300\", \"Nearest Neighbors 400\", \"Nearest Neighbors 500\",\n",
    "         \"Linear SVM 0.015\", \"Linear SVM 0.02\", \"Linear SVM 0.025\", \"Linear SVM 0.055\", \"Linear SVM 0.075\",\n",
    "         \"RBF SVM\", \"MLP 1\", \"MLP 2\", \"MLP 3\", \"MLP 4\", \"MLP 5\", \"Naive Bayes\"]\n",
    "\n",
    "classifiers = [\n",
    "    KNeighborsClassifier(5),\n",
    "    KNeighborsClassifier(25),\n",
    "    KNeighborsClassifier(50),\n",
    "    KNeighborsClassifier(75),\n",
    "    KNeighborsClassifier(100),\n",
    "    KNeighborsClassifier(200),\n",
    "    KNeighborsClassifier(300),\n",
    "    KNeighborsClassifier(400),\n",
    "    KNeighborsClassifier(500),\n",
    "    SVC(kernel=\"linear\", C=0.015),\n",
    "    SVC(kernel=\"linear\", C=0.02),\n",
    "    SVC(kernel=\"linear\", C=0.025),\n",
    "    SVC(kernel=\"linear\", C=0.05),\n",
    "    SVC(kernel=\"linear\", C=0.075),\n",
    "    SVC(gamma=2, C=1),\n",
    "    MLPClassifier(alpha=1, max_iter=1000),\n",
    "    MLPClassifier(alpha=2, max_iter=1000),\n",
    "    MLPClassifier(alpha=3, max_iter=1000),\n",
    "    MLPClassifier(alpha=4, max_iter=1000),\n",
    "    MLPClassifier(alpha=5, max_iter=1000),\n",
    "    GaussianNB()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "261da383",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select one clustering algorithm and a configuration\n",
    "d = config2clusters_2019[\"paraphrase-mpnet-base-v2\"][\"title+abstract\"][\"kmeans\"][\"num_clusters\"][\"25\"][\"distance_thresholds\"][\"null\"][\"min_cluster_sizes\"][\"null\"][\"neighbors\"][\"null\"][\"components\"][\"null\"]\n",
    "cluster2indices = d[\"cluster2indices\"]\n",
    "labels = d[\"cluster2indices\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "81db1f69",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create traning/test set\n",
    "X = []\n",
    "for i, row in df.iterrows():\n",
    "    # only embeddings of papers with year <= last_year\n",
    "    if row[\"year\"] <= last_year:\n",
    "        if -1 in cluster2indices and i in cluster2indices[-1]:\n",
    "            continue\n",
    "        X.append(embeddings[i])\n",
    "        \n",
    "y = [-2 for _ in range(len(X))]\n",
    "for cluster in cluster2indices:\n",
    "    if cluster != -1:\n",
    "        indices = cluster2indices[cluster]\n",
    "        for index in indices:\n",
    "            y[index] = cluster\n",
    "        \n",
    "assert len(y) == len(X), \"{} vs {}\".format(len(y), len(X))\n",
    "assert len([1 for x in y if x == -2]) == 0\n",
    "assert -1 not in y\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e741eca3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of classifier 'Nearest Neighbors 5': 0.712852897473997\n",
      "Accuracy of classifier 'Nearest Neighbors 25': 0.7734026745913819\n",
      "Accuracy of classifier 'Nearest Neighbors 50': 0.7897473997028231\n",
      "Accuracy of classifier 'Nearest Neighbors 75': 0.7956909361069836\n",
      "Accuracy of classifier 'Nearest Neighbors 100': 0.7897473997028231\n",
      "Accuracy of classifier 'Nearest Neighbors 200': 0.787890044576523\n",
      "Accuracy of classifier 'Nearest Neighbors 300': 0.7800891530460624\n",
      "Accuracy of classifier 'Nearest Neighbors 400': 0.7719167904903418\n",
      "Accuracy of classifier 'Nearest Neighbors 500': 0.7618870728083209\n",
      "Accuracy of classifier 'Linear SVM 0.015': 0.9075037147102526\n",
      "Accuracy of classifier 'Linear SVM 0.02': 0.9097325408618128\n",
      "Accuracy of classifier 'Linear SVM 0.025': 0.912704309063893\n",
      "Accuracy of classifier 'Linear SVM 0.055': 0.9104754829123328\n",
      "Accuracy of classifier 'Linear SVM 0.075': 0.9063893016344725\n",
      "Accuracy of classifier 'RBF SVM': 0.08283803863298662\n",
      "Accuracy of classifier 'MLP 1': 0.9067607726597325\n",
      "Accuracy of classifier 'MLP 2': 0.8632986627043091\n",
      "Accuracy of classifier 'MLP 3': 0.8187221396731055\n",
      "Accuracy of classifier 'MLP 4': 0.725482912332838\n",
      "Accuracy of classifier 'MLP 5': 0.6132986627043091\n",
      "Accuracy of classifier 'Naive Bayes': 0.9015601783060921\n"
     ]
    }
   ],
   "source": [
    "best_score = 0\n",
    "best_classifier_by_index = -1\n",
    "\n",
    "for i, (classifier, name) in enumerate(zip(classifiers, names)):\n",
    "    classifier.fit(X_train, y_train)\n",
    "    score = classifier.score(X_test, y_test)\n",
    "    \n",
    "    print(\"Accuracy of classifier '\" + name + \"':\", score)\n",
    "    \n",
    "    if score > best_score:\n",
    "        best_score = score\n",
    "        best_classifier_by_index = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "039572a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best classifier: Linear SVM 0.025\n"
     ]
    }
   ],
   "source": [
    "# Select best classifier for further evalution\n",
    "print(\"Best classifier:\", names[best_classifier_by_index])\n",
    "clf = classifiers[best_classifier_by_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3720ee5b",
   "metadata": {},
   "source": [
    "## 2. Train and classify not clustered embeddings with best classifier and based on clustering data\n",
    "- Do it for all clustering algorithms and configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a22697aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained_models = [\"paraphrase-mpnet-base-v2\"]\n",
    "text_sets = [\"title+abstract\"]\n",
    "cluster_algorithms = [\"kmeans\", \"agglomerative\", \"topic\"] #[\"kmeans\", \"agglomerative\", \"fast\", \"topic\"]\n",
    "num_clusters = {\"kmeans\":[10, 15, 20, 25, 30, 35, 40, 45, 50], \"agglomerative\": [\"null\", 10, 15, 20, 25, 30, 35, 40, 45, 50], \"fast\": [\"null\"], \"topic\":[\"null\"]}\n",
    "distance_thresholds = {\"kmeans\":[\"null\"], \"agglomerative\": [\"null\", 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], \"fast\": [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], \"topic\": [\"null\"]}\n",
    "min_cluster_size = {\"kmeans\": [\"null\"], \"agglomerative\": [\"null\"], \"fast\": [\"null\"], \"topic\": [100, 150, 200, 250, 300, 350, 400, 500]}\n",
    "neighbors = {\"kmeans\": [\"null\"], \"agglomerative\": [\"null\"], \"fast\": [\"null\"], \"topic\": [2, 5, 10, 20, 50, 75, 100, 200, 400]}\n",
    "components = {\"kmeans\": [\"null\"], \"agglomerative\": [\"null\"], \"fast\": [\"null\"], \"topic\": [2, 5, 10, 15, 20, 30, 40, 60, 80, 100, int(len(embeddings[0])/4), int(len(embeddings[0])/2), len(embeddings[0])]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5a1aa1ac",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'load_embeddings' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-f6ec278965fd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mtext_set\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtext_sets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtext_set\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"title+abstract\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_embeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"_titles_abstracts.pkl\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_embeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"_\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtext_set\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\".pkl\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'load_embeddings' is not defined"
     ]
    }
   ],
   "source": [
    "config2data = dict()       # data: (cluster2indices, X_train, y_train, X_predict, y_predicted)\n",
    "\n",
    "for pretrained_model in pretrained_models:   \n",
    "    for text_set in text_sets:\n",
    "        if text_set == \"title+abstract\":\n",
    "            _, embeddings = load_embeddings(pretrained_model + \"_titles_abstracts.pkl\")\n",
    "        else:\n",
    "            _, embeddings = load_embeddings(pretrained_model + \"_\" + text_set + \".pkl\")\n",
    "            \n",
    "        for algorithm in cluster_algorithms:\n",
    "            for num_cluster in num_clusters[algorithm]:\n",
    "                \n",
    "                if algorithm == \"agglomerative\" and num_cluster != \"null\":\n",
    "                    thresholds = [\"null\"]\n",
    "                elif algorithm == \"agglomerative\" and num_cluster == \"null\":\n",
    "                    thresholds = distance_thresholds[algorithm][1:]\n",
    "                else:\n",
    "                    thresholds = distance_thresholds[algorithm]\n",
    "                \n",
    "                for distance_threshold in thresholds:\n",
    "                    for cluster_size in min_cluster_size[algorithm]:\n",
    "                        for num_neighbors in neighbors[algorithm]:\n",
    "                            for component in components[algorithm]:\n",
    "                                \n",
    "                                key = str(last_year) + \" - \" + pretrained_model + \" - \" + text_set + \" - \" + algorithm + \" - \" + str(num_cluster) + \" - \" + str(distance_threshold) + \" - \" + str(cluster_size) + \" - \" + str(num_neighbors) + \" - \" + str(component)\n",
    "                                \n",
    "                                #try:\n",
    "                                cluster2indices, X_train, y_train, X_predict = util.get_classifier_data_sets(config2clusters_2019, df, 2019, embeddings, \n",
    "                                    pretrained_model, text_set, algorithm, num_cluster, distance_threshold, cluster_size, num_neighbors, component)\n",
    "                                #except:\n",
    "                                #    print(\"Error:\", key)\n",
    "                                \n",
    "                                clf.fit(X_train, y_train)\n",
    "                                y_predicted = clf.predict(X_predict)\n",
    "                                \n",
    "                                \n",
    "                                config2data[key] = (cluster2indices, X_train, y_train, X_predict, y_predicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22197512",
   "metadata": {},
   "source": [
    "## 3. Check with semantic scholar topics and cso topics if chosen clusters are appropriate\n",
    "- All clustering algorithms and configurations are tested for current embeddings\n",
    "- Store results of all evaluation metrics in a dataframe\n",
    "- Filter appropriate configurations and store them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7705423",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dataframe for _c_lustering _e_valuation with column 'config'\n",
    "df_ce = pd.DataFrame([[c] for c in config2data], columns=[\"config\"])\n",
    "df_ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fd83b31",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_metrics(df_ce, config2data):   \n",
    "    df_ce[\"num_clusters\"] = 0.0\n",
    "    df_ce[\"min_members\"] = 0.0\n",
    "    df_ce[\"max_members\"] = 0.0\n",
    "    df_ce[\"mean_members\"] = 0.0\n",
    "    df_ce[\"median_members\"] = 0.0\n",
    "\n",
    "    top_n_shared_words = [5, 10, 15]\n",
    "    for n in top_n_shared_words:\n",
    "        df_ce[\"intra_similarity_sem_scholar_\" + str(n)] = 0.0\n",
    "        df_ce[\"inter_similarity_sem_scholar_\" + str(n)] = 0.0\n",
    "        df_ce[\"intra_similarity_cso_enhanced_\" + str(n)] = 0.0\n",
    "        df_ce[\"inter_similarity_cso_enhanced_\" + str(n)] = 0.0\n",
    "\n",
    "    df_ce[\"accuracy_new_papers_sem_scholar\"] = 0.0\n",
    "    df_ce[\"ranking_score_new_papers_sem_scholar\"] = 0.0\n",
    "    df_ce[\"absolute_score_new_papers_sem_scholar\"] = 0.0\n",
    "    df_ce[\"accuracy_new_papers_cso_enhanced\"] = 0.0\n",
    "    df_ce[\"ranking_score_new_papers_cso_enhanced\"] = 0.0\n",
    "    df_ce[\"absolute_score_new_papers_cso_enhanced\"] = 0.0\n",
    "    \n",
    "    df_ce[\"papers_clustered\"] = 0.0\n",
    "\n",
    "    for i, row in df_ce.iterrows():\n",
    "        c = row[\"config\"]\n",
    "        (cluster2indices, X_train, y_train, X_predict, y_predicted) = config2data[c]\n",
    "\n",
    "        # simple metrics\n",
    "        df_ce.at[i, \"num_clusters\"] = cm.get_num_clusters(cluster2indices)\n",
    "        df_ce.at[i, \"min_members\"] = cm.get_min_cluster_size(cluster2indices)\n",
    "        df_ce.at[i, \"max_members\"] = cm.get_max_cluster_size(cluster2indices)\n",
    "        df_ce.at[i, \"mean_members\"] = cm.get_mean_cluster_size(cluster2indices)\n",
    "        df_ce.at[i, \"median_members\"] = cm.get_median_cluster_size(cluster2indices)\n",
    "\n",
    "        # semantic scholar keywords\n",
    "        cluster2keywords_freq_dist = util.get_cluster2words_freq_dist(cluster2indices, \"sem_scholar\", df_clustered)\n",
    "        # cso topics\n",
    "        cluster2topics_freq_dist = util.get_cluster2words_freq_dist(cluster2indices, \"cso\", df_clustered)\n",
    "\n",
    "        # intra cluster similarity keywords semantic scholar and topics cso: mean % of all clusters in a config\n",
    "        n2mean_percentage_sem_scholar_keywords = cm.intra_cluster_similarity(cluster2indices, cluster2keywords_freq_dist, \"sem_scholar\", df_clustered, n=top_n_shared_words)\n",
    "        for n in n2mean_percentage_sem_scholar_keywords:\n",
    "            df_ce.at[i, \"intra_similarity_sem_scholar_\" + str(n)] = n2mean_percentage_sem_scholar_keywords[n]\n",
    "        n2mean_percentage_cso_topics = cm.intra_cluster_similarity(cluster2indices, cluster2topics_freq_dist, \"cso\", df_clustered, n=top_n_shared_words)\n",
    "        for n in n2mean_percentage_cso_topics:\n",
    "            df_ce.at[i, \"intra_similarity_cso_enhanced_\" + str(n)] = n2mean_percentage_cso_topics[n]\n",
    "\n",
    "        # inter cluster similarity key words semantic scholar and topics cso: mean of all cluster pairs\n",
    "        if df_ce.loc[i, \"num_clusters\"] <= 500:\n",
    "            n2mean_correlations_sem_scholar_keywords = cm.inter_cluster_similarity(cluster2indices, cluster2keywords_freq_dist, \"sem_scholar\", n=top_n_shared_words)\n",
    "            for n in n2mean_correlations_sem_scholar_keywords:\n",
    "                df_ce.at[i, \"inter_similarity_sem_scholar_\" + str(n)] = n2mean_correlations_sem_scholar_keywords[n]\n",
    "        if df_ce.loc[i, \"num_clusters\"] <= 500:\n",
    "            n2mean_correlations_cso_topics = cm.inter_cluster_similarity(cluster2indices, cluster2topics_freq_dist, \"cso\", n=top_n_shared_words)\n",
    "            for n in n2mean_correlations_cso_topics:\n",
    "                df_ce.at[i, \"inter_similarity_cso_enhanced_\" + str(n)] = n2mean_correlations_cso_topics[n]\n",
    "\n",
    "        # new unclustered accuracy by semantic scholar keywords and cso topics\n",
    "        # and score for place of correct cluster in list of clusters sorted by \n",
    "        # keyword similarity / topic similarity score for each paper\n",
    "        accuracy, ranking_score, absolute_score = cm.classification_acc_ranking(y_predicted, cluster2keywords_freq_dist, \"sem_scholar\", cluster2indices, df_not_clustered)\n",
    "        df_ce.at[i, \"accuracy_new_papers_sem_scholar\"] = accuracy\n",
    "        df_ce.at[i, \"ranking_score_new_papers_sem_scholar\"] = ranking_score\n",
    "        df_ce.at[i, \"absolute_score_new_papers_sem_scholar\"] = absolute_score\n",
    "        accuracy, ranking_score, absolute_score = cm.classification_acc_ranking(y_predicted, cluster2topics_freq_dist, \"cso\", cluster2indices, df_not_clustered)\n",
    "        df_ce.at[i, \"accuracy_new_papers_cso_enhanced\"] = accuracy\n",
    "        df_ce.at[i, \"ranking_score_new_papers_cso_enhanced\"] = ranking_score\n",
    "        df_ce.at[i, \"absolute_score_new_papers_cso_enhanced\"] = absolute_score\n",
    "        \n",
    "        num_papers = 0\n",
    "        num_clustered = 0\n",
    "        for cluster_index in cluster2indices:\n",
    "            if cluster_index == -1 or cluster_index == \"-1\":\n",
    "                num_papers += len(cluster2indices[cluster_index])\n",
    "            else:\n",
    "                num_papers += len(cluster2indices[cluster_index])\n",
    "                num_clustered += len(cluster2indices[cluster_index])\n",
    "                \n",
    "        df_ce.at[i, \"papers_clustered\"] = num_clustered/num_papers\n",
    "\n",
    "        #if (i+1) % 25 == 0:\n",
    "        print(str(i+1) + \"/\" + str(len(df_ce)) + \" configurations\")\n",
    "\n",
    "    return df_ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5693039d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ce = evaluate_metrics(df_ce, config2data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "e9bd3206",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store df_ce on disk\n",
    "df_ce.to_csv(\"data/clusters/cluster_evaluation_\" + pretrained_models[0] + \"_\" + text_sets[0] + \".csv\", sep=\"|\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4cdab66c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "961\n"
     ]
    }
   ],
   "source": [
    "# Load df_ce from disk\n",
    "df_ce = pd.read_csv(\"data/clusters/cluster_evaluation_\" + pretrained_models[0] + \"_\" + text_sets[0] + \".csv\", sep=\"|\")\n",
    "print(len(df_ce))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8732e775",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter out configurations with too less or too many clusters but keep all kmeans and agglomerative clusterings\n",
    "df_ce_filtered = df_ce[(df_ce[\"num_clusters\"] >= 10.0)]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"num_clusters\"] <= 50.0)]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"median_members\"] >= 150.0) | (df_ce_filtered[\"config\"].str.contains('.*kmeans.*|.*agglomerative.*'))]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"inter_similarity_cso_enhanced_15\"].abs() <= 0.03) | (df_ce_filtered[\"config\"].str.contains('.*kmeans.*|.*agglomerative.*'))]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"inter_similarity_sem_scholar_15\"].abs() <= 0.03) | (df_ce_filtered[\"config\"].str.contains('.*kmeans.*|.*agglomerative.*'))]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"inter_similarity_cso_enhanced_10\"].abs() <= 0.03) | (df_ce_filtered[\"config\"].str.contains('.*kmeans.*|.*agglomerative.*'))]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"inter_similarity_sem_scholar_10\"].abs() <= 0.03) | (df_ce_filtered[\"config\"].str.contains('.*kmeans.*|.*agglomerative.*'))]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"ranking_score_new_papers_sem_scholar\"].abs() >= 0.3) | (df_ce_filtered[\"config\"].str.contains('.*kmeans.*|.*agglomerative.*'))]\n",
    "df_ce_filtered = df_ce_filtered[(df_ce_filtered[\"ranking_score_new_papers_cso_enhanced\"].abs() >= 0.3) | (df_ce_filtered[\"config\"].str.contains('.*kmeans.*|.*agglomerative.*'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c7b96806",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print sorted values of a metric\n",
    "# df_ce_filtered.sort_values(by=['inter_similarity_cso_enhanced_10'])['inter_similarity_cso_enhanced_10']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "362bc8da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of remaining clusterings: 30\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 100 - 2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>config</th>\n",
       "      <th>num_clusters</th>\n",
       "      <th>min_members</th>\n",
       "      <th>max_members</th>\n",
       "      <th>mean_members</th>\n",
       "      <th>median_members</th>\n",
       "      <th>intra_similarity_sem_scholar_5</th>\n",
       "      <th>inter_similarity_sem_scholar_5</th>\n",
       "      <th>intra_similarity_cso_enhanced_5</th>\n",
       "      <th>inter_similarity_cso_enhanced_5</th>\n",
       "      <th>...</th>\n",
       "      <th>intra_similarity_cso_enhanced_10</th>\n",
       "      <th>inter_similarity_cso_enhanced_10</th>\n",
       "      <th>intra_similarity_sem_scholar_15</th>\n",
       "      <th>inter_similarity_sem_scholar_15</th>\n",
       "      <th>intra_similarity_cso_enhanced_15</th>\n",
       "      <th>inter_similarity_cso_enhanced_15</th>\n",
       "      <th>accuracy_new_papers_sem_scholar</th>\n",
       "      <th>ranking_score_new_papers_sem_scholar</th>\n",
       "      <th>accuracy_new_papers_cso_enhanced</th>\n",
       "      <th>ranking_score_new_papers_cso_enhanced</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>231.0</td>\n",
       "      <td>1756.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>1495.0</td>\n",
       "      <td>0.671405</td>\n",
       "      <td>0.015556</td>\n",
       "      <td>0.835039</td>\n",
       "      <td>0.131111</td>\n",
       "      <td>...</td>\n",
       "      <td>0.916281</td>\n",
       "      <td>0.064512</td>\n",
       "      <td>0.842505</td>\n",
       "      <td>-0.011508</td>\n",
       "      <td>0.942581</td>\n",
       "      <td>0.057143</td>\n",
       "      <td>0.185727</td>\n",
       "      <td>0.364594</td>\n",
       "      <td>0.323582</td>\n",
       "      <td>0.532052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>225.0</td>\n",
       "      <td>1333.0</td>\n",
       "      <td>897.133333</td>\n",
       "      <td>959.0</td>\n",
       "      <td>0.701527</td>\n",
       "      <td>-0.017143</td>\n",
       "      <td>0.856118</td>\n",
       "      <td>0.038095</td>\n",
       "      <td>...</td>\n",
       "      <td>0.922405</td>\n",
       "      <td>0.012063</td>\n",
       "      <td>0.859994</td>\n",
       "      <td>0.003946</td>\n",
       "      <td>0.949055</td>\n",
       "      <td>-0.002483</td>\n",
       "      <td>0.167553</td>\n",
       "      <td>0.318922</td>\n",
       "      <td>0.288564</td>\n",
       "      <td>0.478497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>1083.0</td>\n",
       "      <td>672.850000</td>\n",
       "      <td>729.0</td>\n",
       "      <td>0.720542</td>\n",
       "      <td>0.123684</td>\n",
       "      <td>0.870013</td>\n",
       "      <td>0.110526</td>\n",
       "      <td>...</td>\n",
       "      <td>0.926564</td>\n",
       "      <td>0.046252</td>\n",
       "      <td>0.870194</td>\n",
       "      <td>-0.000526</td>\n",
       "      <td>0.952981</td>\n",
       "      <td>0.022218</td>\n",
       "      <td>0.165337</td>\n",
       "      <td>0.291825</td>\n",
       "      <td>0.296543</td>\n",
       "      <td>0.458566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>973.0</td>\n",
       "      <td>538.280000</td>\n",
       "      <td>563.0</td>\n",
       "      <td>0.736495</td>\n",
       "      <td>0.051000</td>\n",
       "      <td>0.878603</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.932463</td>\n",
       "      <td>0.057172</td>\n",
       "      <td>0.880146</td>\n",
       "      <td>0.006845</td>\n",
       "      <td>0.958013</td>\n",
       "      <td>0.032905</td>\n",
       "      <td>0.154699</td>\n",
       "      <td>0.275315</td>\n",
       "      <td>0.288564</td>\n",
       "      <td>0.447454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>30.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>769.0</td>\n",
       "      <td>448.566667</td>\n",
       "      <td>407.5</td>\n",
       "      <td>0.754377</td>\n",
       "      <td>0.001839</td>\n",
       "      <td>0.878668</td>\n",
       "      <td>0.028276</td>\n",
       "      <td>...</td>\n",
       "      <td>0.938208</td>\n",
       "      <td>-0.002382</td>\n",
       "      <td>0.891048</td>\n",
       "      <td>-0.001314</td>\n",
       "      <td>0.960402</td>\n",
       "      <td>0.002397</td>\n",
       "      <td>0.137855</td>\n",
       "      <td>0.250003</td>\n",
       "      <td>0.274823</td>\n",
       "      <td>0.425155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>35.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>700.0</td>\n",
       "      <td>384.485714</td>\n",
       "      <td>367.0</td>\n",
       "      <td>0.749197</td>\n",
       "      <td>0.037479</td>\n",
       "      <td>0.874138</td>\n",
       "      <td>0.042521</td>\n",
       "      <td>...</td>\n",
       "      <td>0.935251</td>\n",
       "      <td>0.013822</td>\n",
       "      <td>0.888284</td>\n",
       "      <td>0.002953</td>\n",
       "      <td>0.957690</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>0.144947</td>\n",
       "      <td>0.242658</td>\n",
       "      <td>0.280585</td>\n",
       "      <td>0.422815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>40.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>602.0</td>\n",
       "      <td>336.425000</td>\n",
       "      <td>341.0</td>\n",
       "      <td>0.751642</td>\n",
       "      <td>0.008333</td>\n",
       "      <td>0.869797</td>\n",
       "      <td>0.018333</td>\n",
       "      <td>...</td>\n",
       "      <td>0.931910</td>\n",
       "      <td>0.023792</td>\n",
       "      <td>0.889370</td>\n",
       "      <td>-0.002445</td>\n",
       "      <td>0.959232</td>\n",
       "      <td>0.007285</td>\n",
       "      <td>0.126330</td>\n",
       "      <td>0.216895</td>\n",
       "      <td>0.261968</td>\n",
       "      <td>0.399544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>45.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>518.0</td>\n",
       "      <td>299.044444</td>\n",
       "      <td>307.0</td>\n",
       "      <td>0.753758</td>\n",
       "      <td>0.018990</td>\n",
       "      <td>0.866670</td>\n",
       "      <td>0.052525</td>\n",
       "      <td>...</td>\n",
       "      <td>0.931871</td>\n",
       "      <td>0.021022</td>\n",
       "      <td>0.887684</td>\n",
       "      <td>-0.000263</td>\n",
       "      <td>0.955677</td>\n",
       "      <td>0.023921</td>\n",
       "      <td>0.120567</td>\n",
       "      <td>0.209862</td>\n",
       "      <td>0.252216</td>\n",
       "      <td>0.386457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>493.0</td>\n",
       "      <td>269.140000</td>\n",
       "      <td>273.5</td>\n",
       "      <td>0.765297</td>\n",
       "      <td>0.011020</td>\n",
       "      <td>0.869220</td>\n",
       "      <td>0.027510</td>\n",
       "      <td>...</td>\n",
       "      <td>0.936448</td>\n",
       "      <td>0.009395</td>\n",
       "      <td>0.891706</td>\n",
       "      <td>0.000945</td>\n",
       "      <td>0.960146</td>\n",
       "      <td>0.004157</td>\n",
       "      <td>0.122340</td>\n",
       "      <td>0.208268</td>\n",
       "      <td>0.241135</td>\n",
       "      <td>0.374593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13124.0</td>\n",
       "      <td>897.133333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.828453</td>\n",
       "      <td>-0.010476</td>\n",
       "      <td>0.822870</td>\n",
       "      <td>0.017143</td>\n",
       "      <td>...</td>\n",
       "      <td>0.908463</td>\n",
       "      <td>0.029683</td>\n",
       "      <td>0.930672</td>\n",
       "      <td>-0.002843</td>\n",
       "      <td>0.958558</td>\n",
       "      <td>0.024591</td>\n",
       "      <td>0.062943</td>\n",
       "      <td>0.460138</td>\n",
       "      <td>0.009309</td>\n",
       "      <td>0.234509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13150.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.837679</td>\n",
       "      <td>0.091111</td>\n",
       "      <td>0.859256</td>\n",
       "      <td>0.015556</td>\n",
       "      <td>...</td>\n",
       "      <td>0.929335</td>\n",
       "      <td>0.000274</td>\n",
       "      <td>0.916010</td>\n",
       "      <td>0.068653</td>\n",
       "      <td>0.937816</td>\n",
       "      <td>0.011592</td>\n",
       "      <td>0.132092</td>\n",
       "      <td>0.544666</td>\n",
       "      <td>0.028812</td>\n",
       "      <td>0.307789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13124.0</td>\n",
       "      <td>897.133333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.828453</td>\n",
       "      <td>-0.010476</td>\n",
       "      <td>0.822870</td>\n",
       "      <td>0.017143</td>\n",
       "      <td>...</td>\n",
       "      <td>0.908463</td>\n",
       "      <td>0.029683</td>\n",
       "      <td>0.930672</td>\n",
       "      <td>-0.002843</td>\n",
       "      <td>0.958558</td>\n",
       "      <td>0.024591</td>\n",
       "      <td>0.062943</td>\n",
       "      <td>0.460138</td>\n",
       "      <td>0.009309</td>\n",
       "      <td>0.234509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13113.0</td>\n",
       "      <td>672.850000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.823173</td>\n",
       "      <td>-0.018421</td>\n",
       "      <td>0.809048</td>\n",
       "      <td>0.028421</td>\n",
       "      <td>...</td>\n",
       "      <td>0.885783</td>\n",
       "      <td>0.024830</td>\n",
       "      <td>0.924758</td>\n",
       "      <td>0.006227</td>\n",
       "      <td>0.960879</td>\n",
       "      <td>0.022852</td>\n",
       "      <td>0.054078</td>\n",
       "      <td>0.344838</td>\n",
       "      <td>0.002660</td>\n",
       "      <td>0.214347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13081.0</td>\n",
       "      <td>538.280000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.818889</td>\n",
       "      <td>-0.009000</td>\n",
       "      <td>0.837097</td>\n",
       "      <td>0.008667</td>\n",
       "      <td>...</td>\n",
       "      <td>0.898473</td>\n",
       "      <td>0.001556</td>\n",
       "      <td>0.925220</td>\n",
       "      <td>0.003504</td>\n",
       "      <td>0.968710</td>\n",
       "      <td>0.022650</td>\n",
       "      <td>0.040780</td>\n",
       "      <td>0.313575</td>\n",
       "      <td>0.000887</td>\n",
       "      <td>0.164979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13048.0</td>\n",
       "      <td>448.566667</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.812563</td>\n",
       "      <td>-0.017701</td>\n",
       "      <td>0.843150</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.904893</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.913753</td>\n",
       "      <td>0.001561</td>\n",
       "      <td>0.972061</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.037677</td>\n",
       "      <td>0.300831</td>\n",
       "      <td>0.000887</td>\n",
       "      <td>0.140675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13040.0</td>\n",
       "      <td>384.485714</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.834102</td>\n",
       "      <td>0.009076</td>\n",
       "      <td>0.855642</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.937133</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.923378</td>\n",
       "      <td>0.020800</td>\n",
       "      <td>0.977246</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.037677</td>\n",
       "      <td>0.295618</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.122359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12982.0</td>\n",
       "      <td>336.425000</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.845266</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.870880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.947815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.920775</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.986045</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.031028</td>\n",
       "      <td>0.207242</td>\n",
       "      <td>0.003989</td>\n",
       "      <td>0.114882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>45.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12931.0</td>\n",
       "      <td>299.044444</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.833814</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.866370</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.942909</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.919577</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.979671</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.021720</td>\n",
       "      <td>0.195073</td>\n",
       "      <td>0.003989</td>\n",
       "      <td>0.106518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12887.0</td>\n",
       "      <td>269.140000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.818313</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.875203</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.944088</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.910090</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.977268</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.017730</td>\n",
       "      <td>0.184983</td>\n",
       "      <td>0.005762</td>\n",
       "      <td>0.099151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>10951.0</td>\n",
       "      <td>1223.363636</td>\n",
       "      <td>213.0</td>\n",
       "      <td>0.709176</td>\n",
       "      <td>-0.010909</td>\n",
       "      <td>0.875340</td>\n",
       "      <td>0.069091</td>\n",
       "      <td>...</td>\n",
       "      <td>0.937584</td>\n",
       "      <td>0.028099</td>\n",
       "      <td>0.864905</td>\n",
       "      <td>0.011104</td>\n",
       "      <td>0.955221</td>\n",
       "      <td>0.016104</td>\n",
       "      <td>0.138298</td>\n",
       "      <td>0.348177</td>\n",
       "      <td>0.277926</td>\n",
       "      <td>0.470488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>6619.0</td>\n",
       "      <td>841.062500</td>\n",
       "      <td>354.5</td>\n",
       "      <td>0.768704</td>\n",
       "      <td>0.054167</td>\n",
       "      <td>0.886962</td>\n",
       "      <td>0.135000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.944986</td>\n",
       "      <td>0.028384</td>\n",
       "      <td>0.893540</td>\n",
       "      <td>0.000387</td>\n",
       "      <td>0.961280</td>\n",
       "      <td>0.017113</td>\n",
       "      <td>0.141844</td>\n",
       "      <td>0.303639</td>\n",
       "      <td>0.335550</td>\n",
       "      <td>0.488527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>7044.0</td>\n",
       "      <td>841.062500</td>\n",
       "      <td>361.5</td>\n",
       "      <td>0.771966</td>\n",
       "      <td>0.055833</td>\n",
       "      <td>0.887615</td>\n",
       "      <td>0.087500</td>\n",
       "      <td>...</td>\n",
       "      <td>0.948332</td>\n",
       "      <td>0.014545</td>\n",
       "      <td>0.897202</td>\n",
       "      <td>-0.016250</td>\n",
       "      <td>0.964992</td>\n",
       "      <td>-0.008214</td>\n",
       "      <td>0.141844</td>\n",
       "      <td>0.301414</td>\n",
       "      <td>0.328457</td>\n",
       "      <td>0.481557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>191.0</td>\n",
       "      <td>6232.0</td>\n",
       "      <td>1035.153846</td>\n",
       "      <td>553.0</td>\n",
       "      <td>0.734251</td>\n",
       "      <td>-0.047436</td>\n",
       "      <td>0.855391</td>\n",
       "      <td>0.030769</td>\n",
       "      <td>...</td>\n",
       "      <td>0.927664</td>\n",
       "      <td>-0.010878</td>\n",
       "      <td>0.876212</td>\n",
       "      <td>-0.017674</td>\n",
       "      <td>0.951381</td>\n",
       "      <td>-0.013965</td>\n",
       "      <td>0.152482</td>\n",
       "      <td>0.320867</td>\n",
       "      <td>0.348848</td>\n",
       "      <td>0.513929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>7671.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>672.0</td>\n",
       "      <td>0.767794</td>\n",
       "      <td>0.017778</td>\n",
       "      <td>0.885364</td>\n",
       "      <td>0.084444</td>\n",
       "      <td>...</td>\n",
       "      <td>0.949692</td>\n",
       "      <td>0.013064</td>\n",
       "      <td>0.891887</td>\n",
       "      <td>-0.024206</td>\n",
       "      <td>0.964024</td>\n",
       "      <td>0.028968</td>\n",
       "      <td>0.170213</td>\n",
       "      <td>0.412949</td>\n",
       "      <td>0.334663</td>\n",
       "      <td>0.507474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>7572.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>653.0</td>\n",
       "      <td>0.768656</td>\n",
       "      <td>0.031111</td>\n",
       "      <td>0.887046</td>\n",
       "      <td>0.062222</td>\n",
       "      <td>...</td>\n",
       "      <td>0.949076</td>\n",
       "      <td>-0.007677</td>\n",
       "      <td>0.890820</td>\n",
       "      <td>0.004206</td>\n",
       "      <td>0.965510</td>\n",
       "      <td>-0.012937</td>\n",
       "      <td>0.169770</td>\n",
       "      <td>0.411343</td>\n",
       "      <td>0.332890</td>\n",
       "      <td>0.508297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>303.0</td>\n",
       "      <td>7772.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>610.0</td>\n",
       "      <td>0.771398</td>\n",
       "      <td>0.044444</td>\n",
       "      <td>0.895215</td>\n",
       "      <td>0.057778</td>\n",
       "      <td>...</td>\n",
       "      <td>0.952527</td>\n",
       "      <td>-0.011717</td>\n",
       "      <td>0.893062</td>\n",
       "      <td>-0.006032</td>\n",
       "      <td>0.967831</td>\n",
       "      <td>-0.020397</td>\n",
       "      <td>0.173759</td>\n",
       "      <td>0.414313</td>\n",
       "      <td>0.338209</td>\n",
       "      <td>0.509125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>220.0</td>\n",
       "      <td>7499.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>627.5</td>\n",
       "      <td>0.773557</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>0.894521</td>\n",
       "      <td>0.068889</td>\n",
       "      <td>...</td>\n",
       "      <td>0.951447</td>\n",
       "      <td>0.028148</td>\n",
       "      <td>0.892259</td>\n",
       "      <td>0.000952</td>\n",
       "      <td>0.969262</td>\n",
       "      <td>0.015873</td>\n",
       "      <td>0.172872</td>\n",
       "      <td>0.419347</td>\n",
       "      <td>0.339539</td>\n",
       "      <td>0.507377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>7612.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>622.0</td>\n",
       "      <td>0.772661</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>0.892122</td>\n",
       "      <td>0.051111</td>\n",
       "      <td>...</td>\n",
       "      <td>0.951950</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.894630</td>\n",
       "      <td>-0.023651</td>\n",
       "      <td>0.969647</td>\n",
       "      <td>-0.007619</td>\n",
       "      <td>0.171543</td>\n",
       "      <td>0.418878</td>\n",
       "      <td>0.332890</td>\n",
       "      <td>0.504011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>7318.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>658.0</td>\n",
       "      <td>0.773099</td>\n",
       "      <td>-0.048889</td>\n",
       "      <td>0.893982</td>\n",
       "      <td>0.015556</td>\n",
       "      <td>...</td>\n",
       "      <td>0.954621</td>\n",
       "      <td>-0.002828</td>\n",
       "      <td>0.894589</td>\n",
       "      <td>-0.004286</td>\n",
       "      <td>0.970750</td>\n",
       "      <td>0.017619</td>\n",
       "      <td>0.173759</td>\n",
       "      <td>0.412965</td>\n",
       "      <td>0.339539</td>\n",
       "      <td>0.509549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2 - title+abstra...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>208.0</td>\n",
       "      <td>7934.0</td>\n",
       "      <td>1223.363636</td>\n",
       "      <td>366.0</td>\n",
       "      <td>0.760200</td>\n",
       "      <td>0.023636</td>\n",
       "      <td>0.882635</td>\n",
       "      <td>0.032727</td>\n",
       "      <td>...</td>\n",
       "      <td>0.945724</td>\n",
       "      <td>0.023030</td>\n",
       "      <td>0.882122</td>\n",
       "      <td>-0.013312</td>\n",
       "      <td>0.964446</td>\n",
       "      <td>0.023247</td>\n",
       "      <td>0.171099</td>\n",
       "      <td>0.367774</td>\n",
       "      <td>0.317376</td>\n",
       "      <td>0.491789</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                config  num_clusters  \\\n",
       "0    2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "1    2019 - paraphrase-mpnet-base-v2 - title+abstra...          15.0   \n",
       "2    2019 - paraphrase-mpnet-base-v2 - title+abstra...          20.0   \n",
       "3    2019 - paraphrase-mpnet-base-v2 - title+abstra...          25.0   \n",
       "4    2019 - paraphrase-mpnet-base-v2 - title+abstra...          30.0   \n",
       "5    2019 - paraphrase-mpnet-base-v2 - title+abstra...          35.0   \n",
       "6    2019 - paraphrase-mpnet-base-v2 - title+abstra...          40.0   \n",
       "7    2019 - paraphrase-mpnet-base-v2 - title+abstra...          45.0   \n",
       "8    2019 - paraphrase-mpnet-base-v2 - title+abstra...          50.0   \n",
       "14   2019 - paraphrase-mpnet-base-v2 - title+abstra...          15.0   \n",
       "16   2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "17   2019 - paraphrase-mpnet-base-v2 - title+abstra...          15.0   \n",
       "18   2019 - paraphrase-mpnet-base-v2 - title+abstra...          20.0   \n",
       "19   2019 - paraphrase-mpnet-base-v2 - title+abstra...          25.0   \n",
       "20   2019 - paraphrase-mpnet-base-v2 - title+abstra...          30.0   \n",
       "21   2019 - paraphrase-mpnet-base-v2 - title+abstra...          35.0   \n",
       "22   2019 - paraphrase-mpnet-base-v2 - title+abstra...          40.0   \n",
       "23   2019 - paraphrase-mpnet-base-v2 - title+abstra...          45.0   \n",
       "24   2019 - paraphrase-mpnet-base-v2 - title+abstra...          50.0   \n",
       "27   2019 - paraphrase-mpnet-base-v2 - title+abstra...          11.0   \n",
       "122  2019 - paraphrase-mpnet-base-v2 - title+abstra...          16.0   \n",
       "134  2019 - paraphrase-mpnet-base-v2 - title+abstra...          16.0   \n",
       "159  2019 - paraphrase-mpnet-base-v2 - title+abstra...          13.0   \n",
       "289  2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "293  2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "296  2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "301  2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "303  2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "314  2019 - paraphrase-mpnet-base-v2 - title+abstra...          10.0   \n",
       "337  2019 - paraphrase-mpnet-base-v2 - title+abstra...          11.0   \n",
       "\n",
       "     min_members  max_members  mean_members  median_members  \\\n",
       "0          231.0       1756.0   1345.700000          1495.0   \n",
       "1          225.0       1333.0    897.133333           959.0   \n",
       "2          159.0       1083.0    672.850000           729.0   \n",
       "3          149.0        973.0    538.280000           563.0   \n",
       "4          148.0        769.0    448.566667           407.5   \n",
       "5          147.0        700.0    384.485714           367.0   \n",
       "6           44.0        602.0    336.425000           341.0   \n",
       "7           89.0        518.0    299.044444           307.0   \n",
       "8           69.0        493.0    269.140000           273.5   \n",
       "14           1.0      13124.0    897.133333             3.0   \n",
       "16           1.0      13150.0   1345.700000             3.0   \n",
       "17           1.0      13124.0    897.133333             3.0   \n",
       "18           1.0      13113.0    672.850000             3.0   \n",
       "19           1.0      13081.0    538.280000             3.0   \n",
       "20           1.0      13048.0    448.566667             3.0   \n",
       "21           1.0      13040.0    384.485714             3.0   \n",
       "22           1.0      12982.0    336.425000             2.5   \n",
       "23           1.0      12931.0    299.044444             3.0   \n",
       "24           1.0      12887.0    269.140000             3.0   \n",
       "27         107.0      10951.0   1223.363636           213.0   \n",
       "122        127.0       6619.0    841.062500           354.5   \n",
       "134        128.0       7044.0    841.062500           361.5   \n",
       "159        191.0       6232.0   1035.153846           553.0   \n",
       "289        259.0       7671.0   1345.700000           672.0   \n",
       "293        322.0       7572.0   1345.700000           653.0   \n",
       "296        303.0       7772.0   1345.700000           610.0   \n",
       "301        220.0       7499.0   1345.700000           627.5   \n",
       "303        227.0       7612.0   1345.700000           622.0   \n",
       "314        212.0       7318.0   1345.700000           658.0   \n",
       "337        208.0       7934.0   1223.363636           366.0   \n",
       "\n",
       "     intra_similarity_sem_scholar_5  inter_similarity_sem_scholar_5  \\\n",
       "0                          0.671405                        0.015556   \n",
       "1                          0.701527                       -0.017143   \n",
       "2                          0.720542                        0.123684   \n",
       "3                          0.736495                        0.051000   \n",
       "4                          0.754377                        0.001839   \n",
       "5                          0.749197                        0.037479   \n",
       "6                          0.751642                        0.008333   \n",
       "7                          0.753758                        0.018990   \n",
       "8                          0.765297                        0.011020   \n",
       "14                         0.828453                       -0.010476   \n",
       "16                         0.837679                        0.091111   \n",
       "17                         0.828453                       -0.010476   \n",
       "18                         0.823173                       -0.018421   \n",
       "19                         0.818889                       -0.009000   \n",
       "20                         0.812563                       -0.017701   \n",
       "21                         0.834102                        0.009076   \n",
       "22                         0.845266                             NaN   \n",
       "23                         0.833814                             NaN   \n",
       "24                         0.818313                             NaN   \n",
       "27                         0.709176                       -0.010909   \n",
       "122                        0.768704                        0.054167   \n",
       "134                        0.771966                        0.055833   \n",
       "159                        0.734251                       -0.047436   \n",
       "289                        0.767794                        0.017778   \n",
       "293                        0.768656                        0.031111   \n",
       "296                        0.771398                        0.044444   \n",
       "301                        0.773557                        0.006667   \n",
       "303                        0.772661                        0.006667   \n",
       "314                        0.773099                       -0.048889   \n",
       "337                        0.760200                        0.023636   \n",
       "\n",
       "     intra_similarity_cso_enhanced_5  inter_similarity_cso_enhanced_5  ...  \\\n",
       "0                           0.835039                         0.131111  ...   \n",
       "1                           0.856118                         0.038095  ...   \n",
       "2                           0.870013                         0.110526  ...   \n",
       "3                           0.878603                         0.050000  ...   \n",
       "4                           0.878668                         0.028276  ...   \n",
       "5                           0.874138                         0.042521  ...   \n",
       "6                           0.869797                         0.018333  ...   \n",
       "7                           0.866670                         0.052525  ...   \n",
       "8                           0.869220                         0.027510  ...   \n",
       "14                          0.822870                         0.017143  ...   \n",
       "16                          0.859256                         0.015556  ...   \n",
       "17                          0.822870                         0.017143  ...   \n",
       "18                          0.809048                         0.028421  ...   \n",
       "19                          0.837097                         0.008667  ...   \n",
       "20                          0.843150                              NaN  ...   \n",
       "21                          0.855642                              NaN  ...   \n",
       "22                          0.870880                              NaN  ...   \n",
       "23                          0.866370                              NaN  ...   \n",
       "24                          0.875203                              NaN  ...   \n",
       "27                          0.875340                         0.069091  ...   \n",
       "122                         0.886962                         0.135000  ...   \n",
       "134                         0.887615                         0.087500  ...   \n",
       "159                         0.855391                         0.030769  ...   \n",
       "289                         0.885364                         0.084444  ...   \n",
       "293                         0.887046                         0.062222  ...   \n",
       "296                         0.895215                         0.057778  ...   \n",
       "301                         0.894521                         0.068889  ...   \n",
       "303                         0.892122                         0.051111  ...   \n",
       "314                         0.893982                         0.015556  ...   \n",
       "337                         0.882635                         0.032727  ...   \n",
       "\n",
       "     intra_similarity_cso_enhanced_10  inter_similarity_cso_enhanced_10  \\\n",
       "0                            0.916281                          0.064512   \n",
       "1                            0.922405                          0.012063   \n",
       "2                            0.926564                          0.046252   \n",
       "3                            0.932463                          0.057172   \n",
       "4                            0.938208                         -0.002382   \n",
       "5                            0.935251                          0.013822   \n",
       "6                            0.931910                          0.023792   \n",
       "7                            0.931871                          0.021022   \n",
       "8                            0.936448                          0.009395   \n",
       "14                           0.908463                          0.029683   \n",
       "16                           0.929335                          0.000274   \n",
       "17                           0.908463                          0.029683   \n",
       "18                           0.885783                          0.024830   \n",
       "19                           0.898473                          0.001556   \n",
       "20                           0.904893                               NaN   \n",
       "21                           0.937133                               NaN   \n",
       "22                           0.947815                               NaN   \n",
       "23                           0.942909                               NaN   \n",
       "24                           0.944088                               NaN   \n",
       "27                           0.937584                          0.028099   \n",
       "122                          0.944986                          0.028384   \n",
       "134                          0.948332                          0.014545   \n",
       "159                          0.927664                         -0.010878   \n",
       "289                          0.949692                          0.013064   \n",
       "293                          0.949076                         -0.007677   \n",
       "296                          0.952527                         -0.011717   \n",
       "301                          0.951447                          0.028148   \n",
       "303                          0.951950                          0.013333   \n",
       "314                          0.954621                         -0.002828   \n",
       "337                          0.945724                          0.023030   \n",
       "\n",
       "     intra_similarity_sem_scholar_15  inter_similarity_sem_scholar_15  \\\n",
       "0                           0.842505                        -0.011508   \n",
       "1                           0.859994                         0.003946   \n",
       "2                           0.870194                        -0.000526   \n",
       "3                           0.880146                         0.006845   \n",
       "4                           0.891048                        -0.001314   \n",
       "5                           0.888284                         0.002953   \n",
       "6                           0.889370                        -0.002445   \n",
       "7                           0.887684                        -0.000263   \n",
       "8                           0.891706                         0.000945   \n",
       "14                          0.930672                        -0.002843   \n",
       "16                          0.916010                         0.068653   \n",
       "17                          0.930672                        -0.002843   \n",
       "18                          0.924758                         0.006227   \n",
       "19                          0.925220                         0.003504   \n",
       "20                          0.913753                         0.001561   \n",
       "21                          0.923378                         0.020800   \n",
       "22                          0.920775                              NaN   \n",
       "23                          0.919577                              NaN   \n",
       "24                          0.910090                              NaN   \n",
       "27                          0.864905                         0.011104   \n",
       "122                         0.893540                         0.000387   \n",
       "134                         0.897202                        -0.016250   \n",
       "159                         0.876212                        -0.017674   \n",
       "289                         0.891887                        -0.024206   \n",
       "293                         0.890820                         0.004206   \n",
       "296                         0.893062                        -0.006032   \n",
       "301                         0.892259                         0.000952   \n",
       "303                         0.894630                        -0.023651   \n",
       "314                         0.894589                        -0.004286   \n",
       "337                         0.882122                        -0.013312   \n",
       "\n",
       "     intra_similarity_cso_enhanced_15  inter_similarity_cso_enhanced_15  \\\n",
       "0                            0.942581                          0.057143   \n",
       "1                            0.949055                         -0.002483   \n",
       "2                            0.952981                          0.022218   \n",
       "3                            0.958013                          0.032905   \n",
       "4                            0.960402                          0.002397   \n",
       "5                            0.957690                          0.005696   \n",
       "6                            0.959232                          0.007285   \n",
       "7                            0.955677                          0.023921   \n",
       "8                            0.960146                          0.004157   \n",
       "14                           0.958558                          0.024591   \n",
       "16                           0.937816                          0.011592   \n",
       "17                           0.958558                          0.024591   \n",
       "18                           0.960879                          0.022852   \n",
       "19                           0.968710                          0.022650   \n",
       "20                           0.972061                               NaN   \n",
       "21                           0.977246                               NaN   \n",
       "22                           0.986045                               NaN   \n",
       "23                           0.979671                               NaN   \n",
       "24                           0.977268                               NaN   \n",
       "27                           0.955221                          0.016104   \n",
       "122                          0.961280                          0.017113   \n",
       "134                          0.964992                         -0.008214   \n",
       "159                          0.951381                         -0.013965   \n",
       "289                          0.964024                          0.028968   \n",
       "293                          0.965510                         -0.012937   \n",
       "296                          0.967831                         -0.020397   \n",
       "301                          0.969262                          0.015873   \n",
       "303                          0.969647                         -0.007619   \n",
       "314                          0.970750                          0.017619   \n",
       "337                          0.964446                          0.023247   \n",
       "\n",
       "     accuracy_new_papers_sem_scholar  ranking_score_new_papers_sem_scholar  \\\n",
       "0                           0.185727                              0.364594   \n",
       "1                           0.167553                              0.318922   \n",
       "2                           0.165337                              0.291825   \n",
       "3                           0.154699                              0.275315   \n",
       "4                           0.137855                              0.250003   \n",
       "5                           0.144947                              0.242658   \n",
       "6                           0.126330                              0.216895   \n",
       "7                           0.120567                              0.209862   \n",
       "8                           0.122340                              0.208268   \n",
       "14                          0.062943                              0.460138   \n",
       "16                          0.132092                              0.544666   \n",
       "17                          0.062943                              0.460138   \n",
       "18                          0.054078                              0.344838   \n",
       "19                          0.040780                              0.313575   \n",
       "20                          0.037677                              0.300831   \n",
       "21                          0.037677                              0.295618   \n",
       "22                          0.031028                              0.207242   \n",
       "23                          0.021720                              0.195073   \n",
       "24                          0.017730                              0.184983   \n",
       "27                          0.138298                              0.348177   \n",
       "122                         0.141844                              0.303639   \n",
       "134                         0.141844                              0.301414   \n",
       "159                         0.152482                              0.320867   \n",
       "289                         0.170213                              0.412949   \n",
       "293                         0.169770                              0.411343   \n",
       "296                         0.173759                              0.414313   \n",
       "301                         0.172872                              0.419347   \n",
       "303                         0.171543                              0.418878   \n",
       "314                         0.173759                              0.412965   \n",
       "337                         0.171099                              0.367774   \n",
       "\n",
       "     accuracy_new_papers_cso_enhanced  ranking_score_new_papers_cso_enhanced  \n",
       "0                            0.323582                               0.532052  \n",
       "1                            0.288564                               0.478497  \n",
       "2                            0.296543                               0.458566  \n",
       "3                            0.288564                               0.447454  \n",
       "4                            0.274823                               0.425155  \n",
       "5                            0.280585                               0.422815  \n",
       "6                            0.261968                               0.399544  \n",
       "7                            0.252216                               0.386457  \n",
       "8                            0.241135                               0.374593  \n",
       "14                           0.009309                               0.234509  \n",
       "16                           0.028812                               0.307789  \n",
       "17                           0.009309                               0.234509  \n",
       "18                           0.002660                               0.214347  \n",
       "19                           0.000887                               0.164979  \n",
       "20                           0.000887                               0.140675  \n",
       "21                           0.000000                               0.122359  \n",
       "22                           0.003989                               0.114882  \n",
       "23                           0.003989                               0.106518  \n",
       "24                           0.005762                               0.099151  \n",
       "27                           0.277926                               0.470488  \n",
       "122                          0.335550                               0.488527  \n",
       "134                          0.328457                               0.481557  \n",
       "159                          0.348848                               0.513929  \n",
       "289                          0.334663                               0.507474  \n",
       "293                          0.332890                               0.508297  \n",
       "296                          0.338209                               0.509125  \n",
       "301                          0.339539                               0.507377  \n",
       "303                          0.332890                               0.504011  \n",
       "314                          0.339539                               0.509549  \n",
       "337                          0.317376                               0.491789  \n",
       "\n",
       "[30 rows x 22 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Number of remaining clusterings:\", len(df_ce_filtered))\n",
    "for c in list(df_ce_filtered[\"config\"]):\n",
    "    print(c)\n",
    "df_ce_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4e2188de",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_configs = []\n",
    "\n",
    "for i, row in df_ce_filtered.iterrows():\n",
    "    s = row[\"config\"].split(\" - \")\n",
    "    best_configs.append({\"algorithm\": s[3], \"num_clusters\": s[4], \"distance_threshold\": s[5], \"min_cluster_size\": s[6], \"neighbors\": s[7], \"components\": s[8]})\n",
    "\n",
    "# Store best configs\n",
    "with open(\"data/clusters/paraphrase-mpnet-base-v2_titles_abstracts_2019_best_configs.json\", \"w\") as jf:\n",
    "    json.dump(best_configs, jf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2aa9c6dc",
   "metadata": {},
   "source": [
    "## 4. Use list of appropriate configurations to cluster all embedding variants \n",
    "- See notebook clustering.ipynb\n",
    "- Results in \"data/clusters/all_embeddings_all_text_sets_best_configs_2019.json\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "539d9e90",
   "metadata": {},
   "source": [
    "## 5. Check with semantic scholar topics and cso topics if chosen clusters are appropriate\n",
    "- All clustering algorithms and appropriate configurations are tested for all embeddings\n",
    "- Store results of all metrics in a dataframe\n",
    "- Find best ~3 configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ec5ebf76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 15 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 20 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 25 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 30 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 35 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 40 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 45 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - kmeans - 50 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - null - 0.7 - null - null - null\n",
      "dict_keys(['0'])\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 15 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 20 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 25 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 30 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 35 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 40 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 45 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - agglomerative - 50 - null - null - null - null\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 100 - 2 - 10\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 100 - 200 - 40\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 100 - 400 - 30\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 150 - 5 - 20\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 200 - 10 - 20\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 200 - 10 - 80\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 200 - 10 - 384\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 200 - 20 - 15\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 200 - 20 - 30\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 200 - 50 - 15\n",
      "2019 - allenai-specter_titles.pkl - allenai-specter - title - topic - null - null - 200 - 100 - 2\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 15 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 20 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 25 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 30 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 35 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 40 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 45 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 50 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - null - 0.7 - null - null - null\n",
      "dict_keys(['0'])\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 15 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 20 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 25 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 30 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 35 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 40 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 45 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - agglomerative - 50 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 100 - 2 - 10\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 100 - 200 - 40\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 150 - 5 - 20\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 200 - 10 - 20\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 200 - 10 - 80\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 200 - 10 - 384\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 200 - 20 - 15\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 200 - 20 - 30\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 200 - 50 - 15\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - topic - null - null - 200 - 100 - 2\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 15 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 20 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 25 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 30 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 35 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 40 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 45 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - kmeans - 50 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - null - 0.7 - null - null - null\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['0'])\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 15 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 20 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 25 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 30 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 35 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 40 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 45 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - agglomerative - 50 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 100 - 2 - 10\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 100 - 200 - 40\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 100 - 400 - 30\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 150 - 5 - 20\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 200 - 10 - 20\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 200 - 10 - 80\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 200 - 10 - 384\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 200 - 20 - 15\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 200 - 20 - 30\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 200 - 50 - 15\n",
      "2019 - allenai-specter_titles_abstracts_sent_mean.pkl - allenai-specter - title+abstract_sent_mean - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 45 - null - null - null - null\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 20 - 15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-distilroberta-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 50 - 15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts_sent_mean.pkl - paraphrase-mpnet-base-v2 - title+abstract_sent_mean - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 200 - 20 - 15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 200 - 100 - 2\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 15 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 25 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 30 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 35 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - null - 0.7 - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 10 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 15 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 20 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 25 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 30 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 35 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 40 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 45 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - agglomerative - 50 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 100 - 2 - 10\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 100 - 200 - 40\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 100 - 400 - 30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 150 - 5 - 20\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 20\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 200 - 10 - 384\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 200 - 20 - 15\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 200 - 50 - 15\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts_sent_mean.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract_sent_mean - topic - null - null - 200 - 100 - 2\n"
     ]
    }
   ],
   "source": [
    "config2data = dict()       # (cluster2indices, X_train, y_train, X_predict, y_predicted)\n",
    "\n",
    "# Load results\n",
    "with open(\"data/clusters/all_embeddings_all_text_sets_best_configs_2019.json\") as jf:\n",
    "    config2clusters_2019 = json.load(jf)\n",
    "    \n",
    "for pretrained_model in config2clusters_2019:\n",
    "    for text_set in config2clusters_2019[pretrained_model]:\n",
    "        for filename in os.listdir(\"data/embeddings/\"):\n",
    "            if pretrained_model in filename:\n",
    "                if text_set == \"title\" and \"titles.pkl\" in filename:\n",
    "                    texts, embeddings = util.load_embeddings(filename)\n",
    "                    break\n",
    "                elif text_set == \"title+abstract\" and \"titles_abstracts.pkl\" in filename:\n",
    "                    texts, embeddings = util.load_embeddings(filename)\n",
    "                    break\n",
    "                elif text_set == \"title+abstract_sent_mean\" and \"titles_abstracts_sent_mean.pkl\" in filename:\n",
    "                    texts, embeddings = util.load_embeddings(filename)\n",
    "                    break\n",
    "        \n",
    "        for config in config2clusters_2019[pretrained_model][text_set]:\n",
    "            algorithm = config[\"algorithm\"]\n",
    "            num_clusters = config[\"num_clusters\"]\n",
    "            threshold = config[\"distance_threshold\"]\n",
    "            min_cluster_size = config[\"min_cluster_size\"]\n",
    "            num_neighbors = config[\"neighbors\"]\n",
    "            components = config[\"components\"]\n",
    "            key = str(last_year) + \" - \" + filename + \" - \" + pretrained_model + \" - \" + text_set + \" - \" + algorithm + \" - \" + str(num_clusters) + \" - \" + str(threshold) + \" - \" + str(min_cluster_size) + \" - \" + str(num_neighbors) + \" - \" + str(components)\n",
    "            print(key)\n",
    "\n",
    "            cluster2indices = config[\"cluster2indices\"]\n",
    "\n",
    "            try:\n",
    "                X_train, y_train, X_predict = util.get_classifier_data_sets_2(cluster2indices, df, 2019, embeddings, \n",
    "                    pretrained_model, text_set, algorithm, num_clusters, threshold, min_cluster_size, num_neighbors, components)\n",
    "            except:\n",
    "                print(\"Error:\", key)\n",
    "\n",
    "            try:\n",
    "                clf.fit(X_train, y_train)\n",
    "                y_predicted = clf.predict(X_predict)\n",
    "\n",
    "                config2data[key] = (cluster2indices, X_train, y_train, X_predict, y_predicted)\n",
    "            except:\n",
    "                print(cluster2indices.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4b9f13ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>config</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>357 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                config\n",
       "0    2019 - allenai-specter_titles.pkl - allenai-sp...\n",
       "1    2019 - allenai-specter_titles.pkl - allenai-sp...\n",
       "2    2019 - allenai-specter_titles.pkl - allenai-sp...\n",
       "3    2019 - allenai-specter_titles.pkl - allenai-sp...\n",
       "4    2019 - allenai-specter_titles.pkl - allenai-sp...\n",
       "..                                                 ...\n",
       "352  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...\n",
       "353  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...\n",
       "354  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...\n",
       "355  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...\n",
       "356  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...\n",
       "\n",
       "[357 rows x 1 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create dataframe for _c_lustering _e_valuation with column 'config'\n",
    "df_ce = pd.DataFrame([[c] for c in config2data], columns=[\"config\"])\n",
    "df_ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c999206a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/357 configurations\n",
      "2/357 configurations\n",
      "3/357 configurations\n",
      "4/357 configurations\n",
      "5/357 configurations\n",
      "6/357 configurations\n",
      "7/357 configurations\n",
      "8/357 configurations\n",
      "9/357 configurations\n",
      "10/357 configurations\n",
      "11/357 configurations\n",
      "12/357 configurations\n",
      "13/357 configurations\n",
      "14/357 configurations\n",
      "15/357 configurations\n",
      "16/357 configurations\n",
      "17/357 configurations\n",
      "18/357 configurations\n",
      "19/357 configurations\n",
      "20/357 configurations\n",
      "21/357 configurations\n",
      "22/357 configurations\n",
      "23/357 configurations\n",
      "24/357 configurations\n",
      "25/357 configurations\n",
      "26/357 configurations\n",
      "27/357 configurations\n",
      "28/357 configurations\n",
      "29/357 configurations\n",
      "30/357 configurations\n",
      "31/357 configurations\n",
      "32/357 configurations\n",
      "33/357 configurations\n",
      "34/357 configurations\n",
      "35/357 configurations\n",
      "36/357 configurations\n",
      "37/357 configurations\n",
      "38/357 configurations\n",
      "39/357 configurations\n",
      "40/357 configurations\n",
      "41/357 configurations\n",
      "42/357 configurations\n",
      "43/357 configurations\n",
      "44/357 configurations\n",
      "45/357 configurations\n",
      "46/357 configurations\n",
      "47/357 configurations\n",
      "48/357 configurations\n",
      "49/357 configurations\n",
      "50/357 configurations\n",
      "51/357 configurations\n",
      "52/357 configurations\n",
      "53/357 configurations\n",
      "54/357 configurations\n",
      "55/357 configurations\n",
      "56/357 configurations\n",
      "57/357 configurations\n",
      "58/357 configurations\n",
      "59/357 configurations\n",
      "60/357 configurations\n",
      "61/357 configurations\n",
      "62/357 configurations\n",
      "63/357 configurations\n",
      "64/357 configurations\n",
      "65/357 configurations\n",
      "66/357 configurations\n",
      "67/357 configurations\n",
      "68/357 configurations\n",
      "69/357 configurations\n",
      "70/357 configurations\n",
      "71/357 configurations\n",
      "72/357 configurations\n",
      "73/357 configurations\n",
      "74/357 configurations\n",
      "75/357 configurations\n",
      "76/357 configurations\n",
      "77/357 configurations\n",
      "78/357 configurations\n",
      "79/357 configurations\n",
      "80/357 configurations\n",
      "81/357 configurations\n",
      "82/357 configurations\n",
      "83/357 configurations\n",
      "84/357 configurations\n",
      "85/357 configurations\n",
      "86/357 configurations\n",
      "87/357 configurations\n",
      "88/357 configurations\n",
      "89/357 configurations\n",
      "90/357 configurations\n",
      "91/357 configurations\n",
      "92/357 configurations\n",
      "93/357 configurations\n",
      "94/357 configurations\n",
      "95/357 configurations\n",
      "96/357 configurations\n",
      "97/357 configurations\n",
      "98/357 configurations\n",
      "99/357 configurations\n",
      "100/357 configurations\n",
      "101/357 configurations\n",
      "102/357 configurations\n",
      "103/357 configurations\n",
      "104/357 configurations\n",
      "105/357 configurations\n",
      "106/357 configurations\n",
      "107/357 configurations\n",
      "108/357 configurations\n",
      "109/357 configurations\n",
      "110/357 configurations\n",
      "111/357 configurations\n",
      "112/357 configurations\n",
      "113/357 configurations\n",
      "114/357 configurations\n",
      "115/357 configurations\n",
      "116/357 configurations\n",
      "117/357 configurations\n",
      "118/357 configurations\n",
      "119/357 configurations\n",
      "120/357 configurations\n",
      "121/357 configurations\n",
      "122/357 configurations\n",
      "123/357 configurations\n",
      "124/357 configurations\n",
      "125/357 configurations\n",
      "126/357 configurations\n",
      "127/357 configurations\n",
      "128/357 configurations\n",
      "129/357 configurations\n",
      "130/357 configurations\n",
      "131/357 configurations\n",
      "132/357 configurations\n",
      "133/357 configurations\n",
      "134/357 configurations\n",
      "135/357 configurations\n",
      "136/357 configurations\n",
      "137/357 configurations\n",
      "138/357 configurations\n",
      "139/357 configurations\n",
      "140/357 configurations\n",
      "141/357 configurations\n",
      "142/357 configurations\n",
      "143/357 configurations\n",
      "144/357 configurations\n",
      "145/357 configurations\n",
      "146/357 configurations\n",
      "147/357 configurations\n",
      "148/357 configurations\n",
      "149/357 configurations\n",
      "150/357 configurations\n",
      "151/357 configurations\n",
      "152/357 configurations\n",
      "153/357 configurations\n",
      "154/357 configurations\n",
      "155/357 configurations\n",
      "156/357 configurations\n",
      "157/357 configurations\n",
      "158/357 configurations\n",
      "159/357 configurations\n",
      "160/357 configurations\n",
      "161/357 configurations\n",
      "162/357 configurations\n",
      "163/357 configurations\n",
      "164/357 configurations\n",
      "165/357 configurations\n",
      "166/357 configurations\n",
      "167/357 configurations\n",
      "168/357 configurations\n",
      "169/357 configurations\n",
      "170/357 configurations\n",
      "171/357 configurations\n",
      "172/357 configurations\n",
      "173/357 configurations\n",
      "174/357 configurations\n",
      "175/357 configurations\n",
      "176/357 configurations\n",
      "177/357 configurations\n",
      "178/357 configurations\n",
      "179/357 configurations\n",
      "180/357 configurations\n",
      "181/357 configurations\n",
      "182/357 configurations\n",
      "183/357 configurations\n",
      "184/357 configurations\n",
      "185/357 configurations\n",
      "186/357 configurations\n",
      "187/357 configurations\n",
      "188/357 configurations\n",
      "189/357 configurations\n",
      "190/357 configurations\n",
      "191/357 configurations\n",
      "192/357 configurations\n",
      "193/357 configurations\n",
      "194/357 configurations\n",
      "195/357 configurations\n",
      "196/357 configurations\n",
      "197/357 configurations\n",
      "198/357 configurations\n",
      "199/357 configurations\n",
      "200/357 configurations\n",
      "201/357 configurations\n",
      "202/357 configurations\n",
      "203/357 configurations\n",
      "204/357 configurations\n",
      "205/357 configurations\n",
      "206/357 configurations\n",
      "207/357 configurations\n",
      "208/357 configurations\n",
      "209/357 configurations\n",
      "210/357 configurations\n",
      "211/357 configurations\n",
      "212/357 configurations\n",
      "213/357 configurations\n",
      "214/357 configurations\n",
      "215/357 configurations\n",
      "216/357 configurations\n",
      "217/357 configurations\n",
      "218/357 configurations\n",
      "219/357 configurations\n",
      "220/357 configurations\n",
      "221/357 configurations\n",
      "222/357 configurations\n",
      "223/357 configurations\n",
      "224/357 configurations\n",
      "225/357 configurations\n",
      "226/357 configurations\n",
      "227/357 configurations\n",
      "228/357 configurations\n",
      "229/357 configurations\n",
      "230/357 configurations\n",
      "231/357 configurations\n",
      "232/357 configurations\n",
      "233/357 configurations\n",
      "234/357 configurations\n",
      "235/357 configurations\n",
      "236/357 configurations\n",
      "237/357 configurations\n",
      "238/357 configurations\n",
      "239/357 configurations\n",
      "240/357 configurations\n",
      "241/357 configurations\n",
      "242/357 configurations\n",
      "243/357 configurations\n",
      "244/357 configurations\n",
      "245/357 configurations\n",
      "246/357 configurations\n",
      "247/357 configurations\n",
      "248/357 configurations\n",
      "249/357 configurations\n",
      "250/357 configurations\n",
      "251/357 configurations\n",
      "252/357 configurations\n",
      "253/357 configurations\n",
      "254/357 configurations\n",
      "255/357 configurations\n",
      "256/357 configurations\n",
      "257/357 configurations\n",
      "258/357 configurations\n",
      "259/357 configurations\n",
      "260/357 configurations\n",
      "261/357 configurations\n",
      "262/357 configurations\n",
      "263/357 configurations\n",
      "264/357 configurations\n",
      "265/357 configurations\n",
      "266/357 configurations\n",
      "267/357 configurations\n",
      "268/357 configurations\n",
      "269/357 configurations\n",
      "270/357 configurations\n",
      "271/357 configurations\n",
      "272/357 configurations\n",
      "273/357 configurations\n",
      "274/357 configurations\n",
      "275/357 configurations\n",
      "276/357 configurations\n",
      "277/357 configurations\n",
      "278/357 configurations\n",
      "279/357 configurations\n",
      "280/357 configurations\n",
      "281/357 configurations\n",
      "282/357 configurations\n",
      "283/357 configurations\n",
      "284/357 configurations\n",
      "285/357 configurations\n",
      "286/357 configurations\n",
      "287/357 configurations\n",
      "288/357 configurations\n",
      "289/357 configurations\n",
      "290/357 configurations\n",
      "291/357 configurations\n",
      "292/357 configurations\n",
      "293/357 configurations\n",
      "294/357 configurations\n",
      "295/357 configurations\n",
      "296/357 configurations\n",
      "297/357 configurations\n",
      "298/357 configurations\n",
      "299/357 configurations\n",
      "300/357 configurations\n",
      "301/357 configurations\n",
      "302/357 configurations\n",
      "303/357 configurations\n",
      "304/357 configurations\n",
      "305/357 configurations\n",
      "306/357 configurations\n",
      "307/357 configurations\n",
      "308/357 configurations\n",
      "309/357 configurations\n",
      "310/357 configurations\n",
      "311/357 configurations\n",
      "312/357 configurations\n",
      "313/357 configurations\n",
      "314/357 configurations\n",
      "315/357 configurations\n",
      "316/357 configurations\n",
      "317/357 configurations\n",
      "318/357 configurations\n",
      "319/357 configurations\n",
      "320/357 configurations\n",
      "321/357 configurations\n",
      "322/357 configurations\n",
      "323/357 configurations\n",
      "324/357 configurations\n",
      "325/357 configurations\n",
      "326/357 configurations\n",
      "327/357 configurations\n",
      "328/357 configurations\n",
      "329/357 configurations\n",
      "330/357 configurations\n",
      "331/357 configurations\n",
      "332/357 configurations\n",
      "333/357 configurations\n",
      "334/357 configurations\n",
      "335/357 configurations\n",
      "336/357 configurations\n",
      "337/357 configurations\n",
      "338/357 configurations\n",
      "339/357 configurations\n",
      "340/357 configurations\n",
      "341/357 configurations\n",
      "342/357 configurations\n",
      "343/357 configurations\n",
      "344/357 configurations\n",
      "345/357 configurations\n",
      "346/357 configurations\n",
      "347/357 configurations\n",
      "348/357 configurations\n",
      "349/357 configurations\n",
      "350/357 configurations\n",
      "351/357 configurations\n",
      "352/357 configurations\n",
      "353/357 configurations\n",
      "354/357 configurations\n",
      "355/357 configurations\n",
      "356/357 configurations\n",
      "357/357 configurations\n"
     ]
    }
   ],
   "source": [
    "# Evaluate metrics \n",
    "df_ce = evaluate_metrics(df_ce, config2data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a3d465c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "357\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>config</th>\n",
       "      <th>num_clusters</th>\n",
       "      <th>min_members</th>\n",
       "      <th>max_members</th>\n",
       "      <th>mean_members</th>\n",
       "      <th>median_members</th>\n",
       "      <th>intra_similarity_sem_scholar_5</th>\n",
       "      <th>inter_similarity_sem_scholar_5</th>\n",
       "      <th>intra_similarity_cso_enhanced_5</th>\n",
       "      <th>inter_similarity_cso_enhanced_5</th>\n",
       "      <th>...</th>\n",
       "      <th>inter_similarity_sem_scholar_15</th>\n",
       "      <th>intra_similarity_cso_enhanced_15</th>\n",
       "      <th>inter_similarity_cso_enhanced_15</th>\n",
       "      <th>accuracy_new_papers_sem_scholar</th>\n",
       "      <th>ranking_score_new_papers_sem_scholar</th>\n",
       "      <th>absolute_score_new_papers_sem_scholar</th>\n",
       "      <th>accuracy_new_papers_cso_enhanced</th>\n",
       "      <th>ranking_score_new_papers_cso_enhanced</th>\n",
       "      <th>absolute_score_new_papers_cso_enhanced</th>\n",
       "      <th>papers_clustered</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>834.0</td>\n",
       "      <td>1775.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>1392.0</td>\n",
       "      <td>0.685426</td>\n",
       "      <td>9.555556e-02</td>\n",
       "      <td>0.837812</td>\n",
       "      <td>0.028889</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014127</td>\n",
       "      <td>0.943590</td>\n",
       "      <td>0.029206</td>\n",
       "      <td>0.204344</td>\n",
       "      <td>0.384592</td>\n",
       "      <td>0.131923</td>\n",
       "      <td>0.280142</td>\n",
       "      <td>0.479073</td>\n",
       "      <td>1.196837</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>408.0</td>\n",
       "      <td>1551.0</td>\n",
       "      <td>897.133333</td>\n",
       "      <td>883.0</td>\n",
       "      <td>0.703512</td>\n",
       "      <td>2.952381e-02</td>\n",
       "      <td>0.853639</td>\n",
       "      <td>0.087619</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.020714</td>\n",
       "      <td>0.953988</td>\n",
       "      <td>0.017959</td>\n",
       "      <td>0.162677</td>\n",
       "      <td>0.325285</td>\n",
       "      <td>0.153040</td>\n",
       "      <td>0.279699</td>\n",
       "      <td>0.452538</td>\n",
       "      <td>1.362130</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>337.0</td>\n",
       "      <td>1102.0</td>\n",
       "      <td>672.850000</td>\n",
       "      <td>703.5</td>\n",
       "      <td>0.709529</td>\n",
       "      <td>9.578947e-02</td>\n",
       "      <td>0.855512</td>\n",
       "      <td>0.032632</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001241</td>\n",
       "      <td>0.952921</td>\n",
       "      <td>0.033684</td>\n",
       "      <td>0.137855</td>\n",
       "      <td>0.276878</td>\n",
       "      <td>0.159520</td>\n",
       "      <td>0.253103</td>\n",
       "      <td>0.408462</td>\n",
       "      <td>1.384199</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>856.0</td>\n",
       "      <td>538.280000</td>\n",
       "      <td>525.0</td>\n",
       "      <td>0.721358</td>\n",
       "      <td>1.866667e-01</td>\n",
       "      <td>0.861880</td>\n",
       "      <td>0.051333</td>\n",
       "      <td>...</td>\n",
       "      <td>0.027940</td>\n",
       "      <td>0.956241</td>\n",
       "      <td>0.046917</td>\n",
       "      <td>0.160461</td>\n",
       "      <td>0.271853</td>\n",
       "      <td>0.168357</td>\n",
       "      <td>0.241135</td>\n",
       "      <td>0.388858</td>\n",
       "      <td>1.410478</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>30.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>758.0</td>\n",
       "      <td>448.566667</td>\n",
       "      <td>448.0</td>\n",
       "      <td>0.723910</td>\n",
       "      <td>8.804598e-02</td>\n",
       "      <td>0.864502</td>\n",
       "      <td>0.074713</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033547</td>\n",
       "      <td>0.956618</td>\n",
       "      <td>0.033892</td>\n",
       "      <td>0.151596</td>\n",
       "      <td>0.249044</td>\n",
       "      <td>0.171595</td>\n",
       "      <td>0.225621</td>\n",
       "      <td>0.368594</td>\n",
       "      <td>1.427782</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>293.0</td>\n",
       "      <td>8298.0</td>\n",
       "      <td>1682.125000</td>\n",
       "      <td>544.5</td>\n",
       "      <td>0.775986</td>\n",
       "      <td>8.095238e-02</td>\n",
       "      <td>0.910386</td>\n",
       "      <td>-0.114286</td>\n",
       "      <td>...</td>\n",
       "      <td>0.036565</td>\n",
       "      <td>0.974500</td>\n",
       "      <td>-0.039966</td>\n",
       "      <td>0.079344</td>\n",
       "      <td>0.435984</td>\n",
       "      <td>0.131612</td>\n",
       "      <td>0.188830</td>\n",
       "      <td>0.402729</td>\n",
       "      <td>1.236817</td>\n",
       "      <td>0.383369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>335.0</td>\n",
       "      <td>7859.0</td>\n",
       "      <td>1682.125000</td>\n",
       "      <td>561.5</td>\n",
       "      <td>0.770013</td>\n",
       "      <td>2.857143e-02</td>\n",
       "      <td>0.900655</td>\n",
       "      <td>-0.114286</td>\n",
       "      <td>...</td>\n",
       "      <td>0.066156</td>\n",
       "      <td>0.975629</td>\n",
       "      <td>-0.002381</td>\n",
       "      <td>0.088209</td>\n",
       "      <td>0.440594</td>\n",
       "      <td>0.133342</td>\n",
       "      <td>0.201241</td>\n",
       "      <td>0.409422</td>\n",
       "      <td>1.232118</td>\n",
       "      <td>0.415992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>332.0</td>\n",
       "      <td>6903.0</td>\n",
       "      <td>1922.428571</td>\n",
       "      <td>602.0</td>\n",
       "      <td>0.780477</td>\n",
       "      <td>3.533333e-01</td>\n",
       "      <td>0.900561</td>\n",
       "      <td>-0.073333</td>\n",
       "      <td>...</td>\n",
       "      <td>0.046905</td>\n",
       "      <td>0.971056</td>\n",
       "      <td>-0.104524</td>\n",
       "      <td>0.122340</td>\n",
       "      <td>0.471950</td>\n",
       "      <td>0.131913</td>\n",
       "      <td>0.211436</td>\n",
       "      <td>0.449288</td>\n",
       "      <td>1.226350</td>\n",
       "      <td>0.487033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>334.0</td>\n",
       "      <td>7699.0</td>\n",
       "      <td>1682.125000</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.772303</td>\n",
       "      <td>5.947623e-18</td>\n",
       "      <td>0.902216</td>\n",
       "      <td>-0.038095</td>\n",
       "      <td>...</td>\n",
       "      <td>0.109184</td>\n",
       "      <td>0.974839</td>\n",
       "      <td>0.015136</td>\n",
       "      <td>0.088652</td>\n",
       "      <td>0.441101</td>\n",
       "      <td>0.132390</td>\n",
       "      <td>0.196809</td>\n",
       "      <td>0.403238</td>\n",
       "      <td>1.221877</td>\n",
       "      <td>0.427881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>304.0</td>\n",
       "      <td>6279.0</td>\n",
       "      <td>1682.125000</td>\n",
       "      <td>506.0</td>\n",
       "      <td>0.816808</td>\n",
       "      <td>9.523810e-02</td>\n",
       "      <td>0.909999</td>\n",
       "      <td>0.009524</td>\n",
       "      <td>...</td>\n",
       "      <td>0.095408</td>\n",
       "      <td>0.977496</td>\n",
       "      <td>-0.054422</td>\n",
       "      <td>0.252216</td>\n",
       "      <td>0.636022</td>\n",
       "      <td>0.130992</td>\n",
       "      <td>0.143174</td>\n",
       "      <td>0.382072</td>\n",
       "      <td>1.174477</td>\n",
       "      <td>0.533403</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>357 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                config  num_clusters  \\\n",
       "0    2019 - allenai-specter_titles.pkl - allenai-sp...          10.0   \n",
       "1    2019 - allenai-specter_titles.pkl - allenai-sp...          15.0   \n",
       "2    2019 - allenai-specter_titles.pkl - allenai-sp...          20.0   \n",
       "3    2019 - allenai-specter_titles.pkl - allenai-sp...          25.0   \n",
       "4    2019 - allenai-specter_titles.pkl - allenai-sp...          30.0   \n",
       "..                                                 ...           ...   \n",
       "352  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...           8.0   \n",
       "353  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...           8.0   \n",
       "354  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...           7.0   \n",
       "355  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...           8.0   \n",
       "356  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...           8.0   \n",
       "\n",
       "     min_members  max_members  mean_members  median_members  \\\n",
       "0          834.0       1775.0   1345.700000          1392.0   \n",
       "1          408.0       1551.0    897.133333           883.0   \n",
       "2          337.0       1102.0    672.850000           703.5   \n",
       "3          145.0        856.0    538.280000           525.0   \n",
       "4          131.0        758.0    448.566667           448.0   \n",
       "..           ...          ...           ...             ...   \n",
       "352        293.0       8298.0   1682.125000           544.5   \n",
       "353        335.0       7859.0   1682.125000           561.5   \n",
       "354        332.0       6903.0   1922.428571           602.0   \n",
       "355        334.0       7699.0   1682.125000           615.0   \n",
       "356        304.0       6279.0   1682.125000           506.0   \n",
       "\n",
       "     intra_similarity_sem_scholar_5  inter_similarity_sem_scholar_5  \\\n",
       "0                          0.685426                    9.555556e-02   \n",
       "1                          0.703512                    2.952381e-02   \n",
       "2                          0.709529                    9.578947e-02   \n",
       "3                          0.721358                    1.866667e-01   \n",
       "4                          0.723910                    8.804598e-02   \n",
       "..                              ...                             ...   \n",
       "352                        0.775986                    8.095238e-02   \n",
       "353                        0.770013                    2.857143e-02   \n",
       "354                        0.780477                    3.533333e-01   \n",
       "355                        0.772303                    5.947623e-18   \n",
       "356                        0.816808                    9.523810e-02   \n",
       "\n",
       "     intra_similarity_cso_enhanced_5  inter_similarity_cso_enhanced_5  ...  \\\n",
       "0                           0.837812                         0.028889  ...   \n",
       "1                           0.853639                         0.087619  ...   \n",
       "2                           0.855512                         0.032632  ...   \n",
       "3                           0.861880                         0.051333  ...   \n",
       "4                           0.864502                         0.074713  ...   \n",
       "..                               ...                              ...  ...   \n",
       "352                         0.910386                        -0.114286  ...   \n",
       "353                         0.900655                        -0.114286  ...   \n",
       "354                         0.900561                        -0.073333  ...   \n",
       "355                         0.902216                        -0.038095  ...   \n",
       "356                         0.909999                         0.009524  ...   \n",
       "\n",
       "     inter_similarity_sem_scholar_15  intra_similarity_cso_enhanced_15  \\\n",
       "0                           0.014127                          0.943590   \n",
       "1                          -0.020714                          0.953988   \n",
       "2                           0.001241                          0.952921   \n",
       "3                           0.027940                          0.956241   \n",
       "4                           0.033547                          0.956618   \n",
       "..                               ...                               ...   \n",
       "352                         0.036565                          0.974500   \n",
       "353                         0.066156                          0.975629   \n",
       "354                         0.046905                          0.971056   \n",
       "355                         0.109184                          0.974839   \n",
       "356                         0.095408                          0.977496   \n",
       "\n",
       "     inter_similarity_cso_enhanced_15  accuracy_new_papers_sem_scholar  \\\n",
       "0                            0.029206                         0.204344   \n",
       "1                            0.017959                         0.162677   \n",
       "2                            0.033684                         0.137855   \n",
       "3                            0.046917                         0.160461   \n",
       "4                            0.033892                         0.151596   \n",
       "..                                ...                              ...   \n",
       "352                         -0.039966                         0.079344   \n",
       "353                         -0.002381                         0.088209   \n",
       "354                         -0.104524                         0.122340   \n",
       "355                          0.015136                         0.088652   \n",
       "356                         -0.054422                         0.252216   \n",
       "\n",
       "     ranking_score_new_papers_sem_scholar  \\\n",
       "0                                0.384592   \n",
       "1                                0.325285   \n",
       "2                                0.276878   \n",
       "3                                0.271853   \n",
       "4                                0.249044   \n",
       "..                                    ...   \n",
       "352                              0.435984   \n",
       "353                              0.440594   \n",
       "354                              0.471950   \n",
       "355                              0.441101   \n",
       "356                              0.636022   \n",
       "\n",
       "     absolute_score_new_papers_sem_scholar  accuracy_new_papers_cso_enhanced  \\\n",
       "0                                 0.131923                          0.280142   \n",
       "1                                 0.153040                          0.279699   \n",
       "2                                 0.159520                          0.253103   \n",
       "3                                 0.168357                          0.241135   \n",
       "4                                 0.171595                          0.225621   \n",
       "..                                     ...                               ...   \n",
       "352                               0.131612                          0.188830   \n",
       "353                               0.133342                          0.201241   \n",
       "354                               0.131913                          0.211436   \n",
       "355                               0.132390                          0.196809   \n",
       "356                               0.130992                          0.143174   \n",
       "\n",
       "     ranking_score_new_papers_cso_enhanced  \\\n",
       "0                                 0.479073   \n",
       "1                                 0.452538   \n",
       "2                                 0.408462   \n",
       "3                                 0.388858   \n",
       "4                                 0.368594   \n",
       "..                                     ...   \n",
       "352                               0.402729   \n",
       "353                               0.409422   \n",
       "354                               0.449288   \n",
       "355                               0.403238   \n",
       "356                               0.382072   \n",
       "\n",
       "     absolute_score_new_papers_cso_enhanced  papers_clustered  \n",
       "0                                  1.196837          1.000000  \n",
       "1                                  1.362130          1.000000  \n",
       "2                                  1.384199          1.000000  \n",
       "3                                  1.410478          1.000000  \n",
       "4                                  1.427782          1.000000  \n",
       "..                                      ...               ...  \n",
       "352                                1.236817          0.383369  \n",
       "353                                1.232118          0.415992  \n",
       "354                                1.226350          0.487033  \n",
       "355                                1.221877          0.427881  \n",
       "356                                1.174477          0.533403  \n",
       "\n",
       "[357 rows x 25 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Number of clusterings:\", len(df_ce))\n",
    "df_ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cf4985b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store df_ce on disk\n",
    "df_ce.to_csv(\"data/clusters/cluster_evaluation_all_models_all_text_sets.csv\", sep=\"|\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "33346bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load df_ce from disk\n",
    "df_ce = pd.read_csv(\"data/clusters/cluster_evaluation_all_models_all_text_sets.csv\", sep=\"|\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "abc20b72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "151\n"
     ]
    }
   ],
   "source": [
    "# Filter out configurations with too less or too many clusters\n",
    "df_ce_filtered = df_ce[df_ce[\"num_clusters\"] >= 10.0]\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"num_clusters\"] <= 50.0]\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"median_members\"] >= 200]\n",
    "print(len(df_ce_filtered))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "87662088",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make absolute inter similarity score because of spearman correlation\n",
    "df_ce_filtered['inter_similarity_sem_scholar_5'] = df_ce_filtered['inter_similarity_sem_scholar_5'].abs()\n",
    "df_ce_filtered['inter_similarity_sem_scholar_10'] = df_ce_filtered['inter_similarity_sem_scholar_10'].abs()\n",
    "df_ce_filtered['inter_similarity_sem_scholar_15'] = df_ce_filtered['inter_similarity_sem_scholar_15'].abs()\n",
    "df_ce_filtered['inter_similarity_cso_enhanced_5'] = df_ce_filtered['inter_similarity_cso_enhanced_5'].abs()\n",
    "df_ce_filtered['inter_similarity_cso_enhanced_10'] = df_ce_filtered['inter_similarity_cso_enhanced_10'].abs()\n",
    "df_ce_filtered['inter_similarity_cso_enhanced_15'] = df_ce_filtered['inter_similarity_cso_enhanced_15'].abs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "5df1cd3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "151\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>config</th>\n",
       "      <th>num_clusters</th>\n",
       "      <th>min_members</th>\n",
       "      <th>max_members</th>\n",
       "      <th>mean_members</th>\n",
       "      <th>median_members</th>\n",
       "      <th>intra_similarity_sem_scholar_5</th>\n",
       "      <th>inter_similarity_sem_scholar_5</th>\n",
       "      <th>intra_similarity_cso_enhanced_5</th>\n",
       "      <th>inter_similarity_cso_enhanced_5</th>\n",
       "      <th>...</th>\n",
       "      <th>inter_similarity_sem_scholar_15</th>\n",
       "      <th>intra_similarity_cso_enhanced_15</th>\n",
       "      <th>inter_similarity_cso_enhanced_15</th>\n",
       "      <th>accuracy_new_papers_sem_scholar</th>\n",
       "      <th>ranking_score_new_papers_sem_scholar</th>\n",
       "      <th>absolute_score_new_papers_sem_scholar</th>\n",
       "      <th>accuracy_new_papers_cso_enhanced</th>\n",
       "      <th>ranking_score_new_papers_cso_enhanced</th>\n",
       "      <th>absolute_score_new_papers_cso_enhanced</th>\n",
       "      <th>papers_clustered</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>834.0</td>\n",
       "      <td>1775.0</td>\n",
       "      <td>1345.700000</td>\n",
       "      <td>1392.0</td>\n",
       "      <td>0.685426</td>\n",
       "      <td>0.095556</td>\n",
       "      <td>0.837812</td>\n",
       "      <td>0.028889</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014127</td>\n",
       "      <td>0.943590</td>\n",
       "      <td>0.029206</td>\n",
       "      <td>0.204344</td>\n",
       "      <td>0.384592</td>\n",
       "      <td>0.131923</td>\n",
       "      <td>0.280142</td>\n",
       "      <td>0.479073</td>\n",
       "      <td>1.196837</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>408.0</td>\n",
       "      <td>1551.0</td>\n",
       "      <td>897.133333</td>\n",
       "      <td>883.0</td>\n",
       "      <td>0.703512</td>\n",
       "      <td>0.029524</td>\n",
       "      <td>0.853639</td>\n",
       "      <td>0.087619</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020714</td>\n",
       "      <td>0.953988</td>\n",
       "      <td>0.017959</td>\n",
       "      <td>0.162677</td>\n",
       "      <td>0.325285</td>\n",
       "      <td>0.153040</td>\n",
       "      <td>0.279699</td>\n",
       "      <td>0.452538</td>\n",
       "      <td>1.362130</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>337.0</td>\n",
       "      <td>1102.0</td>\n",
       "      <td>672.850000</td>\n",
       "      <td>703.5</td>\n",
       "      <td>0.709529</td>\n",
       "      <td>0.095789</td>\n",
       "      <td>0.855512</td>\n",
       "      <td>0.032632</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001241</td>\n",
       "      <td>0.952921</td>\n",
       "      <td>0.033684</td>\n",
       "      <td>0.137855</td>\n",
       "      <td>0.276878</td>\n",
       "      <td>0.159520</td>\n",
       "      <td>0.253103</td>\n",
       "      <td>0.408462</td>\n",
       "      <td>1.384199</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>856.0</td>\n",
       "      <td>538.280000</td>\n",
       "      <td>525.0</td>\n",
       "      <td>0.721358</td>\n",
       "      <td>0.186667</td>\n",
       "      <td>0.861880</td>\n",
       "      <td>0.051333</td>\n",
       "      <td>...</td>\n",
       "      <td>0.027940</td>\n",
       "      <td>0.956241</td>\n",
       "      <td>0.046917</td>\n",
       "      <td>0.160461</td>\n",
       "      <td>0.271853</td>\n",
       "      <td>0.168357</td>\n",
       "      <td>0.241135</td>\n",
       "      <td>0.388858</td>\n",
       "      <td>1.410478</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019 - allenai-specter_titles.pkl - allenai-sp...</td>\n",
       "      <td>30.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>758.0</td>\n",
       "      <td>448.566667</td>\n",
       "      <td>448.0</td>\n",
       "      <td>0.723910</td>\n",
       "      <td>0.088046</td>\n",
       "      <td>0.864502</td>\n",
       "      <td>0.074713</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033547</td>\n",
       "      <td>0.956618</td>\n",
       "      <td>0.033892</td>\n",
       "      <td>0.151596</td>\n",
       "      <td>0.249044</td>\n",
       "      <td>0.171595</td>\n",
       "      <td>0.225621</td>\n",
       "      <td>0.368594</td>\n",
       "      <td>1.427782</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>30.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>651.0</td>\n",
       "      <td>448.566667</td>\n",
       "      <td>486.0</td>\n",
       "      <td>0.706797</td>\n",
       "      <td>0.005517</td>\n",
       "      <td>0.855646</td>\n",
       "      <td>0.077471</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014433</td>\n",
       "      <td>0.953812</td>\n",
       "      <td>0.024080</td>\n",
       "      <td>0.127660</td>\n",
       "      <td>0.231352</td>\n",
       "      <td>0.174889</td>\n",
       "      <td>0.273493</td>\n",
       "      <td>0.426184</td>\n",
       "      <td>1.519676</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>35.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>639.0</td>\n",
       "      <td>384.485714</td>\n",
       "      <td>394.0</td>\n",
       "      <td>0.713623</td>\n",
       "      <td>0.007731</td>\n",
       "      <td>0.863148</td>\n",
       "      <td>0.072605</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001555</td>\n",
       "      <td>0.956959</td>\n",
       "      <td>0.022923</td>\n",
       "      <td>0.107713</td>\n",
       "      <td>0.209076</td>\n",
       "      <td>0.180362</td>\n",
       "      <td>0.222961</td>\n",
       "      <td>0.381110</td>\n",
       "      <td>1.525443</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>40.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>611.0</td>\n",
       "      <td>336.425000</td>\n",
       "      <td>329.5</td>\n",
       "      <td>0.721974</td>\n",
       "      <td>0.007051</td>\n",
       "      <td>0.859576</td>\n",
       "      <td>0.075513</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004574</td>\n",
       "      <td>0.955896</td>\n",
       "      <td>0.023466</td>\n",
       "      <td>0.106383</td>\n",
       "      <td>0.195444</td>\n",
       "      <td>0.182329</td>\n",
       "      <td>0.272163</td>\n",
       "      <td>0.402407</td>\n",
       "      <td>1.553058</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>45.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>621.0</td>\n",
       "      <td>299.044444</td>\n",
       "      <td>308.0</td>\n",
       "      <td>0.737014</td>\n",
       "      <td>0.019293</td>\n",
       "      <td>0.867625</td>\n",
       "      <td>0.021010</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004708</td>\n",
       "      <td>0.959419</td>\n",
       "      <td>0.021670</td>\n",
       "      <td>0.107713</td>\n",
       "      <td>0.193764</td>\n",
       "      <td>0.188680</td>\n",
       "      <td>0.250887</td>\n",
       "      <td>0.385271</td>\n",
       "      <td>1.580437</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>466.0</td>\n",
       "      <td>269.140000</td>\n",
       "      <td>283.0</td>\n",
       "      <td>0.740991</td>\n",
       "      <td>0.006939</td>\n",
       "      <td>0.874662</td>\n",
       "      <td>0.029224</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003114</td>\n",
       "      <td>0.960730</td>\n",
       "      <td>0.006093</td>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.185541</td>\n",
       "      <td>0.193578</td>\n",
       "      <td>0.231826</td>\n",
       "      <td>0.367675</td>\n",
       "      <td>1.577092</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>151 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                config  num_clusters  \\\n",
       "0    2019 - allenai-specter_titles.pkl - allenai-sp...          10.0   \n",
       "1    2019 - allenai-specter_titles.pkl - allenai-sp...          15.0   \n",
       "2    2019 - allenai-specter_titles.pkl - allenai-sp...          20.0   \n",
       "3    2019 - allenai-specter_titles.pkl - allenai-sp...          25.0   \n",
       "4    2019 - allenai-specter_titles.pkl - allenai-sp...          30.0   \n",
       "..                                                 ...           ...   \n",
       "331  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...          30.0   \n",
       "332  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...          35.0   \n",
       "333  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...          40.0   \n",
       "334  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...          45.0   \n",
       "335  2019 - paraphrase-TinyBERT-L6-v2_titles_abstra...          50.0   \n",
       "\n",
       "     min_members  max_members  mean_members  median_members  \\\n",
       "0          834.0       1775.0   1345.700000          1392.0   \n",
       "1          408.0       1551.0    897.133333           883.0   \n",
       "2          337.0       1102.0    672.850000           703.5   \n",
       "3          145.0        856.0    538.280000           525.0   \n",
       "4          131.0        758.0    448.566667           448.0   \n",
       "..           ...          ...           ...             ...   \n",
       "331         94.0        651.0    448.566667           486.0   \n",
       "332        138.0        639.0    384.485714           394.0   \n",
       "333         61.0        611.0    336.425000           329.5   \n",
       "334        115.0        621.0    299.044444           308.0   \n",
       "335         65.0        466.0    269.140000           283.0   \n",
       "\n",
       "     intra_similarity_sem_scholar_5  inter_similarity_sem_scholar_5  \\\n",
       "0                          0.685426                        0.095556   \n",
       "1                          0.703512                        0.029524   \n",
       "2                          0.709529                        0.095789   \n",
       "3                          0.721358                        0.186667   \n",
       "4                          0.723910                        0.088046   \n",
       "..                              ...                             ...   \n",
       "331                        0.706797                        0.005517   \n",
       "332                        0.713623                        0.007731   \n",
       "333                        0.721974                        0.007051   \n",
       "334                        0.737014                        0.019293   \n",
       "335                        0.740991                        0.006939   \n",
       "\n",
       "     intra_similarity_cso_enhanced_5  inter_similarity_cso_enhanced_5  ...  \\\n",
       "0                           0.837812                         0.028889  ...   \n",
       "1                           0.853639                         0.087619  ...   \n",
       "2                           0.855512                         0.032632  ...   \n",
       "3                           0.861880                         0.051333  ...   \n",
       "4                           0.864502                         0.074713  ...   \n",
       "..                               ...                              ...  ...   \n",
       "331                         0.855646                         0.077471  ...   \n",
       "332                         0.863148                         0.072605  ...   \n",
       "333                         0.859576                         0.075513  ...   \n",
       "334                         0.867625                         0.021010  ...   \n",
       "335                         0.874662                         0.029224  ...   \n",
       "\n",
       "     inter_similarity_sem_scholar_15  intra_similarity_cso_enhanced_15  \\\n",
       "0                           0.014127                          0.943590   \n",
       "1                           0.020714                          0.953988   \n",
       "2                           0.001241                          0.952921   \n",
       "3                           0.027940                          0.956241   \n",
       "4                           0.033547                          0.956618   \n",
       "..                               ...                               ...   \n",
       "331                         0.014433                          0.953812   \n",
       "332                         0.001555                          0.956959   \n",
       "333                         0.004574                          0.955896   \n",
       "334                         0.004708                          0.959419   \n",
       "335                         0.003114                          0.960730   \n",
       "\n",
       "     inter_similarity_cso_enhanced_15  accuracy_new_papers_sem_scholar  \\\n",
       "0                            0.029206                         0.204344   \n",
       "1                            0.017959                         0.162677   \n",
       "2                            0.033684                         0.137855   \n",
       "3                            0.046917                         0.160461   \n",
       "4                            0.033892                         0.151596   \n",
       "..                                ...                              ...   \n",
       "331                          0.024080                         0.127660   \n",
       "332                          0.022923                         0.107713   \n",
       "333                          0.023466                         0.106383   \n",
       "334                          0.021670                         0.107713   \n",
       "335                          0.006093                         0.104167   \n",
       "\n",
       "     ranking_score_new_papers_sem_scholar  \\\n",
       "0                                0.384592   \n",
       "1                                0.325285   \n",
       "2                                0.276878   \n",
       "3                                0.271853   \n",
       "4                                0.249044   \n",
       "..                                    ...   \n",
       "331                              0.231352   \n",
       "332                              0.209076   \n",
       "333                              0.195444   \n",
       "334                              0.193764   \n",
       "335                              0.185541   \n",
       "\n",
       "     absolute_score_new_papers_sem_scholar  accuracy_new_papers_cso_enhanced  \\\n",
       "0                                 0.131923                          0.280142   \n",
       "1                                 0.153040                          0.279699   \n",
       "2                                 0.159520                          0.253103   \n",
       "3                                 0.168357                          0.241135   \n",
       "4                                 0.171595                          0.225621   \n",
       "..                                     ...                               ...   \n",
       "331                               0.174889                          0.273493   \n",
       "332                               0.180362                          0.222961   \n",
       "333                               0.182329                          0.272163   \n",
       "334                               0.188680                          0.250887   \n",
       "335                               0.193578                          0.231826   \n",
       "\n",
       "     ranking_score_new_papers_cso_enhanced  \\\n",
       "0                                 0.479073   \n",
       "1                                 0.452538   \n",
       "2                                 0.408462   \n",
       "3                                 0.388858   \n",
       "4                                 0.368594   \n",
       "..                                     ...   \n",
       "331                               0.426184   \n",
       "332                               0.381110   \n",
       "333                               0.402407   \n",
       "334                               0.385271   \n",
       "335                               0.367675   \n",
       "\n",
       "     absolute_score_new_papers_cso_enhanced  papers_clustered  \n",
       "0                                  1.196837               1.0  \n",
       "1                                  1.362130               1.0  \n",
       "2                                  1.384199               1.0  \n",
       "3                                  1.410478               1.0  \n",
       "4                                  1.427782               1.0  \n",
       "..                                      ...               ...  \n",
       "331                                1.519676               1.0  \n",
       "332                                1.525443               1.0  \n",
       "333                                1.553058               1.0  \n",
       "334                                1.580437               1.0  \n",
       "335                                1.577092               1.0  \n",
       "\n",
       "[151 rows x 25 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Number of remaining clusterings:\", len(df_ce_filtered))\n",
    "df_ce_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "684e04e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 10 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 20 - 30\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 10 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 45 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-TinyBERT-L6-v2_titles_abstracts.pkl - paraphrase-TinyBERT-L6-v2 - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 100 - 400 - 30\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - topic - null - null - 200 - 20 - 30\n",
      "2019 - paraphrase-mpnet-base-v2_titles_abstracts.pkl - paraphrase-mpnet-base-v2 - title+abstract - kmeans - 50 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 40 - null - null - null - null\n",
      "2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 40 - null - null - null - null\n",
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 20 - null - null - null - null\n",
      "2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - topic - null - null - 200 - 10 - 80\n",
      "2019 - paraphrase-distilroberta-base-v2_titles_abstracts.pkl - paraphrase-distilroberta-base-v2 - title+abstract - kmeans - 15 - null - null - null - null\n"
     ]
    }
   ],
   "source": [
    "# Show best config for each metric\n",
    "print(df_ce_filtered.sort_values(by=['accuracy_new_papers_sem_scholar'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['accuracy_new_papers_cso_enhanced'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['ranking_score_new_papers_sem_scholar'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['ranking_score_new_papers_cso_enhanced'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['absolute_score_new_papers_sem_scholar'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['absolute_score_new_papers_cso_enhanced'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['intra_similarity_sem_scholar_5'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['intra_similarity_sem_scholar_10'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['intra_similarity_sem_scholar_15'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['intra_similarity_cso_enhanced_5'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['intra_similarity_cso_enhanced_10'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['intra_similarity_cso_enhanced_15'], ascending=False).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['inter_similarity_sem_scholar_5']).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['inter_similarity_sem_scholar_10']).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['inter_similarity_sem_scholar_15']).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['inter_similarity_cso_enhanced_5']).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['inter_similarity_cso_enhanced_10']).iloc[0][\"config\"])\n",
    "print(df_ce_filtered.sort_values(by=['inter_similarity_cso_enhanced_15']).iloc[0][\"config\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9c1b1616",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manually check metrics, one metric should not filter too many configurations\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"inter_similarity_sem_scholar_5\"] <= 0.1]\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"inter_similarity_sem_scholar_10\"] <= 0.1]\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"inter_similarity_sem_scholar_15\"] <= 0.1]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"inter_similarity_cso_enhanced_5\"] <= 0.1]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"inter_similarity_cso_enhanced_10\"] <= 0.2]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"inter_similarity_cso_enhanced_15\"] <= 0.2]\n",
    "\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"intra_similarity_sem_scholar_5\"] >= 0.72]\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"intra_similarity_sem_scholar_10\"] >= 0.8]\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"intra_similarity_sem_scholar_15\"] >= 0.8]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"intra_similarity_cso_enhanced_5\"] >= 0.8]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"intra_similarity_cso_enhanced_10\"] >= 0.9]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"intra_similarity_cso_enhanced_15\"] >= 0.9]\n",
    "\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"accuracy_new_papers_sem_scholar\"] >= 0.15]\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"ranking_score_new_papers_sem_scholar\"] >= 0.3]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"absolute_score_new_papers_sem_scholar\"] >= 0.1]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"accuracy_new_papers_cso_enhanced\"] >= 0.1]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"ranking_score_new_papers_cso_enhanced\"] >= 0.2]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"absolute_score_new_papers_cso_enhanced\"] >= 1.3]\n",
    "#df_ce_filtered = df_ce_filtered[df_ce_filtered[\"absolute_score_new_papers_cso_enhanced\"] >= 1.0]\n",
    "\n",
    "df_ce_filtered = df_ce_filtered[df_ce_filtered[\"papers_clustered\"] >= 0.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "dea40913",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "['2019 - allenai-specter_titles_abstracts.pkl - allenai-specter - title+abstract - kmeans - 15 - null - null - null - null', '2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 20 - null - null - null - null', '2019 - paraphrase-mpnet-base-v2_titles.pkl - paraphrase-mpnet-base-v2 - title - kmeans - 15 - null - null - null - null', '2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - paraphrase-TinyBERT-L6-v2 - title - kmeans - 20 - null - null - null - null']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>config</th>\n",
       "      <th>num_clusters</th>\n",
       "      <th>min_members</th>\n",
       "      <th>max_members</th>\n",
       "      <th>mean_members</th>\n",
       "      <th>median_members</th>\n",
       "      <th>intra_similarity_sem_scholar_5</th>\n",
       "      <th>inter_similarity_sem_scholar_5</th>\n",
       "      <th>intra_similarity_cso_enhanced_5</th>\n",
       "      <th>inter_similarity_cso_enhanced_5</th>\n",
       "      <th>...</th>\n",
       "      <th>inter_similarity_sem_scholar_15</th>\n",
       "      <th>intra_similarity_cso_enhanced_15</th>\n",
       "      <th>inter_similarity_cso_enhanced_15</th>\n",
       "      <th>accuracy_new_papers_sem_scholar</th>\n",
       "      <th>ranking_score_new_papers_sem_scholar</th>\n",
       "      <th>absolute_score_new_papers_sem_scholar</th>\n",
       "      <th>accuracy_new_papers_cso_enhanced</th>\n",
       "      <th>ranking_score_new_papers_cso_enhanced</th>\n",
       "      <th>absolute_score_new_papers_cso_enhanced</th>\n",
       "      <th>papers_clustered</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2019 - allenai-specter_titles_abstracts.pkl - ...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>469.0</td>\n",
       "      <td>1282.0</td>\n",
       "      <td>897.133333</td>\n",
       "      <td>930.0</td>\n",
       "      <td>0.723567</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.857161</td>\n",
       "      <td>0.051429</td>\n",
       "      <td>...</td>\n",
       "      <td>0.027857</td>\n",
       "      <td>0.958907</td>\n",
       "      <td>0.002075</td>\n",
       "      <td>0.176862</td>\n",
       "      <td>0.320137</td>\n",
       "      <td>0.176693</td>\n",
       "      <td>0.308954</td>\n",
       "      <td>0.496687</td>\n",
       "      <td>1.438828</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>2019 - paraphrase-distilroberta-base-v2_titles...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>1070.0</td>\n",
       "      <td>672.850000</td>\n",
       "      <td>695.0</td>\n",
       "      <td>0.727509</td>\n",
       "      <td>0.008947</td>\n",
       "      <td>0.875047</td>\n",
       "      <td>0.000526</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013365</td>\n",
       "      <td>0.952762</td>\n",
       "      <td>0.009004</td>\n",
       "      <td>0.186170</td>\n",
       "      <td>0.312755</td>\n",
       "      <td>0.171167</td>\n",
       "      <td>0.302305</td>\n",
       "      <td>0.444324</td>\n",
       "      <td>1.482432</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>2019 - paraphrase-mpnet-base-v2_titles.pkl - p...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>329.0</td>\n",
       "      <td>1397.0</td>\n",
       "      <td>897.133333</td>\n",
       "      <td>858.0</td>\n",
       "      <td>0.729748</td>\n",
       "      <td>0.064762</td>\n",
       "      <td>0.863383</td>\n",
       "      <td>0.076190</td>\n",
       "      <td>...</td>\n",
       "      <td>0.022211</td>\n",
       "      <td>0.949832</td>\n",
       "      <td>0.030646</td>\n",
       "      <td>0.164007</td>\n",
       "      <td>0.332899</td>\n",
       "      <td>0.146624</td>\n",
       "      <td>0.236702</td>\n",
       "      <td>0.405862</td>\n",
       "      <td>1.345371</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - ...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1078.0</td>\n",
       "      <td>672.850000</td>\n",
       "      <td>747.0</td>\n",
       "      <td>0.751635</td>\n",
       "      <td>0.032105</td>\n",
       "      <td>0.875585</td>\n",
       "      <td>0.019474</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015301</td>\n",
       "      <td>0.957040</td>\n",
       "      <td>0.002914</td>\n",
       "      <td>0.156915</td>\n",
       "      <td>0.306498</td>\n",
       "      <td>0.177697</td>\n",
       "      <td>0.258422</td>\n",
       "      <td>0.409958</td>\n",
       "      <td>1.445785</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                config  num_clusters  \\\n",
       "30   2019 - allenai-specter_titles_abstracts.pkl - ...          15.0   \n",
       "89   2019 - paraphrase-distilroberta-base-v2_titles...          20.0   \n",
       "178  2019 - paraphrase-mpnet-base-v2_titles.pkl - p...          15.0   \n",
       "269  2019 - paraphrase-TinyBERT-L6-v2_titles.pkl - ...          20.0   \n",
       "\n",
       "     min_members  max_members  mean_members  median_members  \\\n",
       "30         469.0       1282.0    897.133333           930.0   \n",
       "89         286.0       1070.0    672.850000           695.0   \n",
       "178        329.0       1397.0    897.133333           858.0   \n",
       "269        150.0       1078.0    672.850000           747.0   \n",
       "\n",
       "     intra_similarity_sem_scholar_5  inter_similarity_sem_scholar_5  \\\n",
       "30                         0.723567                        0.020000   \n",
       "89                         0.727509                        0.008947   \n",
       "178                        0.729748                        0.064762   \n",
       "269                        0.751635                        0.032105   \n",
       "\n",
       "     intra_similarity_cso_enhanced_5  inter_similarity_cso_enhanced_5  ...  \\\n",
       "30                          0.857161                         0.051429  ...   \n",
       "89                          0.875047                         0.000526  ...   \n",
       "178                         0.863383                         0.076190  ...   \n",
       "269                         0.875585                         0.019474  ...   \n",
       "\n",
       "     inter_similarity_sem_scholar_15  intra_similarity_cso_enhanced_15  \\\n",
       "30                          0.027857                          0.958907   \n",
       "89                          0.013365                          0.952762   \n",
       "178                         0.022211                          0.949832   \n",
       "269                         0.015301                          0.957040   \n",
       "\n",
       "     inter_similarity_cso_enhanced_15  accuracy_new_papers_sem_scholar  \\\n",
       "30                           0.002075                         0.176862   \n",
       "89                           0.009004                         0.186170   \n",
       "178                          0.030646                         0.164007   \n",
       "269                          0.002914                         0.156915   \n",
       "\n",
       "     ranking_score_new_papers_sem_scholar  \\\n",
       "30                               0.320137   \n",
       "89                               0.312755   \n",
       "178                              0.332899   \n",
       "269                              0.306498   \n",
       "\n",
       "     absolute_score_new_papers_sem_scholar  accuracy_new_papers_cso_enhanced  \\\n",
       "30                                0.176693                          0.308954   \n",
       "89                                0.171167                          0.302305   \n",
       "178                               0.146624                          0.236702   \n",
       "269                               0.177697                          0.258422   \n",
       "\n",
       "     ranking_score_new_papers_cso_enhanced  \\\n",
       "30                                0.496687   \n",
       "89                                0.444324   \n",
       "178                               0.405862   \n",
       "269                               0.409958   \n",
       "\n",
       "     absolute_score_new_papers_cso_enhanced  papers_clustered  \n",
       "30                                 1.438828               1.0  \n",
       "89                                 1.482432               1.0  \n",
       "178                                1.345371               1.0  \n",
       "269                                1.445785               1.0  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Number of remaining clusterings:\", len(df_ce_filtered))\n",
    "print(list(df_ce_filtered[\"config\"]))\n",
    "df_ce_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "abdccfbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019 - paraphrase-distilroberta-base-v2_titles.pkl - paraphrase-distilroberta-base-v2 - title - kmeans - 20 - null - null - null - null\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "config                                    2019 - paraphrase-distilroberta-base-v2_titles...\n",
       "num_clusters                                                                           20.0\n",
       "min_members                                                                           286.0\n",
       "max_members                                                                          1070.0\n",
       "mean_members                                                                         672.85\n",
       "median_members                                                                        695.0\n",
       "intra_similarity_sem_scholar_5                                                     0.727509\n",
       "inter_similarity_sem_scholar_5                                                     0.008947\n",
       "intra_similarity_cso_enhanced_5                                                    0.875047\n",
       "inter_similarity_cso_enhanced_5                                                    0.000526\n",
       "intra_similarity_sem_scholar_10                                                    0.830112\n",
       "inter_similarity_sem_scholar_10                                                    0.013652\n",
       "intra_similarity_cso_enhanced_10                                                    0.93067\n",
       "inter_similarity_cso_enhanced_10                                                   0.006061\n",
       "intra_similarity_sem_scholar_15                                                    0.867879\n",
       "inter_similarity_sem_scholar_15                                                    0.013365\n",
       "intra_similarity_cso_enhanced_15                                                   0.952762\n",
       "inter_similarity_cso_enhanced_15                                                   0.009004\n",
       "accuracy_new_papers_sem_scholar                                                     0.18617\n",
       "ranking_score_new_papers_sem_scholar                                               0.312755\n",
       "absolute_score_new_papers_sem_scholar                                              0.171167\n",
       "accuracy_new_papers_cso_enhanced                                                   0.302305\n",
       "ranking_score_new_papers_cso_enhanced                                              0.444324\n",
       "absolute_score_new_papers_cso_enhanced                                             1.482432\n",
       "papers_clustered                                                                        1.0\n",
       "Name: 89, dtype: object"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Best model, text set, algorithm and config\n",
    "# Select this one because all papers are classified, other metrics where very good\n",
    "print(df_ce_filtered.loc[89][\"config\"])\n",
    "df_ce_filtered.loc[89]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}