03_LexicalDiversity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lexical Diversity Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "from math import sqrt, log\n",
    "from nltk import word_tokenize\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ttr(text):\n",
    "    tok_text = word_tokenize(text)\n",
    "    return len(set(tok_text)) / len(tok_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def root_ttr(text):\n",
    "    return sqrt(ttr(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def corrected_ttr(text):\n",
    "    tok_text = word_tokenize(text)\n",
    "    return sqrt(len(set(tok_text)) / (2 * len(tok_text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_ttr(text):\n",
    "    tok_text = word_tokenize(text)\n",
    "    if log(len(tok_text),2) == 0:\n",
    "        print(text)\n",
    "        return 0\n",
    "    return log(len(set(tok_text)),2) / log(len(tok_text),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def uber_index(text):\n",
    "    tok_text = word_tokenize(text)\n",
    "    if log(len(tok_text),2) != 0 and log(len(set(tok_text)),2) != 0:\n",
    "        return (log(len(tok_text),2) ** 2) / (log(len(set(tok_text)),2) / log(len(tok_text),2))\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def yule_s_k(text):\n",
    "    tok_text = word_tokenize(text)\n",
    "    token_counter = Counter(tok.upper() for tok in tok_text)\n",
    "    m1 = sum(token_counter.values())\n",
    "    m2 = sum([freq ** 2 for freq in token_counter.values()])\n",
    "    if m2-m1 != 0:\n",
    "        i = (m1*m1) / (m2-m1)\n",
    "        k = 10000 / i\n",
    "        return k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Copyright 2017 John Frens\n",
    "#\n",
    "#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n",
    "#\n",
    "#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n",
    "#\n",
    "#THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n",
    "\n",
    "import string\n",
    "\n",
    "# Global trandform for removing punctuation from words\n",
    "remove_punctuation = str.maketrans('', '', string.punctuation)\n",
    "\n",
    "# MTLD internal implementation\n",
    "def mtld_calc(word_array, ttr_threshold):\n",
    "    current_ttr = 1.0\n",
    "    token_count = 0\n",
    "    type_count = 0\n",
    "    types = set()\n",
    "    factors = 0.0\n",
    "    \n",
    "    for token in word_array:\n",
    "        token = token.translate(remove_punctuation).lower() # trim punctuation, make lowercase\n",
    "        token_count += 1\n",
    "        if token not in types:\n",
    "            type_count +=1\n",
    "            types.add(token)\n",
    "        current_ttr = type_count / token_count\n",
    "        if current_ttr <= ttr_threshold:\n",
    "            factors += 1\n",
    "            token_count = 0\n",
    "            type_count = 0\n",
    "            types = set()\n",
    "            current_ttr = 1.0\n",
    "    \n",
    "    excess = 1.0 - current_ttr\n",
    "    excess_val = 1.0 - ttr_threshold\n",
    "    factors += excess / excess_val\n",
    "    if factors != 0:\n",
    "        return len(word_array) / factors\n",
    "    return -1\n",
    "\n",
    "# MTLD implementation\n",
    "def mtld(word_array, ttr_threshold=0.72):\n",
    "    word_array = word_tokenize(word_array)\n",
    "    \n",
    "    return (mtld_calc(word_array, ttr_threshold) + mtld_calc(word_array[::-1], ttr_threshold)) / 2\n",
    "\n",
    "\n",
    "# HD-D internals\n",
    "\n",
    "# x! = x(x-1)(x-2)...(1)\n",
    "def factorial(x):\n",
    "    if x <= 1:\n",
    "        return 1\n",
    "    else:\n",
    "        return x * factorial(x - 1)\n",
    "\n",
    "# n choose r = n(n-1)(n-2)...(n-r+1)/(r!)\n",
    "def combination(n, r):\n",
    "    r_fact = factorial(r)\n",
    "    numerator = 1.0\n",
    "    num = n-r+1.0\n",
    "    while num < n+1.0:\n",
    "        numerator *= num\n",
    "        num += 1.0\n",
    "    return numerator / r_fact\n",
    "\n",
    "# hypergeometric probability: the probability that an n-trial hypergeometric experiment results \n",
    "#  in exactly x successes, when the population consists of N items, k of which are classified as successes.\n",
    "#  (here, population = N, population_successes = k, sample = n, sample_successes = x)\n",
    "#  h(x; N, n, k) = [ kCx ] * [ N-kCn-x ] / [ NCn ]\n",
    "def hypergeometric(population, population_successes, sample, sample_successes):\n",
    "    return (combination(population_successes, sample_successes) *\\\n",
    "            combination(population - population_successes, sample - sample_successes)) /\\\n",
    "            combination(population, sample)\n",
    "    \n",
    "# HD-D implementation\n",
    "def hdd(word_array, sample_size=42.0):\n",
    "    word_array = word_tokenize(word_array)\n",
    "    \n",
    "    # Create a dictionary of counts for each type\n",
    "    type_counts = {}\n",
    "    for token in word_array:\n",
    "        token = token.translate(remove_punctuation).lower() # trim punctuation, make lowercase\n",
    "        if token in type_counts:\n",
    "            type_counts[token] += 1.0\n",
    "        else:\n",
    "            type_counts[token] = 1.0\n",
    "    # Sum the contribution of each token - \"If the sample size is 42, the mean contribution of any given\n",
    "    #  type is 1/42 multiplied by the percentage of combinations in which the type would be found.\" (McCarthy & Jarvis 2010)\n",
    "    hdd_value = 0.0\n",
    "    for token_type in type_counts.keys():\n",
    "        contribution = (1.0 - hypergeometric(len(word_array), sample_size, type_counts[token_type], 0.0)) / sample_size\n",
    "        hdd_value += contribution\n",
    "\n",
    "    return hdd_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "LD_feature_functions = [ttr, root_ttr, corrected_ttr, log_ttr, uber_index, yule_s_k, mtld, hdd]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_3_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:18<00:00,  2.30s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_5_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:18<00:00,  2.30s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_7_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:17<00:00,  2.24s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_14_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:07<00:00,  1.03it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_12_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:07<00:00,  1.03it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_10_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:18<00:00,  2.32s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_2_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:08<00:00,  1.10s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_9_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:19<00:00,  2.49s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_3_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:09<00:00,  1.14s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_7_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:08<00:00,  1.04s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_11_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:06<00:00,  1.16it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_13_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:06<00:00,  1.15it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_8_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:16<00:00,  2.05s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_4_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:17<00:00,  2.14s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_12_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:16<00:00,  2.02s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_13_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:15<00:00,  1.99s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_9_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:06<00:00,  1.15it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_8_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:07<00:00,  1.14it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_15_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:15<00:00,  1.98s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_1_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 3/8 [00:02<00:04,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15-01-2012\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:07<00:00,  1.04it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_5_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:07<00:00,  1.10it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_10_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:06<00:00,  1.14it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_6_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:07<00:00,  1.13it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_15_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:06<00:00,  1.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_1_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:17<00:00,  2.23s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_6_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:16<00:00,  2.07s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_14_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:15<00:00,  1.97s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_11_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:16<00:00,  2.01s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_2_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:17<00:00,  2.24s/it]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_4_df.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:07<00:00,  1.09it/s]\n"
     ]
    }
   ],
   "source": [
    "input_files = os.listdir(\"/home/soeren/Documents/TextDifficultyAssessmentGerman/datasets/02_BasicFeatures\")\n",
    "input_files = [input_file for input_file in input_files if not input_file in os.listdir(\"/home/soeren/Documents/TextDifficultyAssessmentGerman/datasets/03_LexicalDiversity\")]\n",
    "\n",
    "\n",
    "def save_df(path,df):\n",
    "    df.to_csv(path,sep=\"|\",index=False)\n",
    "\n",
    "for input_file in input_files:\n",
    "    print(input_file)\n",
    "    df = pd.read_csv(\"datasets/02_BasicFeatures/\" + input_file, sep=\"|\", lineterminator = '\\n')\n",
    "\n",
    "    for function in tqdm(LD_feature_functions):\n",
    "        df[\"LD-\"+function.__name__]  = [function(text) for text in df[\"cleanedText\"]]\n",
    "\n",
    "    filename = \"datasets/03_LexicalDiversity/\" + input_file\n",
    "    save_df(filename, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>newLevel</th>\n",
       "      <th>RFTagger</th>\n",
       "      <th>parsedText</th>\n",
       "      <th>preprocessedText</th>\n",
       "      <th>BF-average_sentence_length</th>\n",
       "      <th>BF-average_syllable_word</th>\n",
       "      <th>BF-Flesch_Reading_Ease_Score</th>\n",
       "      <th>BF-Flesch_Kincaid_readability</th>\n",
       "      <th>BF-percentag_hard_words</th>\n",
       "      <th>...</th>\n",
       "      <th>BF-average_character_per_word</th>\n",
       "      <th>BF-text_length</th>\n",
       "      <th>LD-ttr</th>\n",
       "      <th>LD-root_ttr</th>\n",
       "      <th>LD-corrected_ttr</th>\n",
       "      <th>LD-log_ttr</th>\n",
       "      <th>LD-uber_index</th>\n",
       "      <th>LD-yule_s_k</th>\n",
       "      <th>LD-mtld</th>\n",
       "      <th>LD-hdd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Deine Frau und Babys geht nach Hause, oder Hos...</td>\n",
       "      <td>A</td>\n",
       "      <td>[[['Deine', 'PRO', 'Poss', 'Attr', '-', 'Nom',...</td>\n",
       "      <td>['(ROOT (CS (S (NP (ART Deine) (CNP (NN Frau) ...</td>\n",
       "      <td>[[['Deine', 'PRO', 'Poss', 'Attr', '-', 'Nom',...</td>\n",
       "      <td>8.250</td>\n",
       "      <td>1.581818</td>\n",
       "      <td>64.639432</td>\n",
       "      <td>7.281818</td>\n",
       "      <td>0.109091</td>\n",
       "      <td>...</td>\n",
       "      <td>4.800000</td>\n",
       "      <td>66</td>\n",
       "      <td>0.803030</td>\n",
       "      <td>0.896120</td>\n",
       "      <td>0.633652</td>\n",
       "      <td>0.947642</td>\n",
       "      <td>38.553281</td>\n",
       "      <td>142.332415</td>\n",
       "      <td>66.880000</td>\n",
       "      <td>0.768626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Nach der Kneipe ich befahrt den junge Mann zu ...</td>\n",
       "      <td>A</td>\n",
       "      <td>[[['Nach', 'APPR', 'Dat'], ['der', 'ART', 'Def...</td>\n",
       "      <td>['(ROOT (NUR (PP (APPR Nach) (ART der) (NN Kne...</td>\n",
       "      <td>[[['Nach', 'APPR', 'Dat'], ['der', 'ART', 'Def...</td>\n",
       "      <td>10.750</td>\n",
       "      <td>1.565789</td>\n",
       "      <td>63.457961</td>\n",
       "      <td>8.089474</td>\n",
       "      <td>0.078947</td>\n",
       "      <td>...</td>\n",
       "      <td>4.605263</td>\n",
       "      <td>86</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.909718</td>\n",
       "      <td>0.643268</td>\n",
       "      <td>0.957625</td>\n",
       "      <td>43.348401</td>\n",
       "      <td>89.840137</td>\n",
       "      <td>111.543158</td>\n",
       "      <td>0.839343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lieber Ingo, Wie gehst dir? Wenn ich nur die b...</td>\n",
       "      <td>A</td>\n",
       "      <td>[[['Lieber', 'N', 'Reg', 'Nom', 'Sg', 'Masc'],...</td>\n",
       "      <td>['(ROOT (S (S (ADJD Lieber) (NP (NE Ingo))) ($...</td>\n",
       "      <td>[[['Lieber', 'N', 'Reg', 'Nom', 'Sg', 'Masc'],...</td>\n",
       "      <td>8.125</td>\n",
       "      <td>1.642857</td>\n",
       "      <td>59.602411</td>\n",
       "      <td>7.964286</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>...</td>\n",
       "      <td>4.785714</td>\n",
       "      <td>65</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>0.919866</td>\n",
       "      <td>0.650444</td>\n",
       "      <td>0.959981</td>\n",
       "      <td>37.780860</td>\n",
       "      <td>80.473373</td>\n",
       "      <td>84.500000</td>\n",
       "      <td>0.819016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Er ist nicht verheiratet und hat keine Kinder....</td>\n",
       "      <td>A</td>\n",
       "      <td>[[['Er', 'PRO', 'Pers', 'Subst', '3', 'Nom', '...</td>\n",
       "      <td>['(ROOT (CS (S (PPER Er) (VAFIN ist) (VP (PTKN...</td>\n",
       "      <td>[[['Er', 'PRO', 'Pers', 'Subst', '3', 'Nom', '...</td>\n",
       "      <td>10.875</td>\n",
       "      <td>1.597403</td>\n",
       "      <td>60.656615</td>\n",
       "      <td>8.518831</td>\n",
       "      <td>0.129870</td>\n",
       "      <td>...</td>\n",
       "      <td>4.857143</td>\n",
       "      <td>87</td>\n",
       "      <td>0.758621</td>\n",
       "      <td>0.870988</td>\n",
       "      <td>0.615882</td>\n",
       "      <td>0.938142</td>\n",
       "      <td>44.248667</td>\n",
       "      <td>132.117849</td>\n",
       "      <td>62.929054</td>\n",
       "      <td>0.790037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Du wahrscheinlich hast kein Qualifikation für ...</td>\n",
       "      <td>A</td>\n",
       "      <td>[[['Du', 'PRO', 'Pers', 'Subst', '2', 'Nom', '...</td>\n",
       "      <td>['(ROOT (S (AP (PPER Du) (ADJD wahrscheinlich)...</td>\n",
       "      <td>[[['Du', 'PRO', 'Pers', 'Subst', '2', 'Nom', '...</td>\n",
       "      <td>10.625</td>\n",
       "      <td>1.583333</td>\n",
       "      <td>62.100625</td>\n",
       "      <td>8.250000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>...</td>\n",
       "      <td>4.513889</td>\n",
       "      <td>85</td>\n",
       "      <td>0.735632</td>\n",
       "      <td>0.857690</td>\n",
       "      <td>0.606478</td>\n",
       "      <td>0.931251</td>\n",
       "      <td>44.576064</td>\n",
       "      <td>150.614348</td>\n",
       "      <td>42.270204</td>\n",
       "      <td>0.774356</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text newLevel  \\\n",
       "0  Deine Frau und Babys geht nach Hause, oder Hos...        A   \n",
       "1  Nach der Kneipe ich befahrt den junge Mann zu ...        A   \n",
       "2  Lieber Ingo, Wie gehst dir? Wenn ich nur die b...        A   \n",
       "3  Er ist nicht verheiratet und hat keine Kinder....        A   \n",
       "4  Du wahrscheinlich hast kein Qualifikation für ...        A   \n",
       "\n",
       "                                            RFTagger  \\\n",
       "0  [[['Deine', 'PRO', 'Poss', 'Attr', '-', 'Nom',...   \n",
       "1  [[['Nach', 'APPR', 'Dat'], ['der', 'ART', 'Def...   \n",
       "2  [[['Lieber', 'N', 'Reg', 'Nom', 'Sg', 'Masc'],...   \n",
       "3  [[['Er', 'PRO', 'Pers', 'Subst', '3', 'Nom', '...   \n",
       "4  [[['Du', 'PRO', 'Pers', 'Subst', '2', 'Nom', '...   \n",
       "\n",
       "                                          parsedText  \\\n",
       "0  ['(ROOT (CS (S (NP (ART Deine) (CNP (NN Frau) ...   \n",
       "1  ['(ROOT (NUR (PP (APPR Nach) (ART der) (NN Kne...   \n",
       "2  ['(ROOT (S (S (ADJD Lieber) (NP (NE Ingo))) ($...   \n",
       "3  ['(ROOT (CS (S (PPER Er) (VAFIN ist) (VP (PTKN...   \n",
       "4  ['(ROOT (S (AP (PPER Du) (ADJD wahrscheinlich)...   \n",
       "\n",
       "                                    preprocessedText  \\\n",
       "0  [[['Deine', 'PRO', 'Poss', 'Attr', '-', 'Nom',...   \n",
       "1  [[['Nach', 'APPR', 'Dat'], ['der', 'ART', 'Def...   \n",
       "2  [[['Lieber', 'N', 'Reg', 'Nom', 'Sg', 'Masc'],...   \n",
       "3  [[['Er', 'PRO', 'Pers', 'Subst', '3', 'Nom', '...   \n",
       "4  [[['Du', 'PRO', 'Pers', 'Subst', '2', 'Nom', '...   \n",
       "\n",
       "   BF-average_sentence_length  BF-average_syllable_word  \\\n",
       "0                       8.250                  1.581818   \n",
       "1                      10.750                  1.565789   \n",
       "2                       8.125                  1.642857   \n",
       "3                      10.875                  1.597403   \n",
       "4                      10.625                  1.583333   \n",
       "\n",
       "   BF-Flesch_Reading_Ease_Score  BF-Flesch_Kincaid_readability  \\\n",
       "0                     64.639432                       7.281818   \n",
       "1                     63.457961                       8.089474   \n",
       "2                     59.602411                       7.964286   \n",
       "3                     60.656615                       8.518831   \n",
       "4                     62.100625                       8.250000   \n",
       "\n",
       "   BF-percentag_hard_words    ...     BF-average_character_per_word  \\\n",
       "0                 0.109091    ...                          4.800000   \n",
       "1                 0.078947    ...                          4.605263   \n",
       "2                 0.142857    ...                          4.785714   \n",
       "3                 0.129870    ...                          4.857143   \n",
       "4                 0.083333    ...                          4.513889   \n",
       "\n",
       "   BF-text_length    LD-ttr  LD-root_ttr  LD-corrected_ttr  LD-log_ttr  \\\n",
       "0              66  0.803030     0.896120          0.633652    0.947642   \n",
       "1              86  0.827586     0.909718          0.643268    0.957625   \n",
       "2              65  0.846154     0.919866          0.650444    0.959981   \n",
       "3              87  0.758621     0.870988          0.615882    0.938142   \n",
       "4              85  0.735632     0.857690          0.606478    0.931251   \n",
       "\n",
       "   LD-uber_index  LD-yule_s_k     LD-mtld    LD-hdd  \n",
       "0      38.553281   142.332415   66.880000  0.768626  \n",
       "1      43.348401    89.840137  111.543158  0.839343  \n",
       "2      37.780860    80.473373   84.500000  0.819016  \n",
       "3      44.248667   132.117849   62.929054  0.790037  \n",
       "4      44.576064   150.614348   42.270204  0.774356  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}