Kalevala.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fastai.text as fatext\n",
    "import numpy as np\n",
    "import sentencepiece as sp\n",
    "from typing import List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spm = sp.SentencePieceProcessor()\n",
    "spm.Load(\"iso-lc.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SPMTokenizer:\n",
    "\tdef __init__(self, lang: str):\n",
    "\t\tpass\n",
    "\tdef tokenizer(self, text: str) -> List[str]:\n",
    "\t\treturn spm.EncodeAsPieces(text.lower())\n",
    "\tdef add_special_cases(self, *args):\n",
    "\t\tpass\n",
    "\n",
    "tokenizer = fatext.transform.Tokenizer(SPMTokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadVocab(filename):\n",
    "\twith open(filename, \"r\") as f:\n",
    "\t\ttokens = [l.strip().split()[0] for l in f]\n",
    "\t\n",
    "\treturn fatext.transform.Vocab(tokens)\n",
    "\n",
    "vocab = loadVocab(\"iso-lc.vocab\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ensimmäinen runo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mieleni minun tekevi,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aivoni ajattelevi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>lähteäni laulamahan,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>saa'ani sanelemahan,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sukuvirttä suoltamahan,</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Ensimmäinen runo\n",
       "0    Mieleni minun tekevi,\n",
       "1        aivoni ajattelevi\n",
       "2     lähteäni laulamahan,\n",
       "3     saa'ani sanelemahan,\n",
       "4  sukuvirttä suoltamahan,"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = fatext.pd.read_csv(\"data/kalevala.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>▁xx bo s ▁vaka ▁vanha ▁väinämöinen ▁xx bo s ▁vai voja ▁veti stä jä llä ! ▁xx bo s ▁itsensä ▁metsän ▁emännän , ▁xx bo s ▁meren ▁pa uhun ▁parta ha lla . ▁xx bo s ▁antaisi pa ▁armo luo ja : ▁xx bo s ▁kulki ▁vuoren ▁kukkula lle ▁xx bo s ▁tuo ▁venettä , ▁tuon en ▁ty tti , ▁xx bo s ▁hurme ▁kosken a ▁koh isi :</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>▁ki uka ha sta , ▁xx bo s ▁susi ▁päälle ▁sui mista i kse , ▁xx bo s ▁pahan ▁pur ren ▁parra s puu ksi !\" ▁xx bo s ▁kate he lta ▁nukkuva lta !\" ▁xx bo s ▁\" tuo lla ▁ah tola ▁asuvi , ▁xx bo s ▁suo hon ▁su kkoa ▁vajo nnut . ▁xx bo s ▁laulu ▁laulu lle ▁teho si . ▁xx bo s ▁näillä ▁tuonela</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>bo s ▁v etelä isessä ▁veny vi ; ▁xx bo s ▁särki ▁lie kun ▁leh mu ksi sen , ▁xx bo s ▁jää tä ▁re ellä ▁reu to ello s ▁xx bo s ▁olipa ▁akka ▁lattialla , ▁xx bo s ▁jo ▁olen ▁joutunut ▁johon ki : ▁xx bo s ▁saa ka mme ▁san ele mahan , ▁xx bo s ▁kaunis ▁kasvo a ▁kana sen . ▁xx bo s ▁se</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>▁on ▁jää mähän ▁parempi ▁xx bo s ▁tuonne ▁ma ▁kana ▁kato an , ▁xx bo s ▁lauloi ▁lie to ▁lemminkäinen , ▁xx bo s ▁var sa ▁varrella ▁samo si , ▁xx bo s ▁kanta ja ni ▁kai ke ro itti : ▁xx bo s ▁vai ko ▁laiho a ▁li ho itti , ▁xx bo s ▁sinä ▁ilmoi s na ▁ikä nä , ▁xx bo s ▁sepä ▁kielen ▁kantele vi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>n ▁la ella , ▁xx bo s ▁nenä hän ▁ utui sen ▁niemen , ▁xx bo s ▁\" mi ▁minusta ▁laatima sta , ▁xx bo s ▁kun ▁et ▁itke ▁itke ttä issä , ▁xx bo s ▁karjalan ▁ kuninka ha ksi , ▁xx bo s ▁väärä s ääri ▁vääntä mähän , ▁xx bo s ▁helma ssa ▁he lo ttavi lla . ▁xx bo s ▁oisi ▁kahta ▁kauni hi mpi</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = fatext.data.TextLMDataBunch.from_csv(\".\", \"data/kalevala.csv\", text_cols=0, tokenizer=tokenizer, vocab=vocab)\n",
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (18296 items)\n",
       "x: LMTextList\n",
       "▁xx bo s ▁matka nsa ▁majo ilta ▁nä iltä ,,▁xx bo s ▁työn ▁pahan ▁pakene n ta han ,,▁xx bo s ▁uro ho ise hen ▁vä ke hen ,,▁xx bo s ▁mulle ▁nuorta ▁morsian ta !\",▁xx bo s ▁pyy t ▁leh ossa ▁pyr his tellä ;\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (4575 items)\n",
       "x: LMTextList\n",
       "▁xx bo s ▁alasti ▁ala ki ve llä ,,▁xx bo s ▁\" oh oh ▁lasta ▁lamp ah utta !,▁xx bo s ▁itse ▁seppo ▁ilmari nen,▁xx bo s ▁vievä t ▁vilja mättä hä ltä ,,▁xx bo s ▁itse ▁istu vi ▁re ke hen ,\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(24000, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(24000, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1150, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1150, 1150, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1150, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=24000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fde96c74048>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[RNNTrainer\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (18296 items)\n",
       "x: LMTextList\n",
       "▁xx bo s ▁matka nsa ▁majo ilta ▁nä iltä ,,▁xx bo s ▁työn ▁pahan ▁pakene n ta han ,,▁xx bo s ▁uro ho ise hen ▁vä ke hen ,,▁xx bo s ▁mulle ▁nuorta ▁morsian ta !\",▁xx bo s ▁pyy t ▁leh ossa ▁pyr his tellä ;\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (4575 items)\n",
       "x: LMTextList\n",
       "▁xx bo s ▁alasti ▁ala ki ve llä ,,▁xx bo s ▁\" oh oh ▁lasta ▁lamp ah utta !,▁xx bo s ▁itse ▁seppo ▁ilmari nen,▁xx bo s ▁vievä t ▁vilja mättä hä ltä ,,▁xx bo s ▁itse ▁istu vi ▁re ke hen ,\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(24000, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(24000, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1150, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1150, 1150, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1150, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=24000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fde96c74048>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(24000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(24000, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=24000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(24000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(24000, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=24000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner = fatext.learner.language_model_learner(data, fatext.models.AWD_LSTM, pretrained=False)\n",
    "learner.load(\"lstm2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 1:12:11 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.453632</td>\n",
       "      <td>4.211301</td>\n",
       "      <td>0.361674</td>\n",
       "      <td>08:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.156702</td>\n",
       "      <td>3.627470</td>\n",
       "      <td>0.454375</td>\n",
       "      <td>07:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.815404</td>\n",
       "      <td>3.435194</td>\n",
       "      <td>0.467321</td>\n",
       "      <td>07:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.589210</td>\n",
       "      <td>3.329134</td>\n",
       "      <td>0.476250</td>\n",
       "      <td>07:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.426903</td>\n",
       "      <td>3.264029</td>\n",
       "      <td>0.481295</td>\n",
       "      <td>07:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.319840</td>\n",
       "      <td>3.221448</td>\n",
       "      <td>0.485580</td>\n",
       "      <td>07:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.212602</td>\n",
       "      <td>3.193276</td>\n",
       "      <td>0.488281</td>\n",
       "      <td>07:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>3.136412</td>\n",
       "      <td>3.179980</td>\n",
       "      <td>0.490223</td>\n",
       "      <td>07:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.089723</td>\n",
       "      <td>3.173864</td>\n",
       "      <td>0.490647</td>\n",
       "      <td>07:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.062368</td>\n",
       "      <td>3.173602</td>\n",
       "      <td>0.490647</td>\n",
       "      <td>07:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "learner.fit_one_cycle(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      ". poika, mies, joka on varma siitä, ettei toinen välitä kuolemasta, hän alistuu luutastettuun suuhun ja sen jälkeen se toimii wurachin koulua. jossa hän itsepintaisesti, filosofisesti, itsekäs rohkeni lausunut minulle, owenin 2019 \"par wars\" (1962) on korvattiin kapearaiteiseksi 50 kilogramman painoiseksi viuruamerikkalaiseksi lyhytwheri-merkiksi. vuonna 1972 sim 1980 m/s papiksi tulivat \"/klerdori\". alus aloitti viktor ferdinandin–mmuksen vuonna 1959, ja sen pituus oli 45,3 km, tilavuus päältä 8,8s ohjeet mukaan lukien lisäpituus 889 mm. raamattu ( elokuvasta) provasti 16 034 siionin tähdistötietokoneissa on ns. punapallo tai kummituslyhteen valopaja. olla vielä lampunkantaja. onko etiikka homeikkoa niin mielijohteestansa kuin minäkin\". sillävälin hän ilmoitti verrata suurena mahdollisuus saada velaksi ja opettaa ihmiset tekemään oikean kiduttaa monet\n"
     ]
    }
   ],
   "source": [
    "print(learner.predict(\"xxbos\", 200).replace(\" \", \"\").replace(\"▁\", \" \").replace(\"xxbos\", \"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save(\"lstm2-kalevala\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}