5_Tokenization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "5. Tokenization.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyP2wdHbKcFM1DUqNdQNWoFF"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JbLC_nlI7-9w",
        "colab_type": "text"
      },
      "source": [
        "Melakukan split document berdasarkan kata atau kalimat, tergantung pada kebutuhan penelitian."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Flm8UTEN8Iao",
        "colab_type": "text"
      },
      "source": [
        "# 1. Menggunakan NLTK"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oEZz0Ob575hd",
        "colab_type": "code",
        "outputId": "a1ab37bb-2dd4-4c06-8648-9e96fb4a9b9e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLoloh1Z8QMP",
        "colab_type": "code",
        "outputId": "1a0a7494-1465-4939-dc20-2967438de93c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "kalimat_1 = \"Halo, teman-teman kita sekarang belajar stop word. bisa dilakukan per kata atau per kalimat\"\n",
        "print(\"Kata    :\" ,nltk.word_tokenize(kalimat_1)) # Word tokenization\n",
        "print(\"Kalimat :\" ,nltk.sent_tokenize(kalimat_1)) # Sentence tokenization"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Kata    : ['Halo', ',', 'teman-teman', 'kita', 'sekarang', 'belajar', 'stop', 'word', '.', 'bisa', 'dilakukan', 'per', 'kata', 'atau', 'per', 'kalimat']\n",
            "Kalimat : ['Halo, teman-teman kita sekarang belajar stop word.', 'bisa dilakukan per kata atau per kalimat']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mf7uQ3PZ8UBp",
        "colab_type": "code",
        "outputId": "3b3f0ce7-ac23-4132-a247-6b8b8ecdd7a7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "# menggunakan variabel fungsi untuk per kata\n",
        "def word_tokenization(inisial_argument):\n",
        "    item_kata = nltk.word_tokenize(inisial_argument)\n",
        "    return item_kata\n",
        "    \n",
        "kalimat_2 = \"Saya suka belajar. Karena ingin menjadi pintar. Selain itu, saya ingin membuat bahagia kedua orang tua.\"\n",
        "word_tokenization(kalimat_2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Saya',\n",
              " 'suka',\n",
              " 'belajar',\n",
              " '.',\n",
              " 'Karena',\n",
              " 'ingin',\n",
              " 'menjadi',\n",
              " 'pintar',\n",
              " '.',\n",
              " 'Selain',\n",
              " 'itu',\n",
              " ',',\n",
              " 'saya',\n",
              " 'ingin',\n",
              " 'membuat',\n",
              " 'bahagia',\n",
              " 'kedua',\n",
              " 'orang',\n",
              " 'tua',\n",
              " '.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qpl69Ro08Z5-",
        "colab_type": "code",
        "outputId": "17e80435-4a2a-44a8-c7dd-07b3fbfec31b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "# menggunakan variabel fungsi untuk per kalimat\n",
        "def sentence_tokenization(nama_argument):\n",
        "    item_kalimat = nltk.sent_tokenize(nama_argument)    \n",
        "    return item_kalimat\n",
        "\n",
        "kalimat_3 = \"Saya suka belajar. Karena ingin menjadi pintar. Selain itu, saya ingin membuat bahagia kedua orang tua.\"\n",
        "sentence_tokenization(kalimat_3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Saya suka belajar.',\n",
              " 'Karena ingin menjadi pintar.',\n",
              " 'Selain itu, saya ingin membuat bahagia kedua orang tua.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v1h-472l8goz",
        "colab_type": "text"
      },
      "source": [
        "# 2. Menggunakan SPACY"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZeXdMWaC8kEw",
        "colab_type": "code",
        "outputId": "7c63e492-eeba-40e1-f43b-d0e5bbf48db6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# SPACY English\n",
        "\n",
        "import spacy\n",
        "nlp_spacy_en = spacy.load('en')\n",
        "\n",
        "kalimat_4 = \"Hello, Mr. Man. He smiled!! This, i.e. that, is it.\"\n",
        "word = nlp_spacy_en(kalimat_4)\n",
        "print(\"Kata    :\" ,[token.text for token in word] ) # word segmentation\n",
        "\n",
        "sentences = nlp_spacy_en(kalimat_4).sents\n",
        "print(\"Kalimat :\" ,[str(sentence) for sentence in sentences] ) # sentence segmentation"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Kata    : ['Hello', ',', 'Mr.', 'Man', '.', 'He', 'smiled', '!', '!', 'This', ',', 'i.e.', 'that', ',', 'is', 'it', '.']\n",
            "Kalimat : ['Hello, Mr. Man.', 'He smiled!!', 'This, i.e. that, is it.']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uFajY1jd8qZw",
        "colab_type": "code",
        "outputId": "bdbc7201-a99f-41ae-b7af-a0a723ced871",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# SPACY bahasa Indonesia\n",
        "\n",
        "from spacy.lang.id import Indonesian\n",
        "nlp_spacy_id = Indonesian()\n",
        "\n",
        "kalimat_5 = nlp_spacy_id('Kemarin, saya melihat kupu-kupu di taman.')\n",
        "print(\"Kata :\" ,[token.text for token in kalimat_5])\n",
        "\n",
        "\n",
        "sentencizer = nlp_spacy_id.create_pipe(\"sentencizer\")\n",
        "nlp_spacy_id.add_pipe(sentencizer)\n",
        "kalimat_6 = nlp_spacy_id('Baru seminggu yang lalu, saya belajar python.')\n",
        "kalimat_baru = []\n",
        "for sent in kalimat_6.sents:\n",
        "  kalimat_baru.append(sent)\n",
        "print(\"Kalimat :\" ,kalimat_baru)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Kata : ['Kemarin', ',', 'saya', 'melihat', 'kupu-kupu', 'di', 'taman', '.']\n",
            "Kalimat : [Baru seminggu yang lalu, saya belajar python.]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DM0txWAc8ziY",
        "colab_type": "text"
      },
      "source": [
        "# 3. Menggunakan python SPLIT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xZUSH4aV82O5",
        "colab_type": "code",
        "outputId": "1aa0e0d4-4711-4b24-8e98-4c6372c1f8b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "kalimat_7 = \"Kalau yang ini menggunakan fungsi bawaan python yaitu split\"\n",
        "print(kalimat_7.split())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Kalau', 'yang', 'ini', 'menggunakan', 'fungsi', 'bawaan', 'python', 'yaitu', 'split']\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}