nlp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMvpNVU1WX3dNxlYfNMaUuq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shubh4545/Assignment/blob/main/nlp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N3cySMbupdio",
        "outputId": "29486553-b816-4c36-b0fa-96035681ae2a"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.29.2-py3-none-any.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m69.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m84.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.14.1 tokenizers-0.13.3 transformers-4.29.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install PyPDF2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H79yVGN6omaZ",
        "outputId": "49f8f1ee-1e02-4efc-fabe-de85ed102928"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: PyPDF2 in /usr/local/lib/python3.10/dist-packages (3.0.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "import PyPDF2\n",
        "import transformers\n",
        "import random\n",
        "from nltk.tokenize import sent_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from transformers import pipeline\n",
        "\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cdyT5KiVl9ua",
        "outputId": "dba4bcb6-7dc5-47e6-9a0e-d3fa63265eb6"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example usage\n",
        "\n",
        "pdf_text = read_pdf(\"chapter-2.pdf\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377
        },
        "id": "oF9fehEVqm4R",
        "outputId": "0d03d664-25a0-49cb-f57a-7ebd3083e01c"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:PyPDF2._reader:incorrect startxref pointer(1)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-7745530cf71a>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Example usage\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mpdf_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_pdf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"chapter-2.pdf\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-18-cdcb0d351823>\u001b[0m in \u001b[0;36mread_pdf\u001b[0;34m(file_path)\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mreader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPdfReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0mnum_pages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mpage_num\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_pages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/PyPDF2/_page.py\u001b[0m in \u001b[0;36m__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2061\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2062\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2063\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlength_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2064\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2065\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mPageObject\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/PyPDF2/_reader.py\u001b[0m in \u001b[0;36m_get_num_pages\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    446\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflattened_pages\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    449\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflattened_pages\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    450\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/PyPDF2/_reader.py\u001b[0m in \u001b[0;36m_flatten\u001b[0;34m(self, pages, inherit, indirect_reference)\u001b[0m\n\u001b[1;32m   1099\u001b[0m             \u001b[0;31m# Fix issue 327: set flattened_pages attribute only for\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1100\u001b[0m             \u001b[0;31m# decrypted file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1101\u001b[0;31m             \u001b[0mcatalog\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrailer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mROOT\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1102\u001b[0m             \u001b[0mpages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcatalog\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"/Pages\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1103\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflattened_pages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/PyPDF2/generic/_data_structures.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mPdfObject\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: '/Root'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from PyPDF2 import PdfReader\n",
        "\n",
        "def read_pdf(file_path):\n",
        "    text = \"\"\n",
        "    with open(file_path, \"rb\") as file:\n",
        "        reader = PdfReader(file)\n",
        "        num_pages = len(reader.pages)\n",
        "        \n",
        "        for page_num in range(num_pages):\n",
        "            page = reader.pages[page_num]\n",
        "            text += page.extract_text()\n",
        "    \n",
        "    return text\n",
        "\n"
      ],
      "metadata": {
        "id": "3rDsv0VToegL"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_mca_questions(pdf_text):\n",
        "    # Tokenize the text into sentences\n",
        "    sentences = sent_tokenize(context)\n",
        "    \n",
        "    # Preprocess the sentences\n",
        "    processed_sentences = preprocess_sentences(sentences)\n",
        "    \n",
        "    # Use TF-IDF to find the most important sentence\n",
        "    important_sentence = get_most_important_sentence(processed_sentences)\n",
        "    \n",
        "    # Generate multiple-choice questions based on the important sentence\n",
        "    mca_questions = generate_mca_questions(important_sentence)\n",
        "    \n",
        "    return mca_questions\n",
        "\n",
        "def preprocess_sentences(sentences):\n",
        "    stop_words = set(stopwords.words('english'))\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    processed_sentences = []\n",
        "    \n",
        "    for sentence in sentences:\n",
        "        # Tokenize the sentence into words\n",
        "        words = nltk.word_tokenize(sentence.lower())\n",
        "        \n",
        "        # Remove stop words and lemmatize the words\n",
        "        words = [lemmatizer.lemmatize(word) for word in words if word.isalnum() and word not in stop_words]\n",
        "        \n",
        "        # Convert the words back to sentence\n",
        "        processed_sentence = ' '.join(words)\n",
        "        \n",
        "        processed_sentences.append(processed_sentence)\n",
        "    \n",
        "    return processed_sentences\n",
        "\n",
        "def get_most_important_sentence(processed_sentences):\n",
        "    # Initialize TF-IDF vectorizer\n",
        "    vectorizer = TfidfVectorizer()\n",
        "    \n",
        "    # Compute TF-IDF matrix\n",
        "    tfidf_matrix = vectorizer.fit_transform(processed_sentences)\n",
        "    \n",
        "    # Compute cosine similarity between sentences\n",
        "    similarity_matrix = cosine_similarity(tfidf_matrix)\n",
        "    \n",
        "    # Find the most important sentence based on cosine similarity\n",
        "    important_sentence_index = similarity_matrix.sum(axis=1).argmax()\n",
        "    \n",
        "    return processed_sentences[important_sentence_index]\n",
        "\n",
        "\n",
        "\n",
        "def generate_mca_questions(important_sentence):\n",
        "    # Initialize OpenAI GPT-3.5 pipeline for question generation\n",
        "    generator = pipeline('text-generation', model='EleutherAI/gpt-neo-1.3B')\n",
        "\n",
        "    # Generate multiple-choice questions based on the important sentence\n",
        "    num_questions = 3\n",
        "    questions = []\n",
        "    for _ in range(num_questions):\n",
        "        # Generate the question\n",
        "        question = generator(important_sentence, max_length=30, num_return_sequences=1)[0]['generated_text']\n",
        "        question = question.split('?')[0] + '?'\n",
        "\n",
        "        # Generate the options\n",
        "        options = generate_options(important_sentence)\n",
        "        \n",
        "        # Randomly select the correct answers\n",
        "        correct_answers = random.sample(options, 2)\n",
        "        \n",
        "        # Create the correct options string\n",
        "        correct_options = \"Correct Options: \"\n",
        "        for i, answer in enumerate(correct_answers, 1):\n",
        "            correct_options += f\"({chr(97 + i)}) {answer} \"\n",
        "        \n",
        "        # Shuffle the options\n",
        "        random.shuffle(options)\n",
        "        \n",
        "        # Create the question string with shuffled options\n",
        "        question_with_options = question + '\\n'\n",
        "        for i, option in enumerate(options, 1):\n",
        "            question_with_options += f\"{chr(97 + i)}. {option}\\n\"\n",
        "        \n",
        "        # Append the question to the list\n",
        "        questions.append((question_with_options, correct_options))\n",
        "    \n",
        "    return questions\n",
        "\n",
        "\n",
        "def generate_options(important_sentence):\n",
        "    # Split the important sentence into words\n",
        "    words = important_sentence.split()\n",
        "    \n",
        "    # Shuffle the words\n",
        "    random.shuffle(words)\n",
        "    \n",
        "    # Select two words as correct answers\n",
        "    correct_answers = random.sample(words, 2)\n",
        "    \n",
        "    # Create options with two correct answers and two incorrect answers\n",
        "    options = correct_answers + random.sample([word for word in words if word not in correct_answers], 2)\n",
        "    \n",
        "    # Shuffle the options\n",
        "    random.shuffle(options)\n",
        "    \n",
        "    return options\n",
        "\n",
        "# Example usage\n",
        "context = \"Photosynthesis is a process used by plants and other organisms to convert light energy into chemical energy that, through cellular respiration, can later be released to fuel the organism's activities. Some of this chemical energy is stored in carbohydrate molecules, such as sugars and starches, which are synthesized from carbon dioxide and water. Most plants, algae, and cyanobacteria perform photosynthesis.\"\n",
        "\n",
        "mca_questions = get_mca_questions(context)\n",
        "for i, (question, correct_options) in enumerate(mca_questions, 1):\n",
        "    print(f\"Q{i}: {question}\")\n",
        "    print(f\"{correct_options}\\n\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ct_f3lSJepAK",
        "outputId": "0ea2ceae-6911-4503-a55b-2cf02ed8e10f"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Q1: photosynthesis process used plant organism convert light energy chemical energy cellular respiration later released fuel organism activity and maintenance. These processes are important and must be maintained?\n",
            "b. cellular\n",
            "c. respiration\n",
            "d. organism\n",
            "e. chemical\n",
            "\n",
            "Correct Options: (b) respiration (c) chemical \n",
            "\n",
            "Q2: photosynthesis process used plant organism convert light energy chemical energy cellular respiration later released fuel organism activity. This phenomenon is an important process to plant life process?\n",
            "b. convert\n",
            "c. organism\n",
            "d. activity\n",
            "e. chemical\n",
            "\n",
            "Correct Options: (b) activity (c) organism \n",
            "\n",
            "Q3: photosynthesis process used plant organism convert light energy chemical energy cellular respiration later released fuel organism activity carbon dioxide CO2\n",
            "\n",
            "This will cause to make?\n",
            "b. process\n",
            "c. activity\n",
            "d. cellular\n",
            "e. organism\n",
            "\n",
            "Correct Options: (b) cellular (c) activity \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.tokenize import sent_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from transformers import pipeline\n",
        "import pdfplumber\n",
        "\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "\n",
        "def get_mca_questions(context):\n",
        "    # Tokenize the text into sentences\n",
        "    sentences = sent_tokenize(context)\n",
        "    \n",
        "    # Preprocess the sentences\n",
        "    processed_sentences = preprocess_sentences(sentences)\n",
        "    \n",
        "    # Use TF-IDF to find the most important sentence\n",
        "    important_sentence = get_most_important_sentence(processed_sentences)\n",
        "    \n",
        "    # Generate multiple-choice questions based on the important sentence\n",
        "    mca_questions = generate_mca_questions(important_sentence)\n",
        "    \n",
        "    return mca_questions\n",
        "\n",
        "def preprocess_sentences(sentences):\n",
        "    stop_words = set(stopwords.words('english'))\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    processed_sentences = []\n",
        "    \n",
        "    for sentence in sentences:\n",
        "        # Tokenize the sentence into words\n",
        "        words = nltk.word_tokenize(sentence.lower())\n",
        "        \n",
        "        # Remove stop words and lemmatize the words\n",
        "        words = [lemmatizer.lemmatize(word) for word in words if word.isalnum() and word not in stop_words]\n",
        "        \n",
        "        # Convert the words back to a sentence\n",
        "        processed_sentence = ' '.join(words)\n",
        "        \n",
        "        processed_sentences.append(processed_sentence)\n",
        "    \n",
        "    return processed_sentences\n",
        "\n",
        "def get_most_important_sentence(processed_sentences):\n",
        "    # Initialize TF-IDF vectorizer\n",
        "    vectorizer = TfidfVectorizer()\n",
        "    \n",
        "    # Compute TF-IDF matrix\n",
        "    tfidf_matrix = vectorizer.fit_transform(processed_sentences)\n",
        "    \n",
        "    # Compute cosine similarity between sentences\n",
        "    similarity_matrix = cosine_similarity(tfidf_matrix)\n",
        "    \n",
        "    # Find the most important sentence based on cosine similarity\n",
        "    important_sentence_index = similarity_matrix.sum(axis=1).argmax()\n",
        "    \n",
        "    return processed_sentences[important_sentence_index]\n",
        "\n",
        "def generate_mca_questions(important_sentence):\n",
        "    # Initialize OpenAI GPT-3.5 pipeline for question generation\n",
        "    generator = pipeline('text-generation', model='EleutherAI/gpt-neo-1.3B')\n",
        "    \n",
        "    # Generate multiple-choice questions based on the important sentence\n",
        "    num_questions = 3\n",
        "    questions = []\n",
        "    for _ in range(num_questions):\n",
        "        question = generator(important_sentence, max_length=30, num_return_sequences=1)[0]['generated_text']\n",
        "        questions.append(question.split('?')[0] + '?')\n",
        "    \n",
        "    return questions\n",
        "\n",
        "def read_pdf(file_path):\n",
        "    text = \"\"\n",
        "    with pdfplumber.open(file_path) as pdf:\n",
        "        for page in pdf.pages:\n",
        "            extracted_text = page.extract_text()\n",
        "            text += extracted_text + \"\\n\"\n",
        "    \n",
        "    return text\n",
        "\n",
        "# Example usage\n",
        "file_path = \"/content/chapter-2.pdf\"\n",
        "pdf_text = read_pdf(file_path)\n",
        "\n",
        "mca_questions = get_mca_questions(pdf_text)\n",
        "for i, question in enumerate(mca_questions, 1):\n",
        "    print(f\"Q{i}: {question}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 505
        },
        "id": "BGZVlpZfuwv6",
        "outputId": "1a4357a0-2ddc-40cc-f50a-ec363fb2e631"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "PDFSyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mPDFSyntaxError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-074590b96106>\u001b[0m in \u001b[0;36m<cell line: 87>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[0;31m# Example usage\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0mfile_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"/content/chapter-2.pdf\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m \u001b[0mpdf_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_pdf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m \u001b[0mmca_questions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_mca_questions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpdf_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-25-074590b96106>\u001b[0m in \u001b[0;36mread_pdf\u001b[0;34m(file_path)\u001b[0m\n\u001b[1;32m     76\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_pdf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m     \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0mpdfplumber\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpdf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mpage\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpages\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m             \u001b[0mextracted_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pdfplumber/pdf.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(cls, path_or_fp, pages, laparams, password, strict_metadata)\u001b[0m\n\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m             return cls(\n\u001b[0m\u001b[1;32m     79\u001b[0m                 \u001b[0mstream\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpages\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pdfplumber/pdf.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, stream, stream_is_external, pages, laparams, password, strict_metadata)\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlaparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlaparams\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mLAParams\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mlaparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPDFDocument\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPDFParser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpassword\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpassword\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrsrcmgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPDFResourceManager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetadata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pdfminer/pdfdocument.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, parser, password, caching, fallback)\u001b[0m\n\u001b[1;32m    750\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    751\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 752\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mPDFSyntaxError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"No /Root object! - Is this really a PDF?\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    753\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatalog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Type\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mLITERAL_CATALOG\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    754\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0msettings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSTRICT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mPDFSyntaxError\u001b[0m: No /Root object! - Is this really a PDF?"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install pdfplumber\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 526
        },
        "id": "QvYyXcOIu9R8",
        "outputId": "420ef991-9324-4b7e-c3ee-32e9d5f3a6a1"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pdfplumber\n",
            "  Downloading pdfplumber-0.9.0-py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.1/46.1 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pdfminer.six==20221105 (from pdfplumber)\n",
            "  Downloading pdfminer.six-20221105-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m27.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting Pillow>=9.1 (from pdfplumber)\n",
            "  Downloading Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting Wand>=0.6.10 (from pdfplumber)\n",
            "  Downloading Wand-0.6.11-py2.py3-none-any.whl (143 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.6/143.6 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six==20221105->pdfplumber) (2.0.12)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six==20221105->pdfplumber) (40.0.2)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six==20221105->pdfplumber) (1.15.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six==20221105->pdfplumber) (2.21)\n",
            "Installing collected packages: Wand, Pillow, pdfminer.six, pdfplumber\n",
            "  Attempting uninstall: Pillow\n",
            "    Found existing installation: Pillow 8.4.0\n",
            "    Uninstalling Pillow-8.4.0:\n",
            "      Successfully uninstalled Pillow-8.4.0\n",
            "Successfully installed Pillow-9.5.0 Wand-0.6.11 pdfminer.six-20221105 pdfplumber-0.9.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    }
  ]
}