rag-mistral.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea832f87-92b1-4b9f-81c4-baacde5fa3eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T13:33:03.166951Z",
     "iopub.status.busy": "2024-08-06T13:33:03.166566Z",
     "iopub.status.idle": "2024-08-06T13:33:10.415573Z",
     "shell.execute_reply": "2024-08-06T13:33:10.414779Z",
     "shell.execute_reply.started": "2024-08-06T13:33:03.166921Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting flash-attn\n",
      "  Using cached flash_attn-2.6.3-cp39-cp39-linux_x86_64.whl\n",
      "Collecting tiktoken\n",
      "  Using cached tiktoken-0.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
      "Requirement already satisfied: einops in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (0.7.0)\n",
      "Requirement already satisfied: triton in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (2.1.0)\n",
      "Collecting python-dotenv\n",
      "  Using cached python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n",
      "Collecting pytest\n",
      "  Using cached pytest-8.3.2-py3-none-any.whl.metadata (7.5 kB)\n",
      "Requirement already satisfied: torch in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from flash-attn) (2.1.2)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from tiktoken) (2023.12.25)\n",
      "Requirement already satisfied: requests>=2.26.0 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from tiktoken) (2.31.0)\n",
      "Requirement already satisfied: filelock in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from triton) (3.13.1)\n",
      "Collecting iniconfig (from pytest)\n",
      "  Using cached iniconfig-2.0.0-py3-none-any.whl.metadata (2.6 kB)\n",
      "Requirement already satisfied: packaging in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from pytest) (23.2)\n",
      "Collecting pluggy<2,>=1.5 (from pytest)\n",
      "  Using cached pluggy-1.5.0-py3-none-any.whl.metadata (4.8 kB)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from pytest) (1.2.0)\n",
      "Collecting tomli>=1 (from pytest)\n",
      "  Using cached tomli-2.0.1-py3-none-any.whl.metadata (8.9 kB)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (2.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (2023.11.17)\n",
      "Requirement already satisfied: typing-extensions in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (4.9.0)\n",
      "Requirement already satisfied: sympy in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (1.12)\n",
      "Requirement already satisfied: networkx in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (3.1.3)\n",
      "Requirement already satisfied: fsspec in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (2024.6.1)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (2.18.1)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->flash-attn) (12.3.101)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from jinja2->torch->flash-attn) (2.1.4)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/saturncloud/envs/saturn/lib/python3.9/site-packages (from sympy->torch->flash-attn) (1.3.0)\n",
      "Using cached tiktoken-0.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "Using cached python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
      "Using cached pytest-8.3.2-py3-none-any.whl (341 kB)\n",
      "Using cached pluggy-1.5.0-py3-none-any.whl (20 kB)\n",
      "Using cached tomli-2.0.1-py3-none-any.whl (12 kB)\n",
      "Using cached iniconfig-2.0.0-py3-none-any.whl (5.9 kB)\n",
      "Installing collected packages: tomli, python-dotenv, pluggy, iniconfig, tiktoken, pytest, flash-attn\n",
      "Successfully installed flash-attn-2.6.3 iniconfig-2.0.0 pluggy-1.5.0 pytest-8.3.2 python-dotenv-1.0.1 tiktoken-0.7.0 tomli-2.0.1\n"
     ]
    }
   ],
   "source": [
    "! pip install flash-attn tiktoken einops triton python-dotenv pytest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b68e562c-4b89-479a-9b60-72991e113e51",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T13:33:11.961122Z",
     "iopub.status.busy": "2024-08-06T13:33:11.960687Z",
     "iopub.status.idle": "2024-08-06T13:33:14.961005Z",
     "shell.execute_reply": "2024-08-06T13:33:14.960340Z",
     "shell.execute_reply.started": "2024-08-06T13:33:11.961088Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "from minsearch import Index\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "os.environ[\"HF_HOME\"] = \"/run/cache/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "556ca6fb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T13:34:10.087819Z",
     "iopub.status.busy": "2024-08-06T13:34:10.087272Z",
     "iopub.status.idle": "2024-08-06T13:34:10.651841Z",
     "shell.execute_reply": "2024-08-06T13:34:10.650945Z",
     "shell.execute_reply.started": "2024-08-06T13:34:10.087778Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
      "Token is valid (permission: read).\n",
      "Your token has been saved to /run/cache/token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import login\n",
    "\n",
    "login(token = os.environ['MISTRAL_TOKEN'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eaa388d7-6da8-4751-8106-c335b971e183",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T13:39:18.414951Z",
     "iopub.status.busy": "2024-08-06T13:39:18.414553Z",
     "iopub.status.idle": "2024-08-06T13:41:03.844226Z",
     "shell.execute_reply": "2024-08-06T13:41:03.843595Z",
     "shell.execute_reply.started": "2024-08-06T13:39:18.414924Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead.\n",
      "`low_cpu_mem_usage` was None, now set to True since model is quantized.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fbfb69fbc204b4fb23276347ecd93ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f9a72545dc44825a1a1c57c43600200",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/996 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5fca6f38a614337a03b8e5181dbcda2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9a7764a5e9f46078777e8b7ef6d69b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "902c070b3059406fabc69d40cdfb0a3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/414 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline\n",
    "\n",
    "torch.random.manual_seed(0)\n",
    "model_id = \"mistralai/Mistral-7B-v0.1\"\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id, \n",
    "    torch_dtype=\"auto\",\n",
    "  \n",
    "    \n",
    "   load_in_4bit = True\n",
    ")\n",
    "assert torch.cuda.is_available(), \"This model needs a GPU to run ...\"\n",
    "device = torch.cuda.current_device()\n",
    "#model = model.to(device)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id, padding_side = \"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1c177f03-5f3c-4f15-920c-2cb39ffe180c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T13:56:06.025102Z",
     "iopub.status.busy": "2024-08-06T13:56:06.024679Z",
     "iopub.status.idle": "2024-08-06T13:56:06.029514Z",
     "shell.execute_reply": "2024-08-06T13:56:06.028736Z",
     "shell.execute_reply.started": "2024-08-06T13:56:06.025075Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True 0\n"
     ]
    }
   ],
   "source": [
    "print(torch.cuda.is_available(), torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d114f4d2-391c-4b32-9a08-d4c851c944c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:00:53.054922Z",
     "iopub.status.busy": "2024-08-06T14:00:53.054520Z",
     "iopub.status.idle": "2024-08-06T14:00:53.058705Z",
     "shell.execute_reply": "2024-08-06T14:00:53.057942Z",
     "shell.execute_reply.started": "2024-08-06T14:00:53.054897Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.backends.cuda.enable_mem_efficient_sdp(False)\n",
    "torch.backends.cuda.enable_flash_sdp(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7939d0ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:04:33.866056Z",
     "iopub.status.busy": "2024-08-06T14:04:33.865650Z",
     "iopub.status.idle": "2024-08-06T14:04:37.316809Z",
     "shell.execute_reply": "2024-08-06T14:04:37.316087Z",
     "shell.execute_reply.started": "2024-08-06T14:04:33.866029Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    }
   ],
   "source": [
    "prompt = \"Do you love me ?\"\n",
    "\n",
    "\n",
    "inputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\n",
    "\n",
    "generate_ids = model.generate(**inputs, max_length = 60)\n",
    "\n",
    "ans = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7dced17e-4054-44eb-a20c-cb3f12cbb02d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:04:37.318652Z",
     "iopub.status.busy": "2024-08-06T14:04:37.318226Z",
     "iopub.status.idle": "2024-08-06T14:04:37.323941Z",
     "shell.execute_reply": "2024-08-06T14:04:37.323191Z",
     "shell.execute_reply.started": "2024-08-06T14:04:37.318621Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Do you love me ?\\n\\nI’m not sure if I’m a good person.\\n\\nI’m not sure if I’m a bad person.\\n\\nI’m not sure if I’m a good person.\\n\\nI’m not sure if I'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "7bde8add-3e10-4aaf-84e8-4ae1eefb538e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:54:24.309930Z",
     "iopub.status.busy": "2024-08-06T14:54:24.309523Z",
     "iopub.status.idle": "2024-08-06T14:54:24.313672Z",
     "shell.execute_reply": "2024-08-06T14:54:24.313003Z",
     "shell.execute_reply.started": "2024-08-06T14:54:24.309905Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "generator = pipeline(\"text-generation\", model = model, tokenizer = tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a7e8f677-a5a5-4786-acbd-a26f2d159a6d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:31:36.553065Z",
     "iopub.status.busy": "2024-08-06T14:31:36.552616Z",
     "iopub.status.idle": "2024-08-06T14:31:36.562167Z",
     "shell.execute_reply": "2024-08-06T14:31:36.561524Z",
     "shell.execute_reply.started": "2024-08-06T14:31:36.553036Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "documents = []\n",
    "\n",
    "with open('documents.json', 'r') as file:\n",
    "\n",
    "    docs = json.load(file)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "74440260-3757-44b1-acb8-41c66f3533b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:31:39.102939Z",
     "iopub.status.busy": "2024-08-06T14:31:39.102544Z",
     "iopub.status.idle": "2024-08-06T14:31:39.107192Z",
     "shell.execute_reply": "2024-08-06T14:31:39.106532Z",
     "shell.execute_reply.started": "2024-08-06T14:31:39.102915Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for doc in docs:\n",
    "\n",
    "    for document in doc['documents']:\n",
    "\n",
    "        document['course'] = doc['course'] \n",
    "        documents.append(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "97d96165-f38e-42f1-99ab-2da82d78c452",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:32:27.742178Z",
     "iopub.status.busy": "2024-08-06T14:32:27.741782Z",
     "iopub.status.idle": "2024-08-06T14:32:27.848941Z",
     "shell.execute_reply": "2024-08-06T14:32:27.848111Z",
     "shell.execute_reply.started": "2024-08-06T14:32:27.742154Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<minsearch.Index at 0x7f4d324b1eb0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textfields = [\"text\", \"section\", \"question\"]\n",
    "\n",
    "\n",
    "indobject = Index(text_fields = textfields, keyword_fields = ['course'])\n",
    "indobject.fit(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "65550553-6d4a-49b9-857a-b2bc29a012d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T14:32:31.282936Z",
     "iopub.status.busy": "2024-08-06T14:32:31.282541Z",
     "iopub.status.idle": "2024-08-06T14:32:31.287313Z",
     "shell.execute_reply": "2024-08-06T14:32:31.286448Z",
     "shell.execute_reply.started": "2024-08-06T14:32:31.282911Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def search(query, boost_dict = {\"question\": 3}, filter_dict =  {\"course\":\"mlops-zoomcamp\"}, num_results = 5 ):\n",
    "\n",
    "    context = indobject.search(query = query, boost_dict = boost_dict, filter_dict = filter_dict, num_results = num_results\n",
    "                    )\n",
    "    \n",
    "    return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "ccce04cf-887e-4f49-8059-5c3c4aad55db",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T15:12:11.289926Z",
     "iopub.status.busy": "2024-08-06T15:12:11.289520Z",
     "iopub.status.idle": "2024-08-06T15:12:11.294395Z",
     "shell.execute_reply": "2024-08-06T15:12:11.293556Z",
     "shell.execute_reply.started": "2024-08-06T15:12:11.289900Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def build_prompt(query, related_docs):\n",
    "    \n",
    "    prompt = \"\"\"\n",
    "    Only generate the answer for the query based on the context given\n",
    "    \n",
    "    question:{query}\n",
    "    \n",
    "    context:{context}\n",
    "    \n",
    "    answer:\n",
    "    \n",
    "    \n",
    "    \"\"\".strip()\n",
    "    \n",
    "    \n",
    "    context = \"\"\n",
    "    for doc in related_docs:\n",
    "        \n",
    "        context += f\"question: {doc['question']}\\nanswer: {doc['text']} \\n\\n\"\n",
    "        \n",
    "    \n",
    "    prompt = prompt.format(query = query, context = context).strip()\n",
    "    \n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "94d2f3c7-8301-4c43-8296-ea6064527302",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T15:12:13.845234Z",
     "iopub.status.busy": "2024-08-06T15:12:13.844806Z",
     "iopub.status.idle": "2024-08-06T15:12:13.849608Z",
     "shell.execute_reply": "2024-08-06T15:12:13.848758Z",
     "shell.execute_reply.started": "2024-08-06T15:12:13.845207Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def chat(prompt, generation_args = {}):\n",
    "    \n",
    "    \n",
    "    \n",
    "    ans = generator(prompt.strip(), max_length = 900, temperature = 0.7, top_p = 0.95, num_return_sequences = 1) \n",
    "    print(ans)\n",
    "    \n",
    "    return ans[0]['generated_text']\n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "b24a2e2b-6352-4ff2-bd74-eac1fb422060",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T15:12:15.430068Z",
     "iopub.status.busy": "2024-08-06T15:12:15.429674Z",
     "iopub.status.idle": "2024-08-06T15:12:15.435669Z",
     "shell.execute_reply": "2024-08-06T15:12:15.434672Z",
     "shell.execute_reply.started": "2024-08-06T15:12:15.430041Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def rag(query, boost_dict = {\"question\": 3}, course_filter =  {\"course\": \"data-engineering-zoomcamp\"}, generation_args ={}):\n",
    "    \n",
    "    context = search(query = query, boost_dict = boost_dict, filter_dict = course_filter)\n",
    "    \n",
    "    prompt = build_prompt(query, context)\n",
    "    \n",
    "    answer = chat(prompt, generation_args = generation_args)\n",
    "    \n",
    "    return answer[len(prompt):]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "7a24b43a-3381-49f9-9790-832e4b5b05c4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T15:12:17.615841Z",
     "iopub.status.busy": "2024-08-06T15:12:17.615349Z",
     "iopub.status.idle": "2024-08-06T15:12:17.619566Z",
     "shell.execute_reply": "2024-08-06T15:12:17.618796Z",
     "shell.execute_reply.started": "2024-08-06T15:12:17.615810Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "query = \"How to run spark engine ?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "95975222-73b6-4cda-90c1-843cfe80f735",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T15:12:22.694918Z",
     "iopub.status.busy": "2024-08-06T15:12:22.694540Z",
     "iopub.status.idle": "2024-08-06T15:12:35.266091Z",
     "shell.execute_reply": "2024-08-06T15:12:35.265398Z",
     "shell.execute_reply.started": "2024-08-06T15:12:22.694892Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'generated_text': 'Only generate the answer for the query based on the context given\\n    \\n    question:How to run spark engine ?\\n    \\n    context:question: How to spark standalone cluster is run on windows OS\\nanswer: Change the working directory to the spark directory:\\nif you have setup up your SPARK_HOME variable, use the following;\\ncd %SPARK_HOME%\\nif not, use the following;\\ncd <path to spark installation>\\nCreating a Local Spark Cluster\\nTo start Spark Master:\\nbin\\\\spark-class org.apache.spark.deploy.master.Master --host localhost\\nStarting up a cluster:\\nbin\\\\spark-class org.apache.spark.deploy.worker.Worker spark://localhost:7077 --host localhost \\n\\nquestion: Docker engine stopped_failed to fetch extensions\\nanswer: The docker will keep on crashing continuously\\nNot working after restart\\ndocker engine stopped\\nAnd failed to fetch extensions pop ups will on screen non-stop\\nSolution :\\nTry checking if latest version of docker is installed / Try updating the docker\\nIf Problem still persist then final solution is to reinstall docker\\n(Just have to fetch images again else no issues) \\n\\nquestion: Run Local Cluster Spark in Windows 10 with CMD\\nanswer: Go to %SPARK_HOME%\\\\bin\\nRun spark-class org.apache.spark.deploy.master.Master to run the master. This will give you a URL of the form spark://ip:port\\nRun spark-class org.apache.spark.deploy.worker.Worker spark://ip:port to run the worker. Make sure you use the URL you obtained in step 2.\\nCreate a new Jupyter notebook:\\nspark = SparkSession.builder \\\\\\n.master(\"spark://{ip}:7077\") \\\\\\n.appName(\\'test\\') \\\\\\n.getOrCreate()\\nCheck on Spark UI the master, worker and app. \\n\\nquestion: Spark Streaming - How do I read from multiple topics in the same Spark Session\\nanswer: Initiate a Spark Session\\nspark = (SparkSession\\n.builder\\n.appName(app_name)\\n.master(master=master)\\n.getOrCreate())\\nspark.streams.resetTerminated()\\nquery1 = spark\\n.readStream\\n…\\n…\\n.load()\\nquery2 = spark\\n.readStream\\n…\\n…\\n.load()\\nquery3 = spark\\n.readStream\\n…\\n…\\n.load()\\nquery1.start()\\nquery2.start()\\nquery3.start()\\nspark.streams.awaitAnyTermination() #waits for any one of the query to receive kill signal or error failure. This is asynchronous\\n# On the contrary query3.start().awaitTermination() is a blocking ex call. Works well when we are reading only from one topic. \\n\\nquestion: Python Kafka: ./spark-submit.sh streaming.py - How to check why Spark master connection fails\\nanswer: Start a new terminal\\nRun: docker ps\\nCopy the CONTAINER ID of the spark-master container\\nRun: docker exec -it <spark_master_container_id> bash\\nRun: cat logs/spark-master.out\\nCheck for the log when the error happened\\nGoogle the error message from there \\n\\n\\n    \\n    answer:\\n    spark-submit --class org.apache.spark.sql.streaming.StreamingExamples \\\\\\n    --master yarn \\\\\\n    --deploy-mode cluster \\\\\\n    --executor-memory 1G \\\\\\n    --num-executors 1 \\\\\\n    --executor-cores 1 \\\\\\n    --queue default \\\\\\n    --files hdfs:///user/spark/examples/jars/spark-examples_2.11-2.3.0.jar \\\\\\n    --jars hdfs:///'}]\n"
     ]
    }
   ],
   "source": [
    "answer = rag(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "e0ab650c-6709-4082-b6c4-6264e221aa35",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T15:12:50.945358Z",
     "iopub.status.busy": "2024-08-06T15:12:50.944929Z",
     "iopub.status.idle": "2024-08-06T15:12:50.950184Z",
     "shell.execute_reply": "2024-08-06T15:12:50.949173Z",
     "shell.execute_reply.started": "2024-08-06T15:12:50.945333Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n    spark-submit --class org.apache.spark.sql.streaming.StreamingExamples \\\\\\n    --master yarn \\\\\\n    --deploy-mode cluster \\\\\\n    --executor-memory 1G \\\\\\n    --num-executors 1 \\\\\\n    --executor-cores 1 \\\\\\n    --queue default \\\\\\n    --files hdfs:///user/spark/examples/jars/spark-examples_2.11-2.3.0.jar \\\\\\n    --jars hdfs:///'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "d9ecb4a3-e5ab-4ae5-baf5-8e02117acb0a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-06T15:13:20.730604Z",
     "iopub.status.busy": "2024-08-06T15:13:20.730207Z",
     "iopub.status.idle": "2024-08-06T15:13:20.734420Z",
     "shell.execute_reply": "2024-08-06T15:13:20.733630Z",
     "shell.execute_reply.started": "2024-08-06T15:13:20.730577Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    spark-submit --class org.apache.spark.sql.streaming.StreamingExamples \\\n",
      "    --master yarn \\\n",
      "    --deploy-mode cluster \\\n",
      "    --executor-memory 1G \\\n",
      "    --num-executors 1 \\\n",
      "    --executor-cores 1 \\\n",
      "    --queue default \\\n",
      "    --files hdfs:///user/spark/examples/jars/spark-examples_2.11-2.3.0.jar \\\n",
      "    --jars hdfs:///\n"
     ]
    }
   ],
   "source": [
    "print(_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc1a6fa1-9f9c-41f7-8c4d-7c1d28974480",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "saturn (Python 3)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}