project_vectorizor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import html2text\n",
    "import tiktoken\n",
    "from datetime import datetime\n",
    "from langchain.document_loaders import JSONLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "def read_and_chunk(file_path):\n",
    "    text_array = []\n",
    "    id_array = []\n",
    "    meta_data_array = []\n",
    "\n",
    "    data = json.loads(Path(file_path).read_text())\n",
    "    h = html2text.HTML2Text()\n",
    "\n",
    "    for i in range(len(data)):\n",
    "        created_by = ' Created by: ' + data[i]['CreatedBy'] + ' ' if data[i]['CreatedBy'] else ' '    \n",
    "\n",
    "        date_time = datetime.fromisoformat(data[i]['CreateDate'].replace(\"Z\", \"+00:00\"))\n",
    "        date_string = date_time.strftime(\"%B %d, %Y\")\n",
    "        date_string_two = date_time.strftime(\"%d-%m-%Y\")\n",
    "        date_string_three = date_time.strftime(\"%Y/%m/%d\")\n",
    "        date_string_four = date_time.strftime(\"%A, the %d of %B in the year %Y\")\n",
    "\n",
    "        created_date = ' Project log created date: ' + date_string + ' - ' + date_string_two + ' - ' + date_string_three + ' - ' + date_string_four + ' '\n",
    "\n",
    "        short_description = data[i]['ShortDescription'] if data[i]['ShortDescription'] else  ' '\n",
    "\n",
    "        id = str(data[i]['NodeID'])\n",
    "\n",
    "        metadata = {\n",
    "            'project-number': '11448',\n",
    "            'id': id,\n",
    "            'update-date': created_date\n",
    "        }\n",
    "\n",
    "        content = h.handle(data[i]['Description']) + short_description\n",
    "        tokenizer = tiktoken.get_encoding('cl100k_base')\n",
    "\n",
    "        def tiktoken_len(text):\n",
    "            tokens = tokenizer.encode(\n",
    "                text,\n",
    "                disallowed_special=()\n",
    "            )\n",
    "            return len(tokens)\n",
    "        tiktoken_len(content)\n",
    "\n",
    "        text_splitter = RecursiveCharacterTextSplitter(\n",
    "            chunk_size=1000,\n",
    "            chunk_overlap=20,\n",
    "            length_function=tiktoken_len,\n",
    "            separators=[\"\\n\\n\", \"\\n\", \" \", \"\"]\n",
    "        )\n",
    "\n",
    "        chunks = text_splitter.split_text(content)\n",
    "        # iterate over the chunks and append the created date and created by to the end of the chunk\n",
    "        for j in range(len(chunks)):\n",
    "            compiled_chunk = created_by + created_date + chunks[j]\n",
    "            clean_chunk = compiled_chunk.replace(\"\\n\", \"; \").replace(\"  \", \" \")\n",
    "\n",
    "            text_array.append(clean_chunk)\n",
    "\n",
    "            id_array.append(id + '_' + str(j))\n",
    "\n",
    "            metadata['text'] = compiled_chunk\n",
    "            meta_data_array.append(metadata)\n",
    "    return id_array, text_array, meta_data_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "from time import sleep\n",
    "import openai\n",
    "import pinecone\n",
    "\n",
    "EMBED_MODEL = \"text-embedding-ada-002\"\n",
    "PINECONE_API_KEY = ''\n",
    "PINECONE_ENV = ''\n",
    "PINECONE_INDEX_NAME = ''\n",
    "PINECONE_DIMENSION = 1536\n",
    "OPENAI_API_KEY = \"\"\n",
    "\n",
    "openai.api_key = OPENAI_API_KEY\n",
    "\n",
    "batch_size = 100  # how many embeddings we create and insert at once\n",
    "\n",
    "def create_and_upload_embeddings(id_array, text_array, meta_array):\n",
    "    \n",
    "    for i in tqdm(range(0, len(id_array), batch_size)):\n",
    "        # find end of batch\n",
    "        i_end = min(len(id_array), i+batch_size)\n",
    "        meta_batch = meta_array[i:i_end]\n",
    "        # get ids\n",
    "        ids_batch = id_array[i:i_end]\n",
    "        # get texts to encode\n",
    "        texts = text_array[i:i_end]\n",
    "        # create embeddings (try-except added to avoid RateLimitError)\n",
    "        try:\n",
    "            res = openai.Embedding.create(input=texts, engine=EMBED_MODEL)\n",
    "        except:\n",
    "            done = False\n",
    "            while not done:\n",
    "                sleep(5)\n",
    "                try:\n",
    "                    res = openai.Embedding.create(input=texts, engine=EMBED_MODEL)\n",
    "                    done = True\n",
    "                except:\n",
    "                    pass\n",
    "        embeds = [record['embedding'] for record in res['data']]\n",
    "\n",
    "        to_upsert = list(zip(ids_batch, embeds, meta_batch))\n",
    "\n",
    "        pinecone.init(\n",
    "            api_key=PINECONE_API_KEY,\n",
    "            environment=PINECONE_ENV\n",
    "        )\n",
    "\n",
    "        if PINECONE_INDEX_NAME not in pinecone.list_indexes():\n",
    "            # create a new index if it doesn't exist\n",
    "            pinecone.create_index(\n",
    "                name=PINECONE_INDEX_NAME,\n",
    "                metric='cosine',\n",
    "                dimension=PINECONE_DIMENSION\n",
    "            )\n",
    "\n",
    "        index = pinecone.Index(PINECONE_INDEX_NAME)\n",
    "\n",
    "        # upsert to Pinecone\n",
    "        index.upsert(vectors=to_upsert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_array, text_array, meta_array = read_and_chunk('')\n",
    "create_and_upload_embeddings(id_array, text_array, meta_array)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}