db.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: chromadb in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (0.5.23)\n",
      "Requirement already satisfied: build>=1.0.3 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (1.2.2.post1)\n",
      "Requirement already satisfied: pydantic>=1.9 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (2.10.4)\n",
      "Requirement already satisfied: chroma-hnswlib==0.7.6 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (0.7.6)\n",
      "Requirement already satisfied: fastapi>=0.95.2 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (0.115.6)\n",
      "Requirement already satisfied: uvicorn>=0.18.3 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.34.0)\n",
      "Requirement already satisfied: numpy>=1.22.5 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (2.2.1)\n",
      "Requirement already satisfied: posthog>=2.4.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (3.7.4)\n",
      "Requirement already satisfied: typing_extensions>=4.5.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (4.12.2)\n",
      "Requirement already satisfied: onnxruntime>=1.14.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (1.20.1)\n",
      "Requirement already satisfied: opentelemetry-api>=1.2.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (1.29.0)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (1.29.0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (0.50b0)\n",
      "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (1.29.0)\n",
      "Requirement already satisfied: tokenizers<=0.20.3,>=0.13.2 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (0.20.3)\n",
      "Requirement already satisfied: pypika>=0.48.9 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (0.48.9)\n",
      "Requirement already satisfied: tqdm>=4.65.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (4.67.1)\n",
      "Requirement already satisfied: overrides>=7.3.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (7.7.0)\n",
      "Requirement already satisfied: importlib-resources in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (6.4.5)\n",
      "Requirement already satisfied: grpcio>=1.58.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (1.68.1)\n",
      "Requirement already satisfied: bcrypt>=4.0.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (4.2.1)\n",
      "Requirement already satisfied: typer>=0.9.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (0.15.1)\n",
      "Requirement already satisfied: kubernetes>=28.1.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (31.0.0)\n",
      "Requirement already satisfied: tenacity>=8.2.3 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (9.0.0)\n",
      "Requirement already satisfied: PyYAML>=6.0.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (6.0.2)\n",
      "Requirement already satisfied: mmh3>=4.0.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (5.0.1)\n",
      "Requirement already satisfied: orjson>=3.9.12 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (3.10.12)\n",
      "Requirement already satisfied: httpx>=0.27.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (0.28.1)\n",
      "Requirement already satisfied: rich>=10.11.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from chromadb) (13.9.4)\n",
      "Requirement already satisfied: packaging>=19.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from build>=1.0.3->chromadb) (24.2)\n",
      "Requirement already satisfied: pyproject_hooks in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from build>=1.0.3->chromadb) (1.2.0)\n",
      "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from fastapi>=0.95.2->chromadb) (0.41.3)\n",
      "Requirement already satisfied: anyio in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from httpx>=0.27.0->chromadb) (4.7.0)\n",
      "Requirement already satisfied: certifi in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from httpx>=0.27.0->chromadb) (2024.12.14)\n",
      "Requirement already satisfied: httpcore==1.* in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from httpx>=0.27.0->chromadb) (1.0.7)\n",
      "Requirement already satisfied: idna in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from httpx>=0.27.0->chromadb) (3.10)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from httpcore==1.*->httpx>=0.27.0->chromadb) (0.14.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (1.17.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (2.9.0.post0)\n",
      "Requirement already satisfied: google-auth>=1.0.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (2.37.0)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n",
      "Requirement already satisfied: requests in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (2.32.3)\n",
      "Requirement already satisfied: requests-oauthlib in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\n",
      "Requirement already satisfied: oauthlib>=3.2.2 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
      "Requirement already satisfied: urllib3>=1.24.2 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (2.3.0)\n",
      "Requirement already satisfied: durationpy>=0.7 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from kubernetes>=28.1.0->chromadb) (0.9)\n",
      "Requirement already satisfied: coloredlogs in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
      "Requirement already satisfied: flatbuffers in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from onnxruntime>=1.14.1->chromadb) (24.3.25)\n",
      "Requirement already satisfied: protobuf in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from onnxruntime>=1.14.1->chromadb) (5.29.2)\n",
      "Requirement already satisfied: sympy in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from onnxruntime>=1.14.1->chromadb) (1.13.3)\n",
      "Requirement already satisfied: deprecated>=1.2.6 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.15)\n",
      "Requirement already satisfied: importlib-metadata<=8.5.0,>=6.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-api>=1.2.0->chromadb) (8.5.0)\n",
      "Requirement already satisfied: googleapis-common-protos~=1.52 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.66.0)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.29.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.29.0)\n",
      "Requirement already satisfied: opentelemetry-proto==1.29.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.29.0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.50b0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.50b0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation==0.50b0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.50b0)\n",
      "Requirement already satisfied: opentelemetry-semantic-conventions==0.50b0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.50b0)\n",
      "Requirement already satisfied: opentelemetry-util-http==0.50b0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.50b0)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-instrumentation==0.50b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.17.0)\n",
      "Requirement already satisfied: asgiref~=3.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from opentelemetry-instrumentation-asgi==0.50b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (3.8.1)\n",
      "Requirement already satisfied: monotonic>=1.5 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from pydantic>=1.9->chromadb) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.27.2 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from pydantic>=1.9->chromadb) (2.27.2)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from rich>=10.11.0->chromadb) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from rich>=10.11.0->chromadb) (2.18.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from tokenizers<=0.20.3,>=0.13.2->chromadb) (0.27.0)\n",
      "Requirement already satisfied: click>=8.0.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from typer>=0.9.0->chromadb) (8.1.8)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from typer>=0.9.0->chromadb) (1.5.4)\n",
      "Requirement already satisfied: httptools>=0.6.3 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.4)\n",
      "Requirement already satisfied: python-dotenv>=0.13 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.1)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.21.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.3)\n",
      "Requirement already satisfied: websockets>=10.4 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (14.1)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.5.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.1)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
      "Requirement already satisfied: filelock in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers<=0.20.3,>=0.13.2->chromadb) (3.16.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers<=0.20.3,>=0.13.2->chromadb) (2024.12.0)\n",
      "Requirement already satisfied: zipp>=3.20 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from importlib-metadata<=8.5.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.21.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->chromadb) (0.1.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from requests->kubernetes>=28.1.0->chromadb) (3.4.0)\n",
      "Requirement already satisfied: sniffio>=1.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from anyio->httpx>=0.27.0->chromadb) (1.3.1)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
      "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install chromadb # vector database does semantic search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import chromadb\n",
    "client=chromadb.Client()\n",
    "collection=client.create_collection(name=\"my_collection\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adityaaryan/.cache/chroma/onnx_models/all-MiniLM-L6-v2/onnx.tar.gz: 100%|██████████| 79.3M/79.3M [00:24<00:00, 3.36MiB/s]\n"
     ]
    }
   ],
   "source": [
    "collection.add(\n",
    "    documents=[\n",
    "        \"This document is about New York\",\n",
    "        \"This document is about New Delhi\",\n",
    "    ],\n",
    "    ids=[\n",
    "        'id1',\n",
    "        'id2'\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ids': ['id1', 'id2'],\n",
       " 'embeddings': None,\n",
       " 'documents': ['This document is about New York',\n",
       "  'This document is about New Delhi'],\n",
       " 'uris': None,\n",
       " 'data': None,\n",
       " 'metadatas': [None, None],\n",
       " 'included': [<IncludeEnum.documents: 'documents'>,\n",
       "  <IncludeEnum.metadatas: 'metadatas'>]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_docs=collection.get()\n",
    "all_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ids': [['id2', 'id1']],\n",
       " 'embeddings': None,\n",
       " 'documents': [['This document is about New Delhi',\n",
       "   'This document is about New York']],\n",
       " 'uris': None,\n",
       " 'data': None,\n",
       " 'metadatas': [[None, None]],\n",
       " 'distances': [[1.6222214698791504, 1.96023428440094]],\n",
       " 'included': [<IncludeEnum.distances: 'distances'>,\n",
       "  <IncludeEnum.documents: 'documents'>,\n",
       "  <IncludeEnum.metadatas: 'metadatas'>]}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result=collection.query(\n",
    "    query_texts=[\n",
    "        \"Query about Chole Bhature\",\n",
    "    ],\n",
    "    n_results=2\n",
    ")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#chromadb is a vector database that does semantic search. It is a wrapper around Faiss and Milvus. It is a simple and easy to use library that can be used to create a vector database and perform semantic search on it. It is a great tool for anyone who wants to build a vector database and perform semantic search on it."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}