local_embed.ipynb

{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import logging\n",
                "import os\n",
                "import sys\n",
                "import uuid\n",
                "\n",
                "from azure.core import exceptions\n",
                "from azure.identity import DefaultAzureCredential\n",
                "\n",
                "logger = logging.getLogger(\"azure\")\n",
                "logger.setLevel(logging.DEBUG)\n",
                "\n",
                "# Direct logging output to stdout. Without adding a handler,\n",
                "# no logging output is visible.\n",
                "handler = logging.StreamHandler(stream=sys.stdout)\n",
                "logger.addHandler(handler)\n",
                "\n",
                "print(\n",
                "    f\"Logger enabled for ERROR={logger.isEnabledFor(logging.ERROR)}, \"\n",
                "    f\"WARNING={logger.isEnabledFor(logging.WARNING)}, \"\n",
                "    f\"INFO={logger.isEnabledFor(logging.INFO)}, \"\n",
                "    f\"DEBUG={logger.isEnabledFor(logging.DEBUG)}\"\n",
                ")\n",
                "\n",
                "try:\n",
                "    DefaultAzureCredential(logging_enable=True).get_token(\n",
                "        \"https://cognitiveservices.azure.com/.default\"\n",
                "    )\n",
                "except (exceptions.ClientAuthenticationError, exceptions.HttpResponseError) as e:\n",
                "    print(e.message)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import semchunk\n",
                "import tiktoken\n",
                "\n",
                "_chunk = semchunk.chunkerify(tiktoken.encoding_for_model(\"gpt-4o\"), chunk_size)\n",
                "\n",
                "chunks = _chunk(text)\n",
                "print(chunks)\n",
                "\n",
                "\n",
                "enc = tiktoken.encoding_for_model(\"gpt-4o\")\n",
                "abb = enc.encode(\n",
                "    \"This is a test string for text splitting and I want to know who is better at natation but\"\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# 400 palavras\n",
                "text = \"\"\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis lobortis, ipsum ut euismod cursus, tortor felis ullamcorper orci, sit amet facilisis velit diam a orci. Nulla rhoncus felis sapien, interdum commodo erat sodales vulputate. Nunc consectetur accumsan felis et porta. Pellentesque aliquet, massa id lacinia condimentum, ligula mauris consectetur nisl, sed pellentesque ipsum elit et augue. Mauris leo magna, convallis ut aliquet nec, accumsan vel quam. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed at feugiat ex, in aliquet leo. Aenean ornare, mauris at vestibulum ornare, nibh mi luctus urna, at pharetra nisi velit ut magna. Ut auctor accumsan sem, sed feugiat turpis tincidunt ultricies. In odio quam, dignissim id ante eget, blandit iaculis augue. Vestibulum condimentum consequat urna, at vestibulum augue ullamcorper in. Morbi convallis venenatis dolor. Cras fermentum consequat bibendum. Duis ac dignissim turpis. Nam rhoncus enim lorem, ac molestie est bibendum non. Duis ut bibendum nisi, ut dictum dui.\n",
                "\n",
                "Donec justo odio, pharetra ac ipsum ut, efficitur luctus sapien. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Mauris suscipit in ante non vestibulum. Pellentesque placerat dolor mollis elit pharetra, sed mattis quam pellentesque. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque sed aliquam mauris. Ut dictum eleifend magna, id euismod erat interdum vitae. Cras aliquet molestie ante, a aliquet lectus luctus sed. Duis sagittis volutpat iaculis. Mauris sit amet erat at magna porttitor mattis ut vitae nunc. In quis ipsum ornare, auctor purus eget, varius urna. Maecenas nisi urna, viverra sed auctor eget, euismod at nulla. Quisque non felis nisl.\n",
                "\n",
                "Morbi condimentum, nisl sit amet dapibus placerat, lacus odio tristique nibh, quis finibus quam arcu id erat. Mauris elementum erat vel enim placerat, vel pretium purus efficitur. Duis dignissim risus eget tellus aliquam, in interdum lacus tempor. Phasellus mauris arcu, semper id euismod ac, feugiat in lorem. Maecenas orci sapien, semper ac risus eget, accumsan convallis mi. Donec imperdiet tristique nibh, et ultricies enim commodo at. Nullam ullamcorper luctus purus eget lobortis. Nam justo nisl, vulputate sed aliquet venenatis, pharetra a augue. Sed sed odio consectetur, lobortis tellus ut, commodo risus. Aenean tellus ligula, aliquam vel tortor sit amet, scelerisque vehicula enim. Vestibulum pharetra, quam nec feugiat tempus, felis purus tempor tortor, at elementum nibh massa eu ligula. Pellentesque diam lacus, suscipit vel fringilla finibus, mollis nec quam. Interdum et malesuada fames ac ante ipsum primis.\"\"\""
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "sample_text = [\"hello world\"] * 10000"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from transformers import RobertaTokenizer\n",
                "\n",
                "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from sentence_transformers import SentenceTransformer\n",
                "\n",
                "model = SentenceTransformer(\"allenai/longformer-base-4096\")\n",
                "\n",
                "embedding = model.encode(sample_text)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "array = [\"hello world dayenne\"] * 100\n",
                "test_text = \" \".join(array)\n",
                "abc = model.encode(test_text)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import tiktoken\n",
                "\n",
                "encoder = tiktoken.get_encoding(\"cl100k_base\")\n",
                "defg = len(encoder.encode(test_text))\n",
                "print(defg)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "abc.shape"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": ".venv",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.11.9"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}