create-endpoint-01-sbert.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "142a9f8d-fd69-46b9-b4ef-0091ccd8c4f6",
   "metadata": {
    "tags": []
   },
   "source": [
    "## **Deprecated** SBERT endpoint (This notebook is for a reference. We don't need to create it again.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c2945922-7de9-4213-8293-1f901d544f13",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (4.35.0)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (3.12.4)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (0.17.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (1.26.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (2023.10.3)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.15,>=0.14 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (0.14.1)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (0.4.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from transformers) (4.66.1)\n",
      "Requirement already satisfied: fsspec in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.10.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.8.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.1.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->transformers) (3.3.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->transformers) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->transformers) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->transformers) (2023.7.22)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2f253c52-8daf-4a2f-9d0e-348ebc656a02",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "# Choose transformer model for embeddings\n",
    "from transformers import AutoTokenizer, AutoModel\n",
    "import os\n",
    "import sagemaker\n",
    "import time\n",
    "# saved_model_dir = 'transformer'\n",
    "# os.makedirs(saved_model_dir, exist_ok=True)\n",
    "\n",
    "# tokenizer = AutoTokenizer.from_pretrained(\"sentence-transformers/multi-qa-MiniLM-L6-cos-v1\")\n",
    "# tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
    "# model = AutoModel.from_pretrained(\"sentence-transformers/multi-qa-MiniLM-L6-cos-v1\") \n",
    "\n",
    "# tokenizer.save_pretrained(saved_model_dir)\n",
    "# model.save_pretrained(saved_model_dir)\n",
    "\n",
    "#Defining default bucket for SageMaker pretrained model hosting\n",
    "sagemaker_session = sagemaker.Session()\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "# !cd transformer && tar czvf ../model.tar.gz *\n",
    "\n",
    "# model_data = sagemaker_session.upload_data(path='model.tar.gz', key_prefix='autofaiss-demo/huggingface-models')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8ac2e64b-524c-43b1-b968-bddea52f99ec",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n",
      "----!"
     ]
    }
   ],
   "source": [
    "from sagemaker.huggingface.model import HuggingFaceModel\n",
    "\n",
    "# create Hugging Face Model Class\n",
    "huggingface_model = HuggingFaceModel(\n",
    "   #model_data=model_data,       # path to your model and script\n",
    "   #entry_point = 'predict.py',\n",
    "   #source_dir = 'source_dir',\n",
    "   role=role,                    # iam role with permissions to create an Endpoint\n",
    "   transformers_version=\"4.12\",  # transformers version used\n",
    "   pytorch_version=\"1.9\",        # pytorch version used\n",
    "   py_version='py38',            # python version used\n",
    "   env={ \n",
    "       'HF_TASK':'feature-extraction',\n",
    "       'HF_MODEL_ID': 'sentence-transformers/multi-qa-MiniLM-L6-cos-v1'\n",
    "   }\n",
    ")\n",
    "\n",
    "# deploy the endpoint endpoint\n",
    "predictor = huggingface_model.deploy(\n",
    "    initial_instance_count=1,\n",
    "    instance_type=\"ml.m5.large\",\n",
    "    endpoint_name=\"sm-cc-aws\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "965594b5-7294-4d34-a6a1-018f5486d833",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "No chat template is defined for this tokenizer - using a default chat template that implements the ChatML format. If the default is not appropriate for your model, please set `tokenizer.chat_template` to an appropriate template. See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"<|im_start|>system\\nYou are a Spanish language teacher, and the user made mistakes. You respond with 'ahh, you mean,...' and repeat what the user said in the correct format. Don't further explain, and keep your response in one short sentence.<|im_end|>\\n<|im_start|>user\\nSí, tengo algo tiempo hoy.<|im_end|>\\n\""
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Trying out chat template but the kernel keeps crashing\n",
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"sentence-transformers/multi-qa-MiniLM-L6-cos-v1\")\n",
    "\n",
    "chat = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a Spanish language teacher, and the user made mistakes. You respond with 'ahh, you mean,...' and repeat what the user said in the correct format. Don't further explain, and keep your response in one short sentence.\"},\n",
    "    {\"role\": \"user\", \"content\": \"Sí, tengo algo tiempo hoy.\"},\n",
    "]\n",
    "\n",
    "tokenizer.use_default_system_prompt = False\n",
    "tokenizer.apply_chat_template(chat, tokenize=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9056d615-0fa7-4d34-8122-49c7ca634c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "messages2 = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a Spanish language teacher, and the user made mistakes. You respond with 'ahh, you mean,...' and repeat what the user said in the correct format. Don't further explain, and keep your response in one short sentence.\"},\n",
    "    {\"role\": \"user\", \"content\": \"Sí, tengo algo tiempo hoy.\"},\n",
    "]\n",
    "prompt2 = pipe.tokenizer.apply_chat_template(messages2, tokenize=False, add_generation_prompt=True)\n",
    "outputs2 = pipe(prompt2, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)\n",
    "print(outputs2[0][\"generated_text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "85270a15-fd72-479d-aabf-5a8305947f21",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting accelerate\n",
      "  Downloading accelerate-0.24.1-py3-none-any.whl.metadata (18 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from accelerate) (1.26.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from accelerate) (21.3)\n",
      "Requirement already satisfied: psutil in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from accelerate) (5.9.5)\n",
      "Requirement already satisfied: pyyaml in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from accelerate) (6.0.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from accelerate) (2.0.1)\n",
      "Requirement already satisfied: huggingface-hub in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from accelerate) (0.17.3)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from packaging>=20.0->accelerate) (3.1.1)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.12.4)\n",
      "Requirement already satisfied: typing-extensions in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (4.8.0)\n",
      "Requirement already satisfied: sympy in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.12)\n",
      "Requirement already satisfied: networkx in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.2)\n",
      "Requirement already satisfied: jinja2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from huggingface-hub->accelerate) (2023.10.0)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from huggingface-hub->accelerate) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from huggingface-hub->accelerate) (4.66.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.3.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (2023.7.22)\n",
      "Requirement already satisfied: mpmath>=0.19 in /home/ec2-user/anaconda3/envs/pytorch_p310/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: accelerate\n",
      "Successfully installed accelerate-0.24.1\n"
     ]
    }
   ],
   "source": [
    "!pip install accelerate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5d6de7a8-0b14-4294-83b5-8ccf97401277",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# delete endpoint\n",
    "predictor.delete_endpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d02339e-b02c-40bd-b53e-42141ede50fe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c691f79c-b40c-4904-8a5d-7d1ee3c9e61d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p310",
   "language": "python",
   "name": "conda_pytorch_p310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}