generating_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "82f44947-3703-400b-bf26-03d7964f3274",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d10bb79-e897-4116-9146-f668877d3df3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from typing import Dict\n",
    "\n",
    "from datasets import load_dataset, Dataset, DatasetDict, load_from_disk, concatenate_datasets\n",
    "from huggingface_hub import notebook_login"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85dcc3b3-9fd0-4f82-9bf1-1437268e266a",
   "metadata": {},
   "source": [
    "## Global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "589acac7-b16b-4c08-b6f1-9d3b1e0c0e86",
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_GATH_BAIZE = True\n",
    "SAVE_GLAIVE_FC_V2 = True\n",
    "MERGE_BOTH = True\n",
    "FINAL = True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c42b282-8fd9-46c4-8d32-fa0a97e1d365",
   "metadata": {},
   "source": [
    "## Dataset Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "34354fe2-0703-4a3c-a8b8-3653ce4ec022",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name, dataset_name2 = \"gathnex/Gath_baize\", \"glaiveai/glaive-function-calling-v2\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55b0aaf0-e07a-4abb-8897-1445b27c3d11",
   "metadata": {},
   "source": [
    "## Gath_baize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "922f7b9e-21a9-4b46-8a0a-2a59b034a7ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading a Gath_baize dataset\n",
    "dataset = load_dataset(dataset_name, split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "854600b4-a353-4f05-aa94-1901eefc7c29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['chat_sample', 'dataset_origin'],\n",
      "    num_rows: 210311\n",
      "})\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'The conversation between Human and AI assisatance named Gathnex [INST] I would like to know how to find the path with the maximum minimal weight in a graph.\\n[/INST] That\\'s a great question. To find the path with the maximum minimal weight in a graph, you can use the algorithm called \"Floyd–Warshall algorithm\". Have you heard about it before?\\n[INST] Yes, I have. Can you explain how it works?\\n[/INST] Sure. The Floyd–Warshall algorithm is a dynamic programming algorithm that finds the shortest paths in a weighted graph with positive or negative edge weights. It does this by repeatedly improving an estimate of the shortest path between two points, until the estimate is optimal.\\n[INST] How does it guarantee to find the maximum minimal weight path?\\n[/INST] The algorithm works by computing a matrix of all the possible pairwise shortest paths. From that matrix, we can pick the path with the largest minimum weight. That path will be the maximum minimal weight path.\\n[INST] That sounds interesting. Are there any limitations to using this algorithm?\\n[/INST] Yes, there are some limitations. The main limitation is that the Floyd–Warshall algorithm has a time complexity of O(n^3). This means that it can become very slow for very large graphs. Another limitation is that it is not suitable for finding the shortest path in a graph with negative cycles.'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(dataset)\n",
    "dataset[\"chat_sample\"][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e339b13-fc55-4059-91e4-3f3ae10dcd3e",
   "metadata": {},
   "source": [
    "### Function to split System and chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "197a9ecb-a480-4b67-89ad-1385fd6fb2fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_row(row: str) -> Dict[str, str]:\n",
    "    regex = r\"(?P<system>^.+?)(?=\\[INST\\])|\\[INST\\](?P<user>.+?)\\[\\/INST\\]|(?<=\\[\\/INST\\])(?P<assistant>.+?)(?=\\[INST\\]|$)\"\n",
    "    # Compile the regex for better performance\n",
    "    compiled_regex = re.compile(regex, re.DOTALL)\n",
    "    # Search for matches\n",
    "    matches = compiled_regex.finditer(row)\n",
    "\n",
    "    system_prompt = []\n",
    "    conversation = []\n",
    "    # Process matches\n",
    "    for match in matches:\n",
    "        if match.group(\"system\"):\n",
    "            system = match.group(\"system\").strip()\n",
    "            system_prompt += [{\"role\": \"system\", \"content\": \"The conversation between Human and AI assistant\"}]\n",
    "        if match.group(\"user\"):\n",
    "            user = match.group(\"user\").strip()\n",
    "            conversation += [{\"role\": \"user\", \"content\": user}]\n",
    "        if match.group(\"assistant\"):\n",
    "            assistant = match.group(\"assistant\").strip()\n",
    "            conversation += [{\"role\": \"assistant\", \"content\": assistant}]\n",
    "    \n",
    "    return {'system': system_prompt, 'chat': conversation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d6d08616-0cde-4aeb-b27e-f8e59059998b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'system': [{'role': 'system',\n",
       "   'content': 'The conversation between Human and AI assistant'}],\n",
       " 'chat': [{'role': 'user',\n",
       "   'content': 'I would like to know how to find the path with the maximum minimal weight in a graph.'},\n",
       "  {'role': 'assistant',\n",
       "   'content': 'That\\'s a great question. To find the path with the maximum minimal weight in a graph, you can use the algorithm called \"Floyd–Warshall algorithm\". Have you heard about it before?'},\n",
       "  {'role': 'user', 'content': 'Yes, I have. Can you explain how it works?'},\n",
       "  {'role': 'assistant',\n",
       "   'content': 'Sure. The Floyd–Warshall algorithm is a dynamic programming algorithm that finds the shortest paths in a weighted graph with positive or negative edge weights. It does this by repeatedly improving an estimate of the shortest path between two points, until the estimate is optimal.'},\n",
       "  {'role': 'user',\n",
       "   'content': 'How does it guarantee to find the maximum minimal weight path?'},\n",
       "  {'role': 'assistant',\n",
       "   'content': 'The algorithm works by computing a matrix of all the possible pairwise shortest paths. From that matrix, we can pick the path with the largest minimum weight. That path will be the maximum minimal weight path.'},\n",
       "  {'role': 'user',\n",
       "   'content': 'That sounds interesting. Are there any limitations to using this algorithm?'},\n",
       "  {'role': 'assistant',\n",
       "   'content': 'Yes, there are some limitations. The main limitation is that the Floyd–Warshall algorithm has a time complexity of O(n^3). This means that it can become very slow for very large graphs. Another limitation is that it is not suitable for finding the shortest path in a graph with negative cycles.'}]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "convert_row(dataset[\"chat_sample\"][1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f1ed04e-b242-47b5-a12e-842ed278411c",
   "metadata": {},
   "source": [
    "### Splitting System and Chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "94b7de38-55eb-4fb3-9a29-84eff9708e3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['dataset_origin', 'system', 'chat'],\n",
      "    num_rows: 210311\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "if SAVE_GATH_BAIZE:\n",
    "    # Use the .map() method to apply convert_row to each row\n",
    "    # Set batched=True if your convert_row function can handle batch processing for efficiency\n",
    "    updated_dataset = dataset.map(lambda examples: convert_row(examples['chat_sample']), remove_columns=['chat_sample'])\n",
    "    \n",
    "    print(updated_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3aa8b4d0-8540-4c61-90b8-1985e8c21a55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset_origin': 'alpaca', 'system': [{'content': 'The conversation between Human and AI assistant', 'role': 'system'}], 'chat': [{'content': 'Generate a headline given a content block.\\nThe Sony Playstation 5 is the latest version of the console. It has improved graphics and faster processing power.', 'role': 'user'}, {'content': 'Experience Amazing Graphics and Speed with the New Sony Playstation 5', 'role': 'assistant'}]}\n"
     ]
    }
   ],
   "source": [
    "if SAVE_GATH_BAIZE:\n",
    "    print(updated_dataset[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc93a0ae-ee1d-41ce-8b88-542e8b2774f9",
   "metadata": {},
   "source": [
    "### Saving new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3a26ec68-b1af-4b1b-9e26-a7cabf66389f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the path where you want to save the dataset\n",
    "save_path = './dataset/hibo-gath_baize'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9ac38881-04a2-4004-9feb-8d0377de9c11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a500ffdcc4a4881bad74d259048c6d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/210311 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset saved to ./dataset/hibo-gath_baize\n"
     ]
    }
   ],
   "source": [
    "if SAVE_GATH_BAIZE:\n",
    "    # Save the dataset to disk\n",
    "    updated_dataset.save_to_disk(save_path)\n",
    "    \n",
    "    print(f\"Dataset saved to {save_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2f0fc8c-4fa1-40a9-9352-fa76c60a8d80",
   "metadata": {},
   "source": [
    "### Reloading new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9316c353-921f-44ec-807b-f38b5bb4e885",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['dataset_origin', 'system', 'chat'],\n",
      "    num_rows: 210311\n",
      "})\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'dataset_origin': 'alpaca',\n",
       " 'system': [{'content': 'The conversation between Human and AI assistant',\n",
       "   'role': 'system'}],\n",
       " 'chat': [{'content': 'Generate a headline given a content block.\\nThe Sony Playstation 5 is the latest version of the console. It has improved graphics and faster processing power.',\n",
       "   'role': 'user'},\n",
       "  {'content': 'Experience Amazing Graphics and Speed with the New Sony Playstation 5',\n",
       "   'role': 'assistant'}]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the dataset from disk\n",
    "reloaded_dataset = load_from_disk(save_path)\n",
    "\n",
    "print(reloaded_dataset)\n",
    "reloaded_dataset[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f78ed57-8400-44ce-8b7c-ea01cb5dd7ba",
   "metadata": {},
   "source": [
    "## glaive-function-calling-v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7d79e0d7-11db-4945-84e9-026413febe76",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading a Gath_baize_function_calling dataset\n",
    "dataset2 = load_dataset(dataset_name2, split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8fbd9d1a-4523-4580-9fd5-d34bd594fb10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['chat', 'system'],\n",
      "    num_rows: 112960\n",
      "})\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'USER: Hi, I would like to create a new user profile.\\n\\n\\nASSISTANT: Sure, I can help with that. Could you please provide me with your name, age, and email address? <|endoftext|>\\n\\n\\nUSER: My name is John Doe, I am 30 years old and my email is johndoe@example.com.\\n\\n\\nASSISTANT: <functioncall> {\"name\": \"create_user_profile\", \"arguments\": \\'{\"name\": \"John Doe\", \"age\": 30, \"email\": \"johndoe@example.com\"}\\'} <|endoftext|>\\n\\n\\nFUNCTION RESPONSE: {\"status\": \"success\", \"message\": \"User profile created successfully\", \"user_id\": \"12345\"}\\n\\n\\nASSISTANT: Your user profile has been created successfully. Your user ID is 12345. Please keep it safe as you will need it for future reference. <|endoftext|>\\n\\n\\n'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(dataset2)\n",
    "dataset2['chat'][112911]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da6b810f-6baa-4f52-be6c-8c8d1e3ea97d",
   "metadata": {},
   "source": [
    "### Function to split System and chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "392328c7-9bbf-4570-aed6-4e82c5ceff0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_row_d2(chat: str, system: str) -> Dict[str, str]:\n",
    "    regex = r\"(?=USER:\\s)|USER:\\s(?P<user>.+?)(?=ASSISTANT:\\s|FUNCTION RESPONSE:\\s|$)|ASSISTANT:\\s(?P<assistant>.+?)(?=USER:\\s|FUNCTION RESPONSE:\\s|ASSISTANT:\\s|$)|FUNCTION RESPONSE:\\s(?P<function_response>.+?)(?=USER:\\s|ASSISTANT:\\s|FUNCTION RESPONSE:\\s|$)\"\n",
    "    # Compile the regex for better performance\n",
    "    compiled_regex = re.compile(regex, re.DOTALL)\n",
    "    # Search for matches\n",
    "    matches = compiled_regex.finditer(chat)\n",
    "\n",
    "    system_conv = [{\"role\": \"system\", \"content\": system.replace(\"SYSTEM: \", \"\").strip()}]\n",
    "    conversation = []\n",
    "    # Process matches\n",
    "    for match in matches:\n",
    "        if match.group(\"user\"):\n",
    "            user = match.group(\"user\").replace(\"<|endoftext|>\", \"\").strip()\n",
    "            conversation += [{\"role\": \"user\", \"content\": user}]\n",
    "        if match.group(\"assistant\"):\n",
    "            assistant = match.group(\"assistant\").replace(\"<|endoftext|>\", \"\").strip()\n",
    "            conversation += [{\"role\": \"assistant\", \"content\": assistant}]\n",
    "        if match.group(\"function_response\"):\n",
    "            function_response = match.group(\"function_response\").strip()\n",
    "            conversation += [{\"role\": \"function_response\", \"content\": function_response}]\n",
    "    return {'system': system_conv, 'chat': conversation, 'dataset_origin': 'glaiveai/glaive-function-calling-v2'}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a3a40d0f-fb64-4573-a5b4-b658d488abc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'system': [{'role': 'system',\n",
       "   'content': 'You are a helpful assistant with access to the following functions. Use them if required -\\n{\\n    \"name\": \"get_exchange_rate\",\\n    \"description\": \"Get the exchange rate between two currencies\",\\n    \"parameters\": {\\n        \"type\": \"object\",\\n        \"properties\": {\\n            \"base_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert from\"\\n            },\\n            \"target_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert to\"\\n            }\\n        },\\n        \"required\": [\\n            \"base_currency\",\\n            \"target_currency\"\\n        ]\\n    }\\n}'}],\n",
       " 'chat': [{'role': 'user',\n",
       "   'content': 'Can you book a flight for me from New York to London?'},\n",
       "  {'role': 'assistant',\n",
       "   'content': \"I'm sorry, but I don't have the capability to book flights. My current function allows me to get the exchange rate between two currencies. If you need help with that, feel free to ask!\"}],\n",
       " 'dataset_origin': 'glaiveai/glaive-function-calling-v2'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"\"\"USER: Can you book a flight for me from New York to London?\\n\\n\\nASSISTANT: I'm sorry, but I don't have the capability to book flights. My current function allows me to get the exchange rate between two currencies. If you need help with that, feel free to ask! <|endoftext|>\\n\\n\\n\"\"\"\n",
    "system = \"\"\"SYSTEM: You are a helpful assistant with access to the following functions. Use them if required -\\n{\\n    \"name\": \"get_exchange_rate\",\\n    \"description\": \"Get the exchange rate between two currencies\",\\n    \"parameters\": {\\n        \"type\": \"object\",\\n        \"properties\": {\\n            \"base_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert from\"\\n            },\\n            \"target_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert to\"\\n            }\\n        },\\n        \"required\": [\\n            \"base_currency\",\\n            \"target_currency\"\\n        ]\\n    }\\n}\\n\"\"\"\n",
    "convert_row_d2(text, system)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c09394df-d841-4b91-8de9-e5f589717259",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3596b6be7174c48afa722675c0f023e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/112960 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['chat', 'system', 'dataset_origin'],\n",
      "    num_rows: 112960\n",
      "})\n",
      "{'chat': [{'content': 'Can you book a flight for me from New York to London?', 'role': 'user'}, {'content': \"I'm sorry, but I don't have the capability to book flights. My current function allows me to get the exchange rate between two currencies. If you need help with that, feel free to ask!\", 'role': 'assistant'}], 'system': [{'content': 'You are a helpful assistant with access to the following functions. Use them if required -\\n{\\n    \"name\": \"get_exchange_rate\",\\n    \"description\": \"Get the exchange rate between two currencies\",\\n    \"parameters\": {\\n        \"type\": \"object\",\\n        \"properties\": {\\n            \"base_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert from\"\\n            },\\n            \"target_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert to\"\\n            }\\n        },\\n        \"required\": [\\n            \"base_currency\",\\n            \"target_currency\"\\n        ]\\n    }\\n}', 'role': 'system'}], 'dataset_origin': 'glaiveai/glaive-function-calling-v2'}\n"
     ]
    }
   ],
   "source": [
    "if SAVE_GLAIVE_FC_V2:\n",
    "    # Adding 'dataset_origin' column directly within the .map() call\n",
    "    updated_dataset = dataset2.map(lambda examples: convert_row_d2(examples['chat'], examples['system']), batched=False)\n",
    "    print(updated_dataset)\n",
    "    print(updated_dataset[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff2308b8-0279-47d3-8014-66875715d9fb",
   "metadata": {},
   "source": [
    "### Saving new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c836331a-0fc4-478d-ab51-f7af5bc3e411",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the path where you want to save the dataset\n",
    "save_path = './dataset/hibo-glaive-function-calling'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9a6f494f-41fb-40bf-98ad-7331cdf3282d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b17dc8508f640e3a53fa988388539f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/112960 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset saved to ./dataset/hibo-glaive-function-calling\n"
     ]
    }
   ],
   "source": [
    "if SAVE_GLAIVE_FC_V2:\n",
    "    # Save the dataset to disk\n",
    "    updated_dataset.save_to_disk(save_path)\n",
    "    \n",
    "    print(f\"Dataset saved to {save_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5e6659e-537b-480c-8496-319a7c784a2b",
   "metadata": {},
   "source": [
    "### Reloading new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "53dc39d1-ea29-4445-80cd-7705bc6d29bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['chat', 'system', 'dataset_origin'],\n",
      "    num_rows: 112960\n",
      "})\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'chat': [{'content': 'Can you book a flight for me from New York to London?',\n",
       "   'role': 'user'},\n",
       "  {'content': \"I'm sorry, but I don't have the capability to book flights. My current function allows me to get the exchange rate between two currencies. If you need help with that, feel free to ask!\",\n",
       "   'role': 'assistant'}],\n",
       " 'system': [{'content': 'You are a helpful assistant with access to the following functions. Use them if required -\\n{\\n    \"name\": \"get_exchange_rate\",\\n    \"description\": \"Get the exchange rate between two currencies\",\\n    \"parameters\": {\\n        \"type\": \"object\",\\n        \"properties\": {\\n            \"base_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert from\"\\n            },\\n            \"target_currency\": {\\n                \"type\": \"string\",\\n                \"description\": \"The currency to convert to\"\\n            }\\n        },\\n        \"required\": [\\n            \"base_currency\",\\n            \"target_currency\"\\n        ]\\n    }\\n}',\n",
       "   'role': 'system'}],\n",
       " 'dataset_origin': 'glaiveai/glaive-function-calling-v2'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the dataset from disk\n",
    "reloaded_dataset2 = load_from_disk(save_path)\n",
    "\n",
    "print(reloaded_dataset2)\n",
    "reloaded_dataset2[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2b61d07-278e-4100-80f9-ace95ebb5e8a",
   "metadata": {},
   "source": [
    "## hibo-function-calling-v1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f76ac2c-a0f0-4738-90fa-379b436c4d6b",
   "metadata": {},
   "source": [
    "### Merge both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9fdbd259-ab35-47f8-850b-c9972b13806a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'content': 'The conversation between Human and AI assistant', 'role': 'system'}]\n"
     ]
    }
   ],
   "source": [
    "print(reloaded_dataset['system'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "83ae1e3c-39ae-4040-a033-f704f54a0fdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'content': 'You are a helpful assistant with access to the following functions. Use them if required -\\n{\\n    \"name\": \"get_news_headlines\",\\n    \"description\": \"Get the latest news headlines\",\\n    \"parameters\": {\\n        \"type\": \"object\",\\n        \"properties\": {\\n            \"country\": {\\n                \"type\": \"string\",\\n                \"description\": \"The country for which to fetch news\"\\n            }\\n        },\\n        \"required\": [\\n            \"country\"\\n        ]\\n    }\\n}', 'role': 'system'}]\n"
     ]
    }
   ],
   "source": [
    "print(reloaded_dataset2['system'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b45a2c73-a634-4911-9a34-34443e938bfd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['dataset_origin', 'system', 'chat'],\n",
      "    num_rows: 323271\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "if MERGE_BOTH:\n",
    "    # Assuming updated_dataset and updated_dataset_with_origin are your two datasets ready to be merged\n",
    "    merged_dataset = concatenate_datasets([reloaded_dataset, reloaded_dataset2])\n",
    "    \n",
    "    print(merged_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2b67f92-0024-49de-a377-f62b79b71ef2",
   "metadata": {},
   "source": [
    "### Saving merged dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "53f45af3-1bb5-4fba-8305-1696c376dc49",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the path where you want to save the dataset\n",
    "save_path = './dataset/hibo-function-calling-merge'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5d7070ca-713a-4784-ab1d-ea9dff1dbd58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f93956a27c224300a1b8051ffacc04d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/323271 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset saved to ./dataset/hibo-function-calling-merge\n"
     ]
    }
   ],
   "source": [
    "if MERGE_BOTH:\n",
    "    # Save the dataset to disk\n",
    "    merged_dataset.save_to_disk(save_path)\n",
    "    \n",
    "    print(f\"Dataset saved to {save_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48337801-c99d-44bb-b54c-cf78d626b536",
   "metadata": {},
   "source": [
    "### Reloading new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d92f54d0-f41d-4b64-8ded-0a3e168d6ff6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['dataset_origin', 'system', 'chat'],\n",
      "    num_rows: 323271\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "# Load the dataset from disk\n",
    "reloaded_dataset3 = load_from_disk(save_path)\n",
    "print(reloaded_dataset3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6d3a5d00-40a0-4b54-b8be-2e71c97ecae7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dataset_origin': 'alpaca',\n",
       " 'system': [{'content': 'The conversation between Human and AI assistant',\n",
       "   'role': 'system'}],\n",
       " 'chat': [{'content': 'Generate a headline given a content block.\\nThe Sony Playstation 5 is the latest version of the console. It has improved graphics and faster processing power.',\n",
       "   'role': 'user'},\n",
       "  {'content': 'Experience Amazing Graphics and Speed with the New Sony Playstation 5',\n",
       "   'role': 'assistant'}]}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reloaded_dataset3[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b2da0353-4cae-490f-9960-110f7ace0fb2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dataset_origin': 'glaiveai/glaive-function-calling-v2',\n",
       " 'system': [{'content': 'You are a helpful assistant with access to the following functions. Use them if required -\\n{\\n    \"name\": \"create_user_profile\",\\n    \"description\": \"Create a user profile\",\\n    \"parameters\": {\\n        \"type\": \"object\",\\n        \"properties\": {\\n            \"name\": {\\n                \"type\": \"string\",\\n                \"description\": \"The user\\'s name\"\\n            },\\n            \"age\": {\\n                \"type\": \"integer\",\\n                \"description\": \"The user\\'s age\"\\n            },\\n            \"email\": {\\n                \"type\": \"string\",\\n                \"description\": \"The user\\'s email address\"\\n            }\\n        },\\n        \"required\": [\\n            \"name\",\\n            \"age\",\\n            \"email\"\\n        ]\\n    }\\n}',\n",
       "   'role': 'system'}],\n",
       " 'chat': [{'content': 'Hi, I would like to create a new user profile.',\n",
       "   'role': 'user'},\n",
       "  {'content': 'Sure, I can help with that. Could you please provide me with your name, age, and email address?',\n",
       "   'role': 'assistant'},\n",
       "  {'content': 'My name is John Doe, I am 30 years old and my email is johndoe@example.com.',\n",
       "   'role': 'user'},\n",
       "  {'content': '<functioncall> {\"name\": \"create_user_profile\", \"arguments\": \\'{\"name\": \"John Doe\", \"age\": 30, \"email\": \"johndoe@example.com\"}\\'}',\n",
       "   'role': 'assistant'},\n",
       "  {'content': '{\"status\": \"success\", \"message\": \"User profile created successfully\", \"user_id\": \"12345\"}',\n",
       "   'role': 'function_response'},\n",
       "  {'content': 'Your user profile has been created successfully. Your user ID is 12345. Please keep it safe as you will need it for future reference.',\n",
       "   'role': 'assistant'}]}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reloaded_dataset3[323222]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f04bc78d-eaed-4e75-9c1d-b02091b00425",
   "metadata": {},
   "source": [
    "## Split train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e7f94a89-dd94-46f0-a6e0-c05fb6e8e123",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_dict = DatasetDict({\n",
    "    'train': reloaded_dataset3\n",
    "})\n",
    "\n",
    "# Shuffle the 'train' dataset\n",
    "dataset_dict = dataset_dict['train'].shuffle(seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "0021828a-3465-4364-8b60-dbdd4b0108d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['dataset_origin', 'system', 'chat'],\n",
       "    num_rows: 323271\n",
       "})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c5097b85-2dea-40a2-9e40-dabd2d2255c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the path where you want to save the dataset\n",
    "save_path = './dataset/hibo-function-calling-v1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "102eaa9e-2eaa-4431-9398-09247d0d46de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "826e348622b84312a711229f2e84174e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/323271 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset saved to ./dataset/hibo-function-calling-v1\n"
     ]
    }
   ],
   "source": [
    "if FINAL:\n",
    "    # Save the dataset to disk\n",
    "    dataset_dict.save_to_disk(save_path)\n",
    "    \n",
    "    print(f\"Dataset saved to {save_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "cf2c5ecf-dc8e-4ddb-83f2-7d2fddc99aec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d91b38ed910a4e8581cfb179e6b45432",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e2b8c397cde4339b8c13f1c112908c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/324 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a67ef8b85a4e4a1caebd085a6b597d31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/4.52k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thiba\\.virtualenvs\\mistral-7b-instruct-CB05KjAH\\Lib\\site-packages\\huggingface_hub\\file_download.py:149: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\thiba\\.cache\\huggingface\\hub\\datasets--thibaud-perrin--hibo-function-calling-v1. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    }
   ],
   "source": [
    "if FINAL:\n",
    "    dataset_dict.push_to_hub(\"thibaud-perrin/hibo-function-calling-v1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fd0f1cb-8e43-49ae-ad6b-0a4e6b04048e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}