Cat Herder - Simple Debugger using LLM Reflection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c146540a",
   "metadata": {},
   "source": [
    "<img src=\"./cat_herder.png\" alt=\"Alt text\" style=\"width:100px; float:left; margin-right:10px;\"/>\n",
    "\n",
    "<div style=\"overflow: auto;\">\n",
    "    <h1> Cat Herder - Simple Debugger using LLM Reflection\n",
    "<p><p>\n",
    "    Galen Wilkerson</h1>\n",
    "\n",
    "    One of the most common tasks using LLMs is for programming.\n",
    "\n",
    "    However, for several reasons, LLMs are not perfect programmers (yet). Very often, you get the code back and it has bugs, either problems with import statements, runtime errors, etc., and you find yourself re-submitting your code with the errors and hoping the LLM gets it right.\n",
    "\n",
    "    This program lets you:\n",
    "\n",
    "    * modify existing code or do something new\n",
    "        * it asks you how many debug iterations you would like\n",
    "        * it then accesses ChatGPT to write the program\n",
    "        * executes the program locally\n",
    "        * detects if there were any errors\n",
    "    * re-submits the new code with the errors\n",
    "        * if no errors, it stops iterating\n",
    "</div>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "90515be5",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Do you want to (1) enter a new code prompt or (2) modify existing code? Enter 1 or 2: 1\n",
      "Please enter the code prompt: make this into a script to run from the command line, with -h and other input parameters:  import openai import os  def read_api_key(file_path):     \"\"\"     Reads the API key from a specified file.      Parameters:     file_path (str): The path to the file containing the API key.      Returns:     str: The API key.     \"\"\"     with open(file_path, 'r') as file:         return file.read().strip()  def extract_code(response, code_type='python'):     \"\"\"     Extracts the code from the OpenAI API response based on the specified type.      Parameters:     response (dict): The response dictionary from the OpenAI API.     code_type (str): The type of code block (default is 'python').      Returns:     str: The extracted code.     \"\"\"     # Extract the message content     content = response['choices'][0]['message']['content']      # Find the start and end of the code block     start_tag = f\"```{code_type}\\n\"     start_index = content.find(start_tag) + len(start_tag)     end_index = content.find(\"\\n```\", start_index)      # Extract the code     code = content[start_index:end_index]          return code  def call_chatgpt(prompt, client, code_type='python', special_instructions='Implement a script in a single code block to perform this task: '):     \"\"\"     Calls the OpenAI API to generate code based on the provided prompt.      Parameters:     prompt (str): The prompt describing the desired code.     client (openai.OpenAI): The OpenAI client.     code_type (str): The type of code block (default is 'python').      Returns:     str: The generated code.     \"\"\"     special_instructions = f\"Implement a {code_type} script in a single code block to perform this task: \"     response = client.chat.completions.create(         #         model=\"gpt-3.5-turbo\",         model=\"gpt-4\",         messages=[             {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},             {\"role\": \"user\", \"content\": special_instructions + prompt}         ]     )     # print(response)  # Add this line to inspect the response structure     return extract_code(response.dict(), code_type=code_type)  def debug_code_with_chatgpt(original_prompt, code, error, client, code_type='python'):     \"\"\"     Calls the OpenAI API to debug the provided code based on the encountered error.      Parameters:     original_prompt (str): The original prompt describing the desired code.     code (str): The code that produced an error.     error (str): The error message.     client (openai.OpenAI): The OpenAI client.     code_type (str): The type of code block (default is 'python').      Returns:     str: The suggested fix for the code.     \"\"\"     prompt = f\"The original prompt was:\\n\\n{original_prompt}\\n\\nHere is the code:\\n\\n{code}\\n\\nIt produced the following error:\\n\\n{error}\\n\\nPlease help me fix it.\"     return call_chatgpt(prompt, client, code_type=code_type)  def save_code_to_file(code, code_type, iteration, status):     \"\"\"     Saves the provided code to a specified file.      Parameters:     code (str): The code to be saved.     code_type (str): The type of code (e.g., python, latex, html).     iteration (int): The iteration number of the code.     status (str): The status of the code (e.g., iteration number or 'debugged').     \"\"\"     # Ensure the scripts directory exists     if not os.path.exists('./scripts'):         os.makedirs('./scripts')      # Determine the file extension based on the code type     ext = 'py' if code_type == 'python' else code_type      file_path = f\"./scripts/script_{status}.{ext}\"     with open(file_path, 'w') as file:         file.write(code)  def main():     \"\"\"     Main function to read API key, generate code, and debug it iteratively.     \"\"\"     api_key = read_api_key('api_key.txt')     openai.api_key = api_key     client = openai      choice = input(\"Do you want to (1) enter a new code prompt or (2) modify existing code? Enter 1 or 2: \")          if choice == '1':         user_prompt = input(\"Please enter the code prompt: \")         code_type = input(\"Please enter the code type (default: python, latex, html, etc.): \") or 'python'         code = call_chatgpt(user_prompt, client, code_type)     elif choice == '2':         file_path = input(\"Please enter the path to the existing code file: \")         with open(file_path, 'r') as file:             code = file.read()         user_prompt = input(\"Please describe the modifications you want to make: \")         code_type = input(\"Please enter the code type (default: python, latex, html, etc.): \") or 'python'         code = call_chatgpt(user_prompt, client, code_type, special_instructions='Modify the following code:\\n\\n' + code + '\\n\\n')     else:         print(\"Invalid choice.\")         return      if code_type == 'latex':         save_code_to_file(code, code_type, 0, 'initial')         print(f\"Code saved to ./scripts/script_initial.{code_type}\")         return      iterations = int(input(\"How many debug iterations should be done? \"))      print('------------------')     print(\"Initial code generated by ChatGPT:\\n\", code)     print('------------------')      save_code_to_file(code, code_type, 0, 'initial')      last_error_message = None      for i in range(1, iterations + 1):         print(f\"\\nIteration {i}:\")          try:             if code_type == 'python':                 exec(code)             else:                 print(\"Execution not supported for this code type in the current implementation.\")                 break             print(\"Code executed successfully.\")             save_code_to_file(code, code_type, i, 'debugged')             print(f'Saved to ./scripts/script_debugged.{code_type}')             break         except Exception as e:             error_message = str(e)             print(\"Error encountered:\", error_message)              if error_message == last_error_message:                 print(\"Encountered the same error. Stopping iterations.\")                 break             last_error_message = error_message                          # Get help from ChatGPT             code = debug_code_with_chatgpt(user_prompt, code, error_message, client, code_type)             print('------------------')             print(\"ChatGPT suggests the following fix:\\n\", code)             print('------------------')              save_code_to_file(code, code_type, i, f'iteration_{i}')      # Save the last version of the code     save_code_to_file(code, code_type, iterations, f'final')     # Determine the file extension based on the code type     ext = 'py' if code_type == 'python' else code_type     print(f\"Final version saved to ./scripts/script_final.{ext}\")  if __name__ == \"__main__\":     main()\n",
      "Please enter the code type (default: python, latex, html, etc.): \n",
      "How many debug iterations should be done? 5\n",
      "------------------\n",
      "Initial code generated by ChatGPT:\n",
      " import argparse\n",
      "import openai\n",
      "import os\n",
      "\n",
      "# function definitions stay the same...\n",
      "\n",
      "def main(file_path, code_prompt=None, code_type='python', debug_iterations=0):\n",
      "    # main function changes to use the arguments file_path, code_prompt, code_type and debug_iterations\n",
      "    # ...\n",
      "\n",
      "if __name__ == \"__main__\":\n",
      "    parser = argparse.ArgumentParser(description=\"Generate and debug code using OpenAI's GPT-3.\")\n",
      "    \n",
      "    parser.add_argument('file_path', type=str, help='The path to the file containing the OpenAI API key')\n",
      "    parser.add_argument('--prompt', '-p', type=str, help='The code prompt for generating the initial code', default=None)\n",
      "    parser.add_argument('--type', '-t', type=str, help='The type of code to generate (default: python)', default='python')\n",
      "    parser.add_argument('--debug', '-d', type=int, help='Number of debug iterations to run (default: 0)', default=0)\n",
      "    \n",
      "    args = parser.parse_args()\n",
      "    \n",
      "    main(args.file_path, args.prompt, args.type, args.debug)\n",
      "------------------\n",
      "\n",
      "Iteration 1:\n",
      "Error encountered: expected an indented block after function definition on line 7 (<string>, line 11)\n",
      "------------------\n",
      "ChatGPT suggests the following fix:\n",
      " import argparse\n",
      "import openai\n",
      "import os\n",
      "\n",
      "def read_api_key(file_path): \n",
      "    with open(file_path, 'r') as file: \n",
      "        return file.read().strip()\n",
      "\n",
      "def extract_code(response, code_type='python'): \n",
      "    content = response['choices'][0]['message']['content']\n",
      "    start_tag = f\"```{code_type}\\n\"\n",
      "    start_index = content.find(start_tag) + len(start_tag)\n",
      "    end_index = content.find(\"\\n```\", start_index)\n",
      "    code = content[start_index:end_index]\n",
      "    return code\n",
      "\n",
      "def call_chatgpt(prompt, client, code_type='python', special_instructions='Implement a script in a single code block to perform this task: '): \n",
      "    special_instructions = f\"Implement a {code_type} script in a single code block to perform this task: \"\n",
      "    response = client.chat.completions.create(\n",
      "        model=\"gpt-4\",\n",
      "        messages=[\n",
      "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
      "            {\"role\": \"user\", \"content\": special_instructions + prompt}\n",
      "        ]\n",
      "    )\n",
      "    return extract_code(response.dict(), code_type=code_type)\n",
      "\n",
      "def debug_code_with_chatgpt(original_prompt, code, error, client, code_type='python'): \n",
      "    prompt = f\"The original prompt was:\\n\\n{original_prompt}\\n\\nHere is the code:\\n\\n{code}\\n\\nIt produced the following error:\\n\\n{error}\\n\\nPlease help me fix it.\"\n",
      "    return call_chatgpt(prompt, client, code_type=code_type) \n",
      "\n",
      "def save_code_to_file(code, code_type, iteration, status): \n",
      "    if not os.path.exists('./scripts'):\n",
      "        os.makedirs('./scripts')\n",
      "\n",
      "    ext = 'py' if code_type == 'python' else code_type\n",
      "    file_path = f\"./scripts/script_{status}.{ext}\"\n",
      "\n",
      "    with open(file_path, 'w') as file:\n",
      "        file.write(code)\n",
      "\n",
      "def main(file_path, code_prompt=None, code_type='python', debug_iterations=0):\n",
      "    api_key = read_api_key(file_path)\n",
      "    openai.api_key = api_key\n",
      "    client = openai\n",
      "    code = call_chatgpt(code_prompt, client, code_type)\n",
      "\n",
      "    save_code_to_file(code, code_type, 0, 'initial')\n",
      "\n",
      "    last_error_message = None\n",
      "\n",
      "    for i in range(1, debug_iterations+1):\n",
      "        try:\n",
      "            if code_type == 'python': exec(code)\n",
      "            else: \n",
      "                print(\"Execution not supported for this code type in the current implementation.\")\n",
      "                break\n",
      "            print(\"Code executed successfully.\")\n",
      "            save_code_to_file(code, code_type, i, 'debugged')\n",
      "            print(f'Saved to ./scripts/script_debugged.{code_type}')\n",
      "            break\n",
      "\n",
      "        except Exception as e:\n",
      "            error_message = str(e)\n",
      "            if error_message == last_error_message: \n",
      "                print(\"Encountered the same error. Stopping iterations.\")\n",
      "                break\n",
      "            last_error_message = error_message\n",
      "            code = debug_code_with_chatgpt(code_prompt, code, error_message, client, code_type)\n",
      "            save_code_to_file(code, code_type, i, f'iteration_{i}')\n",
      "\n",
      "    save_code_to_file(code, code_type, debug_iterations, f'final')\n",
      "    ext = 'py' if code_type == 'python' else code_type\n",
      "    print(f\"Final version saved to ./scripts/script_final.{ext}\")\n",
      "\n",
      "if __name__ == \"__main__\":\n",
      "    parser = argparse.ArgumentParser(description=\"Generate and debug code using OpenAI's GPT-3.\")\n",
      "    \n",
      "    parser.add_argument('file_path', type=str, help='The path to the file containing the OpenAI API key')\n",
      "    parser.add_argument('--prompt', '-p', type=str, help='The code prompt for generating the initial code', default=None)\n",
      "    parser.add_argument('--type', '-t', type=str, help='The type of code to generate (default: python)', default='python')\n",
      "    parser.add_argument('--debug', '-d', type=int, help='Number of debug iterations to run (default: 0)', default=0)\n",
      "    \n",
      "    args = parser.parse_args()\n",
      "    \n",
      "    main(args.file_path, args.prompt, args.type, args.debug)\n",
      "------------------\n",
      "\n",
      "Iteration 2:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "usage: ipykernel_launcher.py [-h] [--prompt PROMPT] [--type TYPE]\n",
      "                             [--debug DEBUG]\n",
      "                             file_path\n",
      "ipykernel_launcher.py: error: unrecognized arguments: -f\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "2",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/galen/anaconda3/lib/python3.11/site-packages/IPython/core/interactiveshell.py:3534: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "import openai\n",
    "import os\n",
    "\n",
    "def read_api_key(file_path):\n",
    "    \"\"\"\n",
    "    Reads the API key from a specified file.\n",
    "\n",
    "    Parameters:\n",
    "    file_path (str): The path to the file containing the API key.\n",
    "\n",
    "    Returns:\n",
    "    str: The API key.\n",
    "    \"\"\"\n",
    "    with open(file_path, 'r') as file:\n",
    "        return file.read().strip()\n",
    "\n",
    "def extract_code(response, code_type='python'):\n",
    "    \"\"\"\n",
    "    Extracts the code from the OpenAI API response based on the specified type.\n",
    "\n",
    "    Parameters:\n",
    "    response (dict): The response dictionary from the OpenAI API.\n",
    "    code_type (str): The type of code block (default is 'python').\n",
    "\n",
    "    Returns:\n",
    "    str: The extracted code.\n",
    "    \"\"\"\n",
    "    # Extract the message content\n",
    "    content = response['choices'][0]['message']['content']\n",
    "\n",
    "    # Find the start and end of the code block\n",
    "    start_tag = f\"```{code_type}\\n\"\n",
    "    start_index = content.find(start_tag) + len(start_tag)\n",
    "    end_index = content.find(\"\\n```\", start_index)\n",
    "\n",
    "    # Extract the code\n",
    "    code = content[start_index:end_index]\n",
    "    \n",
    "    return code\n",
    "\n",
    "def call_chatgpt(prompt, client, code_type='python', special_instructions='Implement a script in a single code block to perform this task: '):\n",
    "    \"\"\"\n",
    "    Calls the OpenAI API to generate code based on the provided prompt.\n",
    "\n",
    "    Parameters:\n",
    "    prompt (str): The prompt describing the desired code.\n",
    "    client (openai.OpenAI): The OpenAI client.\n",
    "    code_type (str): The type of code block (default is 'python').\n",
    "\n",
    "    Returns:\n",
    "    str: The generated code.\n",
    "    \"\"\"\n",
    "    special_instructions = f\"Implement a {code_type} script in a single code block to perform this task: \"\n",
    "    response = client.chat.completions.create(\n",
    "        #         model=\"gpt-3.5-turbo\",\n",
    "        model=\"gpt-4\",\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "            {\"role\": \"user\", \"content\": special_instructions + prompt}\n",
    "        ]\n",
    "    )\n",
    "    # print(response)  # Add this line to inspect the response structure\n",
    "    return extract_code(response.dict(), code_type=code_type)\n",
    "\n",
    "def debug_code_with_chatgpt(original_prompt, code, error, client, code_type='python'):\n",
    "    \"\"\"\n",
    "    Calls the OpenAI API to debug the provided code based on the encountered error.\n",
    "\n",
    "    Parameters:\n",
    "    original_prompt (str): The original prompt describing the desired code.\n",
    "    code (str): The code that produced an error.\n",
    "    error (str): The error message.\n",
    "    client (openai.OpenAI): The OpenAI client.\n",
    "    code_type (str): The type of code block (default is 'python').\n",
    "\n",
    "    Returns:\n",
    "    str: The suggested fix for the code.\n",
    "    \"\"\"\n",
    "    prompt = f\"The original prompt was:\\n\\n{original_prompt}\\n\\nHere is the code:\\n\\n{code}\\n\\nIt produced the following error:\\n\\n{error}\\n\\nPlease help me fix it.\"\n",
    "    return call_chatgpt(prompt, client, code_type=code_type)\n",
    "\n",
    "def save_code_to_file(code, code_type, iteration, status):\n",
    "    \"\"\"\n",
    "    Saves the provided code to a specified file.\n",
    "\n",
    "    Parameters:\n",
    "    code (str): The code to be saved.\n",
    "    code_type (str): The type of code (e.g., python, latex, html).\n",
    "    iteration (int): The iteration number of the code.\n",
    "    status (str): The status of the code (e.g., iteration number or 'debugged').\n",
    "    \"\"\"\n",
    "    # Ensure the scripts directory exists\n",
    "    if not os.path.exists('./scripts'):\n",
    "        os.makedirs('./scripts')\n",
    "\n",
    "    # Determine the file extension based on the code type\n",
    "    ext = 'py' if code_type == 'python' else code_type\n",
    "\n",
    "    file_path = f\"./scripts/script_{status}.{ext}\"\n",
    "    with open(file_path, 'w') as file:\n",
    "        file.write(code)\n",
    "\n",
    "def main():\n",
    "    \"\"\"\n",
    "    Main function to read API key, generate code, and debug it iteratively.\n",
    "    \"\"\"\n",
    "    api_key = read_api_key('api_key.txt')\n",
    "    openai.api_key = api_key\n",
    "    client = openai\n",
    "\n",
    "    choice = input(\"Do you want to (1) enter a new code prompt or (2) modify existing code? Enter 1 or 2: \")\n",
    "    \n",
    "    if choice == '1':\n",
    "        user_prompt = input(\"Please enter the code prompt: \")\n",
    "        code_type = input(\"Please enter the code type (default: python, latex, html, etc.): \") or 'python'\n",
    "        code = call_chatgpt(user_prompt, client, code_type)\n",
    "    elif choice == '2':\n",
    "        file_path = input(\"Please enter the path to the existing code file: \")\n",
    "        with open(file_path, 'r') as file:\n",
    "            code = file.read()\n",
    "        user_prompt = input(\"Please describe the modifications you want to make: \")\n",
    "        code_type = input(\"Please enter the code type (default: python, latex, html, etc.): \") or 'python'\n",
    "        code = call_chatgpt(user_prompt, client, code_type, special_instructions='Modify the following code:\\n\\n' + code + '\\n\\n')\n",
    "    else:\n",
    "        print(\"Invalid choice.\")\n",
    "        return\n",
    "\n",
    "    if code_type == 'latex':\n",
    "        save_code_to_file(code, code_type, 0, 'initial')\n",
    "        print(f\"Code saved to ./scripts/script_initial.{code_type}\")\n",
    "        return\n",
    "\n",
    "    iterations = int(input(\"How many debug iterations should be done? \"))\n",
    "\n",
    "    print('------------------')\n",
    "    print(\"Initial code generated by ChatGPT:\\n\", code)\n",
    "    print('------------------')\n",
    "\n",
    "    save_code_to_file(code, code_type, 0, 'initial')\n",
    "\n",
    "    last_error_message = None\n",
    "\n",
    "    for i in range(1, iterations + 1):\n",
    "        print(f\"\\nIteration {i}:\")\n",
    "\n",
    "        try:\n",
    "            if code_type == 'python':\n",
    "                exec(code)\n",
    "            else:\n",
    "                print(\"Execution not supported for this code type in the current implementation.\")\n",
    "                break\n",
    "            print(\"Code executed successfully.\")\n",
    "            save_code_to_file(code, code_type, i, 'debugged')\n",
    "            print(f'Saved to ./scripts/script_debugged.{code_type}')\n",
    "            break\n",
    "        except Exception as e:\n",
    "            error_message = str(e)\n",
    "            print(\"Error encountered:\", error_message)\n",
    "\n",
    "            if error_message == last_error_message:\n",
    "                print(\"Encountered the same error. Stopping iterations.\")\n",
    "                break\n",
    "            last_error_message = error_message\n",
    "            \n",
    "            # Get help from ChatGPT\n",
    "            code = debug_code_with_chatgpt(user_prompt, code, error_message, client, code_type)\n",
    "            print('------------------')\n",
    "            print(\"ChatGPT suggests the following fix:\\n\", code)\n",
    "            print('------------------')\n",
    "\n",
    "            save_code_to_file(code, code_type, i, f'iteration_{i}')\n",
    "\n",
    "    # Save the last version of the code\n",
    "    save_code_to_file(code, code_type, iterations, f'final')\n",
    "    # Determine the file extension based on the code type\n",
    "    ext = 'py' if code_type == 'python' else code_type\n",
    "    print(f\"Final version saved to ./scripts/script_final.{ext}\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd56b773",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}