POC.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A3ohlM-B3EAl",
        "outputId": "05d181a0-ce27-4659-ba31-51149a585941"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting openai\n",
            "  Downloading openai-1.25.1-py3-none-any.whl (312 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m312.9/312.9 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai)\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.7.1)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.2)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.11.0)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.2.2)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.18.2)\n",
            "Installing collected packages: h11, httpcore, httpx, openai\n",
            "Successfully installed h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 openai-1.25.1\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.10/dist-packages (3.7.4)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (8.2.3)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.4.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.3.4)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.9.4)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (4.66.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.7.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (24.0)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.4.0)\n",
            "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.25.2)\n",
            "Requirement already satisfied: language-data>=1.2 in /usr/local/lib/python3.10/dist-packages (from langcodes<4.0.0,>=3.2.0->spacy) (1.2.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (2.18.2)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy) (8.1.7)\n",
            "Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<0.4.0,>=0.1.0->spacy) (0.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy) (2.1.5)\n",
            "Requirement already satisfied: marisa-trie>=0.7.7 in /usr/local/lib/python3.10/dist-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy) (1.1.0)\n",
            "Collecting zh-core-web-sm==3.7.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.7.0/zh_core_web_sm-3.7.0-py3-none-any.whl (48.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.5/48.5 MB\u001b[0m \u001b[31m15.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.8.0,>=3.7.0 in /usr/local/lib/python3.10/dist-packages (from zh-core-web-sm==3.7.0) (3.7.4)\n",
            "Collecting spacy-pkuseg<0.1.0,>=0.0.27 (from zh-core-web-sm==3.7.0)\n",
            "  Downloading spacy_pkuseg-0.0.33-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (8.2.3)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.4.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (0.3.4)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (0.9.4)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (4.66.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.7.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (24.0)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (3.4.0)\n",
            "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (1.25.2)\n",
            "Requirement already satisfied: language-data>=1.2 in /usr/local/lib/python3.10/dist-packages (from langcodes<4.0.0,>=3.2.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (1.2.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.18.2)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (8.1.7)\n",
            "Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<0.4.0,>=0.1.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (0.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (2.1.5)\n",
            "Requirement already satisfied: marisa-trie>=0.7.7 in /usr/local/lib/python3.10/dist-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy<3.8.0,>=3.7.0->zh-core-web-sm==3.7.0) (1.1.0)\n",
            "Installing collected packages: spacy-pkuseg, zh-core-web-sm\n",
            "Successfully installed spacy-pkuseg-0.0.33 zh-core-web-sm-3.7.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('zh_core_web_sm')\n",
            "\u001b[38;5;3m⚠ Restart to reload dependencies\u001b[0m\n",
            "If you are in a Jupyter or Colab notebook, you may need to restart Python in\n",
            "order to load all the package's dependencies. You can do this by selecting the\n",
            "'Restart kernel' or 'Restart runtime' option.\n"
          ]
        }
      ],
      "source": [
        "#enviroment\n",
        "!pip install openai\n",
        "!pip install spacy\n",
        "!python -m spacy download zh_core_web_sm\n",
        "\n",
        "\n",
        "import openai\n",
        "from openai import OpenAI\n",
        "import os\n",
        "import spacy\n",
        "\n",
        "OPENAI_API_KEY = 'sk-proj-o9TONJi0MW2tSiDMhRkxT3BlbkFJkUr03XQ5IfUaxamV0e3k'\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zZJkoxNIEnUL"
      },
      "source": [
        "Entity Recognition, Glossary Databse(can be replaced with knowledge graph) Part."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yQsHeY_GOz66",
        "outputId": "7fd52e18-22e9-47e6-dc59-bdcf2f5a05c3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Identified entities: {'黑光纤': 'dark fiber'}\n"
          ]
        }
      ],
      "source": [
        "nlp = spacy.load(\"zh_core_web_sm\")\n",
        "\n",
        "glossary_db = {\n",
        "    '黑光纤': 'dark fiber',\n",
        "    '光纤': 'optical fiber',\n",
        "}\n",
        "\n",
        "def identify_entities(source_text):\n",
        "    doc = nlp(source_text)\n",
        "    entities = []\n",
        "    translations = {}\n",
        "    for ent in doc.ents:\n",
        "        term = ent.text.lower()\n",
        "        if term in glossary_db:\n",
        "            translations[ent.text] = glossary_db[term]\n",
        "    return translations\n",
        "\n",
        "text = \"光纤照上去变成黑光纤了\"\n",
        "translations = identify_entities(text)\n",
        "\n",
        "print(\"Identified entities:\", translations)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e0oo_njSFG3D"
      },
      "source": [
        "Prompt generation based on idea of RAG."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-Dv_Zf9LU5o1",
        "outputId": "19b7c08b-c065-4dcd-9971-439e7ca8aad6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Based on the translation of following term: '黑光纤': 'dark fiber'.Translate the following text from zh into en: 光纤照上去变成黑光纤了\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# prompt with entity translation\n",
        "def prompt_generator(text, source_language, target_language):\n",
        "  terms = identify_entities(text)\n",
        "  terms_translations = '. '.join([f\"'{k}': '{v}'\" for k, v in terms.items()])\n",
        "  translations = f\"Based on the translation of following term: {terms_translations}.\"\n",
        "  prompt = f\"Translate the following text from {source_language} into {target_language}: {text}\\n\"\n",
        "  if terms == {}:\n",
        "    return prompt\n",
        "  prompt = translations + prompt\n",
        "\n",
        "  return prompt\n",
        "\n",
        "prompt = prompt_generator('光纤照上去变成黑光纤了', 'zh', 'en')\n",
        "print(prompt)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NOyAbrUPFZRS"
      },
      "source": [
        "Machine Translation code, can be replaced by other models."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rsJui7hwOuaj"
      },
      "outputs": [],
      "source": [
        "def translate_text(prompt):\n",
        "\n",
        "    client = OpenAI(api_key= OPENAI_API_KEY,)\n",
        "    print(prompt)\n",
        "    response = client.chat.completions.create(\n",
        "      messages=[{\n",
        "            \"role\": \"user\",\n",
        "            \"content\": prompt,\n",
        "          }],\n",
        "      model=\"gpt-3.5-turbo\",)\n",
        "    translation = response.choices[0].message.content.strip().split(\"\\n\")[0]\n",
        "    return translation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UHilkZE9O_pR",
        "outputId": "afc4a6e0-594d-4674-e7b9-23a027b86b3a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Based on the translation of following term: '黑光纤': 'dark fiber'.Translate the following text from zh into en: 光纤照上去变成黑光纤了\n",
            "\n",
            "When the light shines on it, the optical fiber turns into dark fiber.\n"
          ]
        }
      ],
      "source": [
        "#test\n",
        "prompt = prompt_generator('光纤照上去变成黑光纤了', 'zh', 'en')\n",
        "translation = translate_text(prompt)\n",
        "print(translation)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o8wsTQhTGUk8"
      },
      "source": [
        "Quality Estimation Part"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zkyo6iJGGaUL"
      },
      "outputs": [],
      "source": [
        "def quality_estimator(original_text, translated_text):\n",
        "  client = OpenAI(api_key= OPENAI_API_KEY,)\n",
        "  prompt = f\"Evaluate the quality estimation of the following source and translation sentence pairs by following a step-by-step process: \\\n",
        "    Step 1: Estimate the perplexity of the translated sentence.\\\n",
        "    Step 2: Determine the token-level similarity between the source and translatedsentences.\\\n",
        "    Step 3: Combine the results and classify the translation quality into one of the following categories:'No meaning preserved', 'Some meaning preserved, but not understandable', 'Some meaning preserved and understandable', 'Most meaningpreserved, minor issues',or 'Perfect translation'.\\\n",
        "    Source:{original_text}.Translation:{translated_text}\"\n",
        "  print(prompt)\n",
        "  response = client.chat.completions.create(\n",
        "    messages=[{\n",
        "          \"role\": \"user\",\n",
        "          \"content\": prompt,\n",
        "        }],\n",
        "    model=\"gpt-3.5-turbo\",\n",
        "    )\n",
        "  result = response.choices[0].message.content\n",
        "  return result"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lwh5z0ZigpYI"
      },
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d5iBtZFsXdaN"
      },
      "outputs": [],
      "source": [
        "def quality_classifier(evaluation):\n",
        "  start_index = evaluation.find(\"'\")\n",
        "  end_index = evaluation.find(\"'\", start_index + 1)\n",
        "  category = evaluation[start_index+1:end_index]\n",
        "  return category"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VgAzWZBpN2uM",
        "outputId": "0cd3aed8-5017-458f-e46f-6adf6421c247"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Evaluate the quality estimation of the following source and translation sentence pairs by following a step-by-step process:     Step 1: Estimate the perplexity of the translated sentence.    Step 2: Determine the token-level similarity between the source and translatedsentences.    Step 3: Combine the results and classify the translation quality into one of the following categories:'No meaning preserved', 'Some meaning preserved, but not understandable', 'Some meaning preserved and understandable', 'Most meaningpreserved, minor issues',or 'Perfect translation'.    Source:光纤照上去变成黑光纤了.Translation:The fiber optic cable shines and turns into dark fiber.\n",
            "Step 1: Estimate the perplexity of the translated sentence.\n",
            "The translated sentence is grammatically correct and coherent, so the perplexity is relatively low.\n",
            "\n",
            "Step 2: Determine the token-level similarity between the source and translated sentences.\n",
            "Source tokens: 光纤 照 上去 变成 黑 光纤 了\n",
            "Translation tokens: The fiber optic cable shines and turns into dark fiber\n",
            "Token-level similarity: The tokens do not match exactly, but the overall meaning is preserved. \n",
            "\n",
            "Step 3: Combine the results and classify the translation quality\n",
            "Based on the low perplexity and the preservation of the overall meaning, the translation quality can be classified as 'Some meaning preserved and understandable'. The translation conveys the general idea of the source sentence, but there are some discrepancies in word choice and accuracy.\n"
          ]
        }
      ],
      "source": [
        "original_text = \"光纤照上去变成黑光纤了\"\n",
        "translated_text = \"The fiber optic cable shines and turns into dark fiber.\"\n",
        "evaluation = quality_estimator(original_text, translated_text)\n",
        "print(evaluation)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E_kbreI1X4cH",
        "outputId": "64b090c4-65d3-4cf9-bba2-2fd9ebc66bb0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Some meaning preserved and understandable\n"
          ]
        }
      ],
      "source": [
        "category = quality_classifier(evaluation)\n",
        "print(category)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2ciQsduFUn-X"
      },
      "outputs": [],
      "source": [
        "def refine_prompt(original_text, translated_text, evaluation):\n",
        "  prompt = f\"Based on the quality estimation of the following source and translation sentence pairs:{original_text}.Translation:{translated_text}.Category:{evaluation}, provide a refined translation:\"\n",
        "  return prompt\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Ni9dZR3VGy7",
        "outputId": "63a190e6-583b-42b2-bb1b-9a340a2e717c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Based on the quality estimation of the following source and translation sentence pairs:光纤照上去变成黑光纤了.Translation:The fiber optic cable shines and turns into dark fiber..Category:Step 1: Estimate the perplexity of the translated sentence.\n",
            "The translated sentence is grammatically correct and coherent, so the perplexity is relatively low.\n",
            "\n",
            "Step 2: Determine the token-level similarity between the source and translated sentences.\n",
            "Source tokens: 光纤 照 上去 变成 黑 光纤 了\n",
            "Translation tokens: The fiber optic cable shines and turns into dark fiber\n",
            "Token-level similarity: The tokens do not match exactly, but the overall meaning is preserved. \n",
            "\n",
            "Step 3: Combine the results and classify the translation quality\n",
            "Based on the low perplexity and the preservation of the overall meaning, the translation quality can be classified as 'Some meaning preserved and understandable'. The translation conveys the general idea of the source sentence, but there are some discrepancies in word choice and accuracy., provide a refined translation:\n",
            "The fiber optic cable shone and turned into dark fiber.\n"
          ]
        }
      ],
      "source": [
        "prompt = refine_prompt(original_text, translated_text, evaluation)\n",
        "new_translation = translate_text(prompt)\n",
        "print(new_translation)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QWT8eEvnFcaf"
      },
      "source": [
        "User iterface action: edit context(providing few shot for style guide) and rephrase(generate alternative translation)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RIbvLT2PKmrT",
        "outputId": "8a6805ed-f93c-471a-8d0d-2b49a06373c2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CHINESE TO ENGLISH EXAMPLE\n",
            "Translate \"光纤照上去变成黑光纤了\" from the zh language, into the en language\n",
            "                      using the following context style: \"A poetic two line rhyming style\".\n",
            "                      Here are some examples of the style being used: \"Light speeds away, in a blink it will fly.\"\n",
            "              \"Stars reach from afar, with light from eons bold.\"\n",
            "           \n",
            "                      \n",
            "\"Fiber glass shining bright, turns into darkness in the light.\" \n",
            "\n",
            "\n",
            "\n",
            "Translate \"光纤照上去变成黑光纤了\" from the zh language, into the en language\n",
            "                      using the following context style: \"A confused and inquisitive style\".\n",
            "                      Here are some examples of the style being used: \"How does it work so well when it's so unclear?\"\n",
            "              \"Why did it begin so suddenly, what triggered it?\"\n",
            "            \n",
            "                      \n",
            "How did the fiber optic become black when it was lit up? \n",
            "\n",
            "\n",
            "\n",
            "ENGLISH TO CHINESE EXAMPLE\n",
            "Translate \"Lose track of time I'm bugging\" from the en language, into the zh language\n",
            "                      using the following context style: \"A poetic two line rhyming style\".\n",
            "                      Here are some examples of the style being used: \"光逝如飞，瞬息即逝。\"\n",
            "              \"星光遥来，古光犹豪。\"\n",
            "           \n",
            "                      \n",
            "\"时光匆匆，不知何往。\" \n",
            "\n",
            "\n",
            "\n",
            "Translate \"Lose track of time I'm bugging\" from the en language, into the zh language\n",
            "                      using the following context style: \"A confused and inquisitive style\".\n",
            "                      Here are some examples of the style being used: \"当它如此不清楚时，它如何运作得这么好？\"\n",
            "              \"为什么它开始得这么突然，是什么触发了它？\"\n",
            "            \n",
            "                      \n",
            "\"失去时间观念我很困扰\" \n"
          ]
        }
      ],
      "source": [
        "# Few-shot context changing via user input\n",
        "def edit_context(context, original_text, source_language, target_language, few_shot_examples):\n",
        "\n",
        "  edited_context = f'''Translate \"{original_text}\" from the {source_language} language, into the {target_language} language\n",
        "                      using the following context style: \"{context}\".\n",
        "                      Here are some examples of the style being used: {few_shot_examples}\n",
        "                      '''\n",
        "  return translate_text(source_language, target_language, edited_context)\n",
        "\n",
        "\n",
        "# Chinese to Chinese Example\n",
        "print(\"CHINESE TO ENGLISH EXAMPLE\")\n",
        "\n",
        "# Poetic Example\n",
        "examples = '''\"Light speeds away, in a blink it will fly.\"\n",
        "              \"Stars reach from afar, with light from eons bold.\"\n",
        "           '''\n",
        "\n",
        "print(edit_context(\"A poetic two line rhyming style\", \"光纤照上去变成黑光纤了\", \"zh\", \"en\", examples), \"\\n\\n\\n\")\n",
        "\n",
        "# Confused and Inquisitive Example\n",
        "examples = '''\"How does it work so well when it's so unclear?\"\n",
        "              \"Why did it begin so suddenly, what triggered it?\"\n",
        "            '''\n",
        "\n",
        "print(edit_context(\"A confused and inquisitive style\", \"光纤照上去变成黑光纤了\", \"zh\", \"en\", examples), \"\\n\\n\\n\")\n",
        "\n",
        "\n",
        "\n",
        "# English to Chinese Example\n",
        "print(\"ENGLISH TO CHINESE EXAMPLE\")\n",
        "\n",
        "# Poetic Example\n",
        "examples = '''\"光逝如飞，瞬息即逝。\"\n",
        "              \"星光遥来，古光犹豪。\"\n",
        "           '''\n",
        "\n",
        "print(edit_context(\"A poetic two line rhyming style\", \"Lose track of time I'm bugging\", \"en\", \"zh\", examples), \"\\n\\n\\n\")\n",
        "\n",
        "# Confused and Inquisitive Example\n",
        "examples = '''\"当它如此不清楚时，它如何运作得这么好？\"\n",
        "              \"为什么它开始得这么突然，是什么触发了它？\"\n",
        "            '''\n",
        "\n",
        "print(edit_context(\"A confused and inquisitive style\", \"Lose track of time I'm bugging\", \"en\", \"zh\", examples))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        },
        "id": "Gg6GdbTZKnif",
        "outputId": "60814cfe-dfb2-489e-cf80-9eed30c9f063"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Please provide an alternative translation to and rephrase: \"The fiber optic turned into dark fiber when the light shone on it.\" \n",
            "          based on the term '黑光纤': 'dark fiber'.\n",
            "          The original text was 光纤照上去变成黑光纤了.\n",
            "          The source language is zh and the target language is en.\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"The optical fiber transformed into dark fiber under the light's illumination.\""
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "# Feeding the model (OpenAI API) the original translation meta descriptors and prompting for a new/rephrased/alternate translation\n",
        "def rephrase(old_translation, original_source_text, source_language, target_language):\n",
        "\n",
        "  # Get entites so they don't have to be passed, simplifying the function call. Can be changed to passing if more efficient\n",
        "  terms = identify_entities(original_source_text)\n",
        "  entities_joined = '. '.join([f\"'{k}': '{v}'\" for k, v in terms.items()])\n",
        "\n",
        "  # Construct the prompt for rephrased/alternate translation\n",
        "  prompt = f'''Please provide an alternative translation to and rephrase: \"{old_translation}\"\n",
        "          based on the term {entities_joined}.\n",
        "          The original text was {original_source_text}.\n",
        "          The source language is {source_language} and the target language is {target_language}.'''\n",
        "\n",
        "  # Make the call to the model\n",
        "  return translate_text(prompt)\n",
        "\n",
        "# Get alternative translation for demo text\n",
        "rephrase(\"The fiber optic turned into dark fiber when the light shone on it.\", \"光纤照上去变成黑光纤了\", \"zh\", \"en\")\n",
        "\n",
        "\n",
        "# Probably want to use the Translation memory/storage for avoiding repeat translations and get unique results each time"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vvqu8zJ-GH5S"
      },
      "source": [
        "Interactive code for testing(pipeline)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 364
        },
        "id": "i-5nRXM1O4Et",
        "outputId": "d0df25cb-aa75-4eff-df3b-f80b7456b9b3"
      },
      "outputs": [
        {
          "ename": "KeyboardInterrupt",
          "evalue": "Interrupted by user",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-93-c2f3bb3c1c12>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Get user input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Enter the text to translate: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0msource_language\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Enter the source language: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtarget_language\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Enter the target language: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprompt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprompt_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_language\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_language\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprompt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m    849\u001b[0m                 \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m             )\n\u001b[0;32m--> 851\u001b[0;31m         return self._input_request(str(prompt),\n\u001b[0m\u001b[1;32m    852\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    853\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_header\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m    893\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m                 \u001b[0;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Interrupted by user\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    896\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid Message:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: Interrupted by user"
          ]
        }
      ],
      "source": [
        "# Get user input\n",
        "text = input(\"Enter the text to translate: \")\n",
        "source_language = input(\"Enter the source language: \")\n",
        "target_language = input(\"Enter the target language: \")\n",
        "prompt = prompt_generator(source_language, target_language, prompt)\n",
        "translation = translate_text(prompt)\n",
        "evaluation = quality_estimator(text, translation)\n",
        "category = quality_classifier(evaluation)\n",
        "if category == 'Most meaningpreserved, minor issues' or category == 'Perfect translation':\n",
        "  print('Accepted')\n",
        "else:\n",
        "  translation = refine_prompt(text, translation, evaluation)\n",
        "print(translation)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}