modeling/single-layer-LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyONT/EaTKxX0mcQMq3VsRHX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/faezesarlakifar/AllerTrans/blob/main/modeling/single-layer-LSTM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "XeftCMsKQTeN"
      },
      "outputs": [],
      "source": [
        "# @markdown import necessaries\n",
        "from google.colab import drive\n",
        "from tqdm.notebook import tqdm\n",
        "import torch\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "import torch.nn as nn\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "import numpy as np\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from sklearn.metrics import precision_recall_fscore_support"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @markdown mount google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form",
        "id": "v_GUX9IURjEs",
        "outputId": "6afe2c7f-7dd2-4a1d-86eb-4c42d9bf2935"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/allergen-detection/embeddings/'"
      ],
      "metadata": {
        "id": "_iX9XdH1WOhE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @markdown load dataset\n",
        "\n",
        "esm_path = path+'esm-embeddings-with-id/'\n",
        "esm_train = pd.read_csv(esm_path+'esm_train.csv', index_col=0)\n",
        "esm_test = pd.read_csv(esm_path+'esm_test.csv', index_col=0)\n",
        "\n",
        "protbert_path = path+'protBERT-embeddings-with-id/'\n",
        "protbert_train = pd.read_csv(protbert_path+'protBERT_train.csv', index_col=0)\n",
        "protbert_test = pd.read_csv(protbert_path+'protBERT_test.csv', index_col=0)"
      ],
      "metadata": {
        "id": "ss0Wod1kQcIU",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "esm_train.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "YeWiya6JPmmw",
        "outputId": "43ac5e22-3980-48e0-9f06-387e7e5c263b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          0         1         2         3         4         5         6  \\\n",
              "0 -0.003870 -0.033864 -0.047640  0.042528  0.008789 -0.115301  0.116311   \n",
              "1 -0.027616 -0.047448 -0.092680 -0.037921 -0.064493 -0.085917  0.032686   \n",
              "2 -0.060472 -0.009816 -0.085757 -0.042129 -0.005332 -0.114610 -0.023706   \n",
              "3 -0.001879 -0.132213  0.030315  0.017521  0.093525 -0.132252  0.067915   \n",
              "4 -0.004742 -0.012900 -0.069494  0.027048 -0.110435 -0.009641  0.055032   \n",
              "\n",
              "          7         8         9  ...      1272      1273      1274      1275  \\\n",
              "0 -0.056773 -0.034452  0.007469  ...  0.006811  0.046086 -0.028301  0.006903   \n",
              "1 -0.104749  0.044072  0.136332  ... -0.005730 -0.007972 -0.110330  0.143354   \n",
              "2 -0.015580  0.004648  0.080403  ... -0.015167 -0.000249 -0.056284  0.046674   \n",
              "3  0.093191  0.031767  0.070513  ... -0.122606 -0.032942  0.069062 -0.176981   \n",
              "4 -0.205043  0.036597  0.031489  ...  0.036046 -0.113270 -0.020870 -0.023917   \n",
              "\n",
              "       1276      1277      1278      1279  Label        id  \n",
              "0  0.048489 -0.044930 -0.022138  0.003444      0   N_98814  \n",
              "1  0.079251 -0.200938 -0.064969 -0.010540      1    P_8856  \n",
              "2  0.075886 -0.093865 -0.053575  0.069379      1    P_7716  \n",
              "3  0.018370 -0.074525  0.126718  0.107524      0  N_326869  \n",
              "4  0.021013 -0.146265 -0.051615  0.176472      1    P_8031  \n",
              "\n",
              "[5 rows x 1282 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-4faaa53c-0986-4acc-ba17-7741a1a9fb84\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>1272</th>\n",
              "      <th>1273</th>\n",
              "      <th>1274</th>\n",
              "      <th>1275</th>\n",
              "      <th>1276</th>\n",
              "      <th>1277</th>\n",
              "      <th>1278</th>\n",
              "      <th>1279</th>\n",
              "      <th>Label</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.003870</td>\n",
              "      <td>-0.033864</td>\n",
              "      <td>-0.047640</td>\n",
              "      <td>0.042528</td>\n",
              "      <td>0.008789</td>\n",
              "      <td>-0.115301</td>\n",
              "      <td>0.116311</td>\n",
              "      <td>-0.056773</td>\n",
              "      <td>-0.034452</td>\n",
              "      <td>0.007469</td>\n",
              "      <td>...</td>\n",
              "      <td>0.006811</td>\n",
              "      <td>0.046086</td>\n",
              "      <td>-0.028301</td>\n",
              "      <td>0.006903</td>\n",
              "      <td>0.048489</td>\n",
              "      <td>-0.044930</td>\n",
              "      <td>-0.022138</td>\n",
              "      <td>0.003444</td>\n",
              "      <td>0</td>\n",
              "      <td>N_98814</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.027616</td>\n",
              "      <td>-0.047448</td>\n",
              "      <td>-0.092680</td>\n",
              "      <td>-0.037921</td>\n",
              "      <td>-0.064493</td>\n",
              "      <td>-0.085917</td>\n",
              "      <td>0.032686</td>\n",
              "      <td>-0.104749</td>\n",
              "      <td>0.044072</td>\n",
              "      <td>0.136332</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.005730</td>\n",
              "      <td>-0.007972</td>\n",
              "      <td>-0.110330</td>\n",
              "      <td>0.143354</td>\n",
              "      <td>0.079251</td>\n",
              "      <td>-0.200938</td>\n",
              "      <td>-0.064969</td>\n",
              "      <td>-0.010540</td>\n",
              "      <td>1</td>\n",
              "      <td>P_8856</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.060472</td>\n",
              "      <td>-0.009816</td>\n",
              "      <td>-0.085757</td>\n",
              "      <td>-0.042129</td>\n",
              "      <td>-0.005332</td>\n",
              "      <td>-0.114610</td>\n",
              "      <td>-0.023706</td>\n",
              "      <td>-0.015580</td>\n",
              "      <td>0.004648</td>\n",
              "      <td>0.080403</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.015167</td>\n",
              "      <td>-0.000249</td>\n",
              "      <td>-0.056284</td>\n",
              "      <td>0.046674</td>\n",
              "      <td>0.075886</td>\n",
              "      <td>-0.093865</td>\n",
              "      <td>-0.053575</td>\n",
              "      <td>0.069379</td>\n",
              "      <td>1</td>\n",
              "      <td>P_7716</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.001879</td>\n",
              "      <td>-0.132213</td>\n",
              "      <td>0.030315</td>\n",
              "      <td>0.017521</td>\n",
              "      <td>0.093525</td>\n",
              "      <td>-0.132252</td>\n",
              "      <td>0.067915</td>\n",
              "      <td>0.093191</td>\n",
              "      <td>0.031767</td>\n",
              "      <td>0.070513</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.122606</td>\n",
              "      <td>-0.032942</td>\n",
              "      <td>0.069062</td>\n",
              "      <td>-0.176981</td>\n",
              "      <td>0.018370</td>\n",
              "      <td>-0.074525</td>\n",
              "      <td>0.126718</td>\n",
              "      <td>0.107524</td>\n",
              "      <td>0</td>\n",
              "      <td>N_326869</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-0.004742</td>\n",
              "      <td>-0.012900</td>\n",
              "      <td>-0.069494</td>\n",
              "      <td>0.027048</td>\n",
              "      <td>-0.110435</td>\n",
              "      <td>-0.009641</td>\n",
              "      <td>0.055032</td>\n",
              "      <td>-0.205043</td>\n",
              "      <td>0.036597</td>\n",
              "      <td>0.031489</td>\n",
              "      <td>...</td>\n",
              "      <td>0.036046</td>\n",
              "      <td>-0.113270</td>\n",
              "      <td>-0.020870</td>\n",
              "      <td>-0.023917</td>\n",
              "      <td>0.021013</td>\n",
              "      <td>-0.146265</td>\n",
              "      <td>-0.051615</td>\n",
              "      <td>0.176472</td>\n",
              "      <td>1</td>\n",
              "      <td>P_8031</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 1282 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4faaa53c-0986-4acc-ba17-7741a1a9fb84')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-f561abc2-4a23-46db-ba3a-49bdd2c0cbc3\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f561abc2-4a23-46db-ba3a-49bdd2c0cbc3')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-f561abc2-4a23-46db-ba3a-49bdd2c0cbc3 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4faaa53c-0986-4acc-ba17-7741a1a9fb84 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4faaa53c-0986-4acc-ba17-7741a1a9fb84');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "esm_test.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "hRoBRoamQiu5",
        "outputId": "25f08f59-3762-46db-fb27-3c770c9d10c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          0         1         2         3         4         5         6  \\\n",
              "0 -0.013756 -0.056206 -0.010463  0.078954 -0.013414 -0.036181  0.101913   \n",
              "1  0.138220 -0.015468 -0.014704 -0.045518 -0.017561  0.011617 -0.093774   \n",
              "2  0.008007 -0.054583 -0.012078  0.037711 -0.046313 -0.094285  0.004338   \n",
              "3 -0.074227 -0.024113 -0.001488 -0.026625 -0.056001  0.005185 -0.002422   \n",
              "4  0.035093 -0.027917  0.017686  0.011974  0.123146 -0.051364 -0.024959   \n",
              "\n",
              "          7         8         9  ...      1272      1273      1274      1275  \\\n",
              "0 -0.078302 -0.029859  0.155479  ... -0.108262 -0.018523  0.023435 -0.086668   \n",
              "1 -0.057526 -0.016540  0.202359  ... -0.109726  0.094971  0.022741 -0.153795   \n",
              "2 -0.064758 -0.096635  0.059523  ... -0.036893  0.058147 -0.039131  0.076175   \n",
              "3 -0.120850 -0.008313 -0.003280  ... -0.065336  0.041332 -0.066702  0.075858   \n",
              "4  0.176133 -0.043006 -0.027118  ... -0.204552 -0.052534 -0.056887  0.034603   \n",
              "\n",
              "       1276      1277      1278      1279  Label        id  \n",
              "0  0.084220 -0.138123  0.000659  0.111366      0  N_528450  \n",
              "1  0.289212 -0.096826 -0.012760 -0.088842      0  N_479266  \n",
              "2  0.136362 -0.062218 -0.023998  0.022915      1     P_271  \n",
              "3 -0.057383 -0.165823  0.100414  0.105647      0  N_217429  \n",
              "4  0.107830 -0.013466  0.075048 -0.013248      1    P_7456  \n",
              "\n",
              "[5 rows x 1282 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-e094c353-d69f-4e42-aee0-c67d4d24dc3f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>1272</th>\n",
              "      <th>1273</th>\n",
              "      <th>1274</th>\n",
              "      <th>1275</th>\n",
              "      <th>1276</th>\n",
              "      <th>1277</th>\n",
              "      <th>1278</th>\n",
              "      <th>1279</th>\n",
              "      <th>Label</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.013756</td>\n",
              "      <td>-0.056206</td>\n",
              "      <td>-0.010463</td>\n",
              "      <td>0.078954</td>\n",
              "      <td>-0.013414</td>\n",
              "      <td>-0.036181</td>\n",
              "      <td>0.101913</td>\n",
              "      <td>-0.078302</td>\n",
              "      <td>-0.029859</td>\n",
              "      <td>0.155479</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.108262</td>\n",
              "      <td>-0.018523</td>\n",
              "      <td>0.023435</td>\n",
              "      <td>-0.086668</td>\n",
              "      <td>0.084220</td>\n",
              "      <td>-0.138123</td>\n",
              "      <td>0.000659</td>\n",
              "      <td>0.111366</td>\n",
              "      <td>0</td>\n",
              "      <td>N_528450</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.138220</td>\n",
              "      <td>-0.015468</td>\n",
              "      <td>-0.014704</td>\n",
              "      <td>-0.045518</td>\n",
              "      <td>-0.017561</td>\n",
              "      <td>0.011617</td>\n",
              "      <td>-0.093774</td>\n",
              "      <td>-0.057526</td>\n",
              "      <td>-0.016540</td>\n",
              "      <td>0.202359</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.109726</td>\n",
              "      <td>0.094971</td>\n",
              "      <td>0.022741</td>\n",
              "      <td>-0.153795</td>\n",
              "      <td>0.289212</td>\n",
              "      <td>-0.096826</td>\n",
              "      <td>-0.012760</td>\n",
              "      <td>-0.088842</td>\n",
              "      <td>0</td>\n",
              "      <td>N_479266</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.008007</td>\n",
              "      <td>-0.054583</td>\n",
              "      <td>-0.012078</td>\n",
              "      <td>0.037711</td>\n",
              "      <td>-0.046313</td>\n",
              "      <td>-0.094285</td>\n",
              "      <td>0.004338</td>\n",
              "      <td>-0.064758</td>\n",
              "      <td>-0.096635</td>\n",
              "      <td>0.059523</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.036893</td>\n",
              "      <td>0.058147</td>\n",
              "      <td>-0.039131</td>\n",
              "      <td>0.076175</td>\n",
              "      <td>0.136362</td>\n",
              "      <td>-0.062218</td>\n",
              "      <td>-0.023998</td>\n",
              "      <td>0.022915</td>\n",
              "      <td>1</td>\n",
              "      <td>P_271</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.074227</td>\n",
              "      <td>-0.024113</td>\n",
              "      <td>-0.001488</td>\n",
              "      <td>-0.026625</td>\n",
              "      <td>-0.056001</td>\n",
              "      <td>0.005185</td>\n",
              "      <td>-0.002422</td>\n",
              "      <td>-0.120850</td>\n",
              "      <td>-0.008313</td>\n",
              "      <td>-0.003280</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.065336</td>\n",
              "      <td>0.041332</td>\n",
              "      <td>-0.066702</td>\n",
              "      <td>0.075858</td>\n",
              "      <td>-0.057383</td>\n",
              "      <td>-0.165823</td>\n",
              "      <td>0.100414</td>\n",
              "      <td>0.105647</td>\n",
              "      <td>0</td>\n",
              "      <td>N_217429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.035093</td>\n",
              "      <td>-0.027917</td>\n",
              "      <td>0.017686</td>\n",
              "      <td>0.011974</td>\n",
              "      <td>0.123146</td>\n",
              "      <td>-0.051364</td>\n",
              "      <td>-0.024959</td>\n",
              "      <td>0.176133</td>\n",
              "      <td>-0.043006</td>\n",
              "      <td>-0.027118</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.204552</td>\n",
              "      <td>-0.052534</td>\n",
              "      <td>-0.056887</td>\n",
              "      <td>0.034603</td>\n",
              "      <td>0.107830</td>\n",
              "      <td>-0.013466</td>\n",
              "      <td>0.075048</td>\n",
              "      <td>-0.013248</td>\n",
              "      <td>1</td>\n",
              "      <td>P_7456</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 1282 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e094c353-d69f-4e42-aee0-c67d4d24dc3f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-b7d3c14e-76b6-425b-b530-d9d4e4b075b1\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b7d3c14e-76b6-425b-b530-d9d4e4b075b1')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-b7d3c14e-76b6-425b-b530-d9d4e4b075b1 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e094c353-d69f-4e42-aee0-c67d4d24dc3f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e094c353-d69f-4e42-aee0-c67d4d24dc3f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "protbert_train.head()"
      ],
      "metadata": {
        "id": "Jd_tTehkhs-O",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 280
        },
        "outputId": "8e496588-6c31-49f3-c533-3397846026bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         0         1         2        3        4        5        6        7  \\\n",
              "0  0.02951 -0.038240  0.042880 -0.03029 -0.01486  0.09100 -0.02155 -0.07270   \n",
              "1  0.02860 -0.078300  0.044460 -0.03784  0.01002  0.10065 -0.03247 -0.06560   \n",
              "2 -0.04730 -0.002228 -0.031100 -0.01897  0.02316  0.01805  0.02126 -0.07916   \n",
              "3  0.01968  0.079900  0.001131  0.03240  0.02434 -0.07947 -0.01135 -0.12840   \n",
              "4  0.00896  0.022220  0.062230  0.00716  0.03140  0.03087 -0.04764 -0.00948   \n",
              "\n",
              "         8        9  ...      1016      1017      1018     1019      1020  \\\n",
              "0  0.08460 -0.00852  ...  0.017780 -0.064150  0.036220 -0.07806 -0.014550   \n",
              "1  0.07630 -0.01620  ...  0.009390 -0.072000  0.019970 -0.10583 -0.012480   \n",
              "2 -0.03537 -0.05865  ... -0.019170  0.001675  0.035900  0.00710  0.021940   \n",
              "3 -0.03787  0.02830  ... -0.000449 -0.206700 -0.005222  0.01127  0.012085   \n",
              "4  0.04837  0.02878  ... -0.037500 -0.075500 -0.021120  0.04892 -0.012870   \n",
              "\n",
              "       1021     1022     1023  Label        id  \n",
              "0 -0.009850  0.03534  0.02070      1     P_194  \n",
              "1 -0.009700  0.04852  0.02205      1    P_6357  \n",
              "2 -0.014656  0.03111 -0.01790      0  N_341543  \n",
              "3  0.014510  0.04248 -0.02217      0  N_257764  \n",
              "4  0.023820 -0.04000 -0.06240      0  N_243343  \n",
              "\n",
              "[5 rows x 1026 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-658fa7f0-cfcd-4817-bcda-a244aae3a5f0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>1016</th>\n",
              "      <th>1017</th>\n",
              "      <th>1018</th>\n",
              "      <th>1019</th>\n",
              "      <th>1020</th>\n",
              "      <th>1021</th>\n",
              "      <th>1022</th>\n",
              "      <th>1023</th>\n",
              "      <th>Label</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.02951</td>\n",
              "      <td>-0.038240</td>\n",
              "      <td>0.042880</td>\n",
              "      <td>-0.03029</td>\n",
              "      <td>-0.01486</td>\n",
              "      <td>0.09100</td>\n",
              "      <td>-0.02155</td>\n",
              "      <td>-0.07270</td>\n",
              "      <td>0.08460</td>\n",
              "      <td>-0.00852</td>\n",
              "      <td>...</td>\n",
              "      <td>0.017780</td>\n",
              "      <td>-0.064150</td>\n",
              "      <td>0.036220</td>\n",
              "      <td>-0.07806</td>\n",
              "      <td>-0.014550</td>\n",
              "      <td>-0.009850</td>\n",
              "      <td>0.03534</td>\n",
              "      <td>0.02070</td>\n",
              "      <td>1</td>\n",
              "      <td>P_194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.02860</td>\n",
              "      <td>-0.078300</td>\n",
              "      <td>0.044460</td>\n",
              "      <td>-0.03784</td>\n",
              "      <td>0.01002</td>\n",
              "      <td>0.10065</td>\n",
              "      <td>-0.03247</td>\n",
              "      <td>-0.06560</td>\n",
              "      <td>0.07630</td>\n",
              "      <td>-0.01620</td>\n",
              "      <td>...</td>\n",
              "      <td>0.009390</td>\n",
              "      <td>-0.072000</td>\n",
              "      <td>0.019970</td>\n",
              "      <td>-0.10583</td>\n",
              "      <td>-0.012480</td>\n",
              "      <td>-0.009700</td>\n",
              "      <td>0.04852</td>\n",
              "      <td>0.02205</td>\n",
              "      <td>1</td>\n",
              "      <td>P_6357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.04730</td>\n",
              "      <td>-0.002228</td>\n",
              "      <td>-0.031100</td>\n",
              "      <td>-0.01897</td>\n",
              "      <td>0.02316</td>\n",
              "      <td>0.01805</td>\n",
              "      <td>0.02126</td>\n",
              "      <td>-0.07916</td>\n",
              "      <td>-0.03537</td>\n",
              "      <td>-0.05865</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.019170</td>\n",
              "      <td>0.001675</td>\n",
              "      <td>0.035900</td>\n",
              "      <td>0.00710</td>\n",
              "      <td>0.021940</td>\n",
              "      <td>-0.014656</td>\n",
              "      <td>0.03111</td>\n",
              "      <td>-0.01790</td>\n",
              "      <td>0</td>\n",
              "      <td>N_341543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.01968</td>\n",
              "      <td>0.079900</td>\n",
              "      <td>0.001131</td>\n",
              "      <td>0.03240</td>\n",
              "      <td>0.02434</td>\n",
              "      <td>-0.07947</td>\n",
              "      <td>-0.01135</td>\n",
              "      <td>-0.12840</td>\n",
              "      <td>-0.03787</td>\n",
              "      <td>0.02830</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.000449</td>\n",
              "      <td>-0.206700</td>\n",
              "      <td>-0.005222</td>\n",
              "      <td>0.01127</td>\n",
              "      <td>0.012085</td>\n",
              "      <td>0.014510</td>\n",
              "      <td>0.04248</td>\n",
              "      <td>-0.02217</td>\n",
              "      <td>0</td>\n",
              "      <td>N_257764</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.00896</td>\n",
              "      <td>0.022220</td>\n",
              "      <td>0.062230</td>\n",
              "      <td>0.00716</td>\n",
              "      <td>0.03140</td>\n",
              "      <td>0.03087</td>\n",
              "      <td>-0.04764</td>\n",
              "      <td>-0.00948</td>\n",
              "      <td>0.04837</td>\n",
              "      <td>0.02878</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.037500</td>\n",
              "      <td>-0.075500</td>\n",
              "      <td>-0.021120</td>\n",
              "      <td>0.04892</td>\n",
              "      <td>-0.012870</td>\n",
              "      <td>0.023820</td>\n",
              "      <td>-0.04000</td>\n",
              "      <td>-0.06240</td>\n",
              "      <td>0</td>\n",
              "      <td>N_243343</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 1026 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-658fa7f0-cfcd-4817-bcda-a244aae3a5f0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-7a9938c0-843f-476f-9b62-267a816ebb41\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7a9938c0-843f-476f-9b62-267a816ebb41')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-7a9938c0-843f-476f-9b62-267a816ebb41 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-658fa7f0-cfcd-4817-bcda-a244aae3a5f0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-658fa7f0-cfcd-4817-bcda-a244aae3a5f0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "protbert_test.head()"
      ],
      "metadata": {
        "id": "OLpmatDahzk6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 280
        },
        "outputId": "36c5b36b-3042-4377-eda0-6f15f8c7225e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         0        1        2        3         4         5        6        7  \\\n",
              "0 -0.01852  0.08295  0.03818 -0.01380 -0.015580 -0.020190  0.04840 -0.03552   \n",
              "1 -0.02400  0.00681  0.03004 -0.01816 -0.046100 -0.064500 -0.03568 -0.14220   \n",
              "2 -0.01677 -0.04846  0.03910  0.00860 -0.052100 -0.074800 -0.00942 -0.06480   \n",
              "3  0.01593 -0.05563  0.02385 -0.04205 -0.000392 -0.002066 -0.01698 -0.18870   \n",
              "4 -0.00402 -0.08496  0.06910  0.04944  0.025000 -0.015114 -0.06665 -0.06140   \n",
              "\n",
              "          8         9  ...      1016     1017      1018     1019     1020  \\\n",
              "0 -0.000656 -0.060150  ... -0.028060  0.01277  0.047550 -0.03232 -0.02292   \n",
              "1  0.004387 -0.063600  ... -0.007454 -0.04565  0.000140 -0.07220 -0.03534   \n",
              "2 -0.004760 -0.021060  ...  0.003250 -0.04965 -0.008705 -0.03333 -0.04065   \n",
              "3  0.046720  0.049530  ...  0.042270 -0.12450  0.048220  0.06020  0.05258   \n",
              "4 -0.004950  0.002924  ... -0.007980 -0.02780  0.069300  0.03119 -0.00836   \n",
              "\n",
              "       1021     1022      1023  Label        id  \n",
              "0  0.052200  0.02275 -0.023390      1    P_2249  \n",
              "1  0.021550  0.07184  0.006330      1    P_8311  \n",
              "2 -0.021400  0.04794  0.054840      1    P_9307  \n",
              "3  0.003897 -0.00058 -0.027620      0  N_238995  \n",
              "4 -0.053560  0.05910  0.003437      0  N_255457  \n",
              "\n",
              "[5 rows x 1026 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-9912150d-6c06-4267-9856-74cd07b1831c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>1016</th>\n",
              "      <th>1017</th>\n",
              "      <th>1018</th>\n",
              "      <th>1019</th>\n",
              "      <th>1020</th>\n",
              "      <th>1021</th>\n",
              "      <th>1022</th>\n",
              "      <th>1023</th>\n",
              "      <th>Label</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.01852</td>\n",
              "      <td>0.08295</td>\n",
              "      <td>0.03818</td>\n",
              "      <td>-0.01380</td>\n",
              "      <td>-0.015580</td>\n",
              "      <td>-0.020190</td>\n",
              "      <td>0.04840</td>\n",
              "      <td>-0.03552</td>\n",
              "      <td>-0.000656</td>\n",
              "      <td>-0.060150</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.028060</td>\n",
              "      <td>0.01277</td>\n",
              "      <td>0.047550</td>\n",
              "      <td>-0.03232</td>\n",
              "      <td>-0.02292</td>\n",
              "      <td>0.052200</td>\n",
              "      <td>0.02275</td>\n",
              "      <td>-0.023390</td>\n",
              "      <td>1</td>\n",
              "      <td>P_2249</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.02400</td>\n",
              "      <td>0.00681</td>\n",
              "      <td>0.03004</td>\n",
              "      <td>-0.01816</td>\n",
              "      <td>-0.046100</td>\n",
              "      <td>-0.064500</td>\n",
              "      <td>-0.03568</td>\n",
              "      <td>-0.14220</td>\n",
              "      <td>0.004387</td>\n",
              "      <td>-0.063600</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.007454</td>\n",
              "      <td>-0.04565</td>\n",
              "      <td>0.000140</td>\n",
              "      <td>-0.07220</td>\n",
              "      <td>-0.03534</td>\n",
              "      <td>0.021550</td>\n",
              "      <td>0.07184</td>\n",
              "      <td>0.006330</td>\n",
              "      <td>1</td>\n",
              "      <td>P_8311</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.01677</td>\n",
              "      <td>-0.04846</td>\n",
              "      <td>0.03910</td>\n",
              "      <td>0.00860</td>\n",
              "      <td>-0.052100</td>\n",
              "      <td>-0.074800</td>\n",
              "      <td>-0.00942</td>\n",
              "      <td>-0.06480</td>\n",
              "      <td>-0.004760</td>\n",
              "      <td>-0.021060</td>\n",
              "      <td>...</td>\n",
              "      <td>0.003250</td>\n",
              "      <td>-0.04965</td>\n",
              "      <td>-0.008705</td>\n",
              "      <td>-0.03333</td>\n",
              "      <td>-0.04065</td>\n",
              "      <td>-0.021400</td>\n",
              "      <td>0.04794</td>\n",
              "      <td>0.054840</td>\n",
              "      <td>1</td>\n",
              "      <td>P_9307</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.01593</td>\n",
              "      <td>-0.05563</td>\n",
              "      <td>0.02385</td>\n",
              "      <td>-0.04205</td>\n",
              "      <td>-0.000392</td>\n",
              "      <td>-0.002066</td>\n",
              "      <td>-0.01698</td>\n",
              "      <td>-0.18870</td>\n",
              "      <td>0.046720</td>\n",
              "      <td>0.049530</td>\n",
              "      <td>...</td>\n",
              "      <td>0.042270</td>\n",
              "      <td>-0.12450</td>\n",
              "      <td>0.048220</td>\n",
              "      <td>0.06020</td>\n",
              "      <td>0.05258</td>\n",
              "      <td>0.003897</td>\n",
              "      <td>-0.00058</td>\n",
              "      <td>-0.027620</td>\n",
              "      <td>0</td>\n",
              "      <td>N_238995</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-0.00402</td>\n",
              "      <td>-0.08496</td>\n",
              "      <td>0.06910</td>\n",
              "      <td>0.04944</td>\n",
              "      <td>0.025000</td>\n",
              "      <td>-0.015114</td>\n",
              "      <td>-0.06665</td>\n",
              "      <td>-0.06140</td>\n",
              "      <td>-0.004950</td>\n",
              "      <td>0.002924</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.007980</td>\n",
              "      <td>-0.02780</td>\n",
              "      <td>0.069300</td>\n",
              "      <td>0.03119</td>\n",
              "      <td>-0.00836</td>\n",
              "      <td>-0.053560</td>\n",
              "      <td>0.05910</td>\n",
              "      <td>0.003437</td>\n",
              "      <td>0</td>\n",
              "      <td>N_255457</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 1026 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9912150d-6c06-4267-9856-74cd07b1831c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-be67fb01-d05a-4a48-a709-9078051accec\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-be67fb01-d05a-4a48-a709-9078051accec')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-be67fb01-d05a-4a48-a709-9078051accec button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9912150d-6c06-4267-9856-74cd07b1831c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9912150d-6c06-4267-9856-74cd07b1831c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Concatenate training DataFrames\n",
        "train = pd.merge(\n",
        "    left=esm_train,\n",
        "    right=protbert_train,\n",
        "    on='id'\n",
        ")\n",
        "train.head()"
      ],
      "metadata": {
        "id": "YObK6izLh4DV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "cellView": "form",
        "outputId": "64bcc030-060c-4daf-8604-8feeca8ec9b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        0_x       1_x       2_x       3_x       4_x       5_x       6_x  \\\n",
              "0 -0.003870 -0.033864 -0.047640  0.042528  0.008789 -0.115301  0.116311   \n",
              "1 -0.027616 -0.047448 -0.092680 -0.037921 -0.064493 -0.085917  0.032686   \n",
              "2 -0.060472 -0.009816 -0.085757 -0.042129 -0.005332 -0.114610 -0.023706   \n",
              "3 -0.001879 -0.132213  0.030315  0.017521  0.093525 -0.132252  0.067915   \n",
              "4 -0.004742 -0.012900 -0.069494  0.027048 -0.110435 -0.009641  0.055032   \n",
              "\n",
              "        7_x       8_x       9_x  ...    1015_y   1016_y   1017_y   1018_y  \\\n",
              "0 -0.056773 -0.034452  0.007469  ... -0.058780 -0.01512  0.01888  0.02798   \n",
              "1 -0.104749  0.044072  0.136332  ...  0.034600  0.01502 -0.07150  0.05087   \n",
              "2 -0.015580  0.004648  0.080403  ... -0.010414  0.01288 -0.05704  0.01997   \n",
              "3  0.093191  0.031767  0.070513  ... -0.043850 -0.02454 -0.02739  0.04074   \n",
              "4 -0.205043  0.036597  0.031489  ...  0.000988 -0.00882 -0.02419  0.05435   \n",
              "\n",
              "     1019_y   1020_y   1021_y   1022_y    1023_y  Label_y  \n",
              "0  0.005867 -0.03363 -0.04044 -0.02450  0.034730        0  \n",
              "1 -0.082300 -0.01258 -0.01215  0.02357  0.011284        1  \n",
              "2  0.014015 -0.02370  0.06256  0.04718  0.013390        1  \n",
              "3  0.035600 -0.04605 -0.02040  0.04890 -0.019760        0  \n",
              "4 -0.088200 -0.04056 -0.06082  0.02296  0.020160        1  \n",
              "\n",
              "[5 rows x 2307 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-e1df447f-bf9d-4a91-9385-c3b5baa5938a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0_x</th>\n",
              "      <th>1_x</th>\n",
              "      <th>2_x</th>\n",
              "      <th>3_x</th>\n",
              "      <th>4_x</th>\n",
              "      <th>5_x</th>\n",
              "      <th>6_x</th>\n",
              "      <th>7_x</th>\n",
              "      <th>8_x</th>\n",
              "      <th>9_x</th>\n",
              "      <th>...</th>\n",
              "      <th>1015_y</th>\n",
              "      <th>1016_y</th>\n",
              "      <th>1017_y</th>\n",
              "      <th>1018_y</th>\n",
              "      <th>1019_y</th>\n",
              "      <th>1020_y</th>\n",
              "      <th>1021_y</th>\n",
              "      <th>1022_y</th>\n",
              "      <th>1023_y</th>\n",
              "      <th>Label_y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.003870</td>\n",
              "      <td>-0.033864</td>\n",
              "      <td>-0.047640</td>\n",
              "      <td>0.042528</td>\n",
              "      <td>0.008789</td>\n",
              "      <td>-0.115301</td>\n",
              "      <td>0.116311</td>\n",
              "      <td>-0.056773</td>\n",
              "      <td>-0.034452</td>\n",
              "      <td>0.007469</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.058780</td>\n",
              "      <td>-0.01512</td>\n",
              "      <td>0.01888</td>\n",
              "      <td>0.02798</td>\n",
              "      <td>0.005867</td>\n",
              "      <td>-0.03363</td>\n",
              "      <td>-0.04044</td>\n",
              "      <td>-0.02450</td>\n",
              "      <td>0.034730</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.027616</td>\n",
              "      <td>-0.047448</td>\n",
              "      <td>-0.092680</td>\n",
              "      <td>-0.037921</td>\n",
              "      <td>-0.064493</td>\n",
              "      <td>-0.085917</td>\n",
              "      <td>0.032686</td>\n",
              "      <td>-0.104749</td>\n",
              "      <td>0.044072</td>\n",
              "      <td>0.136332</td>\n",
              "      <td>...</td>\n",
              "      <td>0.034600</td>\n",
              "      <td>0.01502</td>\n",
              "      <td>-0.07150</td>\n",
              "      <td>0.05087</td>\n",
              "      <td>-0.082300</td>\n",
              "      <td>-0.01258</td>\n",
              "      <td>-0.01215</td>\n",
              "      <td>0.02357</td>\n",
              "      <td>0.011284</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.060472</td>\n",
              "      <td>-0.009816</td>\n",
              "      <td>-0.085757</td>\n",
              "      <td>-0.042129</td>\n",
              "      <td>-0.005332</td>\n",
              "      <td>-0.114610</td>\n",
              "      <td>-0.023706</td>\n",
              "      <td>-0.015580</td>\n",
              "      <td>0.004648</td>\n",
              "      <td>0.080403</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.010414</td>\n",
              "      <td>0.01288</td>\n",
              "      <td>-0.05704</td>\n",
              "      <td>0.01997</td>\n",
              "      <td>0.014015</td>\n",
              "      <td>-0.02370</td>\n",
              "      <td>0.06256</td>\n",
              "      <td>0.04718</td>\n",
              "      <td>0.013390</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.001879</td>\n",
              "      <td>-0.132213</td>\n",
              "      <td>0.030315</td>\n",
              "      <td>0.017521</td>\n",
              "      <td>0.093525</td>\n",
              "      <td>-0.132252</td>\n",
              "      <td>0.067915</td>\n",
              "      <td>0.093191</td>\n",
              "      <td>0.031767</td>\n",
              "      <td>0.070513</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.043850</td>\n",
              "      <td>-0.02454</td>\n",
              "      <td>-0.02739</td>\n",
              "      <td>0.04074</td>\n",
              "      <td>0.035600</td>\n",
              "      <td>-0.04605</td>\n",
              "      <td>-0.02040</td>\n",
              "      <td>0.04890</td>\n",
              "      <td>-0.019760</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-0.004742</td>\n",
              "      <td>-0.012900</td>\n",
              "      <td>-0.069494</td>\n",
              "      <td>0.027048</td>\n",
              "      <td>-0.110435</td>\n",
              "      <td>-0.009641</td>\n",
              "      <td>0.055032</td>\n",
              "      <td>-0.205043</td>\n",
              "      <td>0.036597</td>\n",
              "      <td>0.031489</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000988</td>\n",
              "      <td>-0.00882</td>\n",
              "      <td>-0.02419</td>\n",
              "      <td>0.05435</td>\n",
              "      <td>-0.088200</td>\n",
              "      <td>-0.04056</td>\n",
              "      <td>-0.06082</td>\n",
              "      <td>0.02296</td>\n",
              "      <td>0.020160</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 2307 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e1df447f-bf9d-4a91-9385-c3b5baa5938a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-a06d47d7-7eeb-4496-923a-df71d62818fc\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a06d47d7-7eeb-4496-923a-df71d62818fc')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-a06d47d7-7eeb-4496-923a-df71d62818fc button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e1df447f-bf9d-4a91-9385-c3b5baa5938a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e1df447f-bf9d-4a91-9385-c3b5baa5938a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train['Label'] = train['Label_x']\n",
        "train = train.drop(columns=['Label_x', 'Label_y', 'id'])\n",
        "train.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "pTXzxDfCcwZ5",
        "outputId": "a99a6826-35ec-4123-bd18-7d3c0fb6f864"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        0_x       1_x       2_x       3_x       4_x       5_x       6_x  \\\n",
              "0 -0.003870 -0.033864 -0.047640  0.042528  0.008789 -0.115301  0.116311   \n",
              "1 -0.027616 -0.047448 -0.092680 -0.037921 -0.064493 -0.085917  0.032686   \n",
              "2 -0.060472 -0.009816 -0.085757 -0.042129 -0.005332 -0.114610 -0.023706   \n",
              "3 -0.001879 -0.132213  0.030315  0.017521  0.093525 -0.132252  0.067915   \n",
              "4 -0.004742 -0.012900 -0.069494  0.027048 -0.110435 -0.009641  0.055032   \n",
              "\n",
              "        7_x       8_x       9_x  ...    1015_y   1016_y   1017_y   1018_y  \\\n",
              "0 -0.056773 -0.034452  0.007469  ... -0.058780 -0.01512  0.01888  0.02798   \n",
              "1 -0.104749  0.044072  0.136332  ...  0.034600  0.01502 -0.07150  0.05087   \n",
              "2 -0.015580  0.004648  0.080403  ... -0.010414  0.01288 -0.05704  0.01997   \n",
              "3  0.093191  0.031767  0.070513  ... -0.043850 -0.02454 -0.02739  0.04074   \n",
              "4 -0.205043  0.036597  0.031489  ...  0.000988 -0.00882 -0.02419  0.05435   \n",
              "\n",
              "     1019_y   1020_y   1021_y   1022_y    1023_y  Label  \n",
              "0  0.005867 -0.03363 -0.04044 -0.02450  0.034730      0  \n",
              "1 -0.082300 -0.01258 -0.01215  0.02357  0.011284      1  \n",
              "2  0.014015 -0.02370  0.06256  0.04718  0.013390      1  \n",
              "3  0.035600 -0.04605 -0.02040  0.04890 -0.019760      0  \n",
              "4 -0.088200 -0.04056 -0.06082  0.02296  0.020160      1  \n",
              "\n",
              "[5 rows x 2305 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-f4f91ec7-f056-442a-aea7-cded7346d671\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0_x</th>\n",
              "      <th>1_x</th>\n",
              "      <th>2_x</th>\n",
              "      <th>3_x</th>\n",
              "      <th>4_x</th>\n",
              "      <th>5_x</th>\n",
              "      <th>6_x</th>\n",
              "      <th>7_x</th>\n",
              "      <th>8_x</th>\n",
              "      <th>9_x</th>\n",
              "      <th>...</th>\n",
              "      <th>1015_y</th>\n",
              "      <th>1016_y</th>\n",
              "      <th>1017_y</th>\n",
              "      <th>1018_y</th>\n",
              "      <th>1019_y</th>\n",
              "      <th>1020_y</th>\n",
              "      <th>1021_y</th>\n",
              "      <th>1022_y</th>\n",
              "      <th>1023_y</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.003870</td>\n",
              "      <td>-0.033864</td>\n",
              "      <td>-0.047640</td>\n",
              "      <td>0.042528</td>\n",
              "      <td>0.008789</td>\n",
              "      <td>-0.115301</td>\n",
              "      <td>0.116311</td>\n",
              "      <td>-0.056773</td>\n",
              "      <td>-0.034452</td>\n",
              "      <td>0.007469</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.058780</td>\n",
              "      <td>-0.01512</td>\n",
              "      <td>0.01888</td>\n",
              "      <td>0.02798</td>\n",
              "      <td>0.005867</td>\n",
              "      <td>-0.03363</td>\n",
              "      <td>-0.04044</td>\n",
              "      <td>-0.02450</td>\n",
              "      <td>0.034730</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.027616</td>\n",
              "      <td>-0.047448</td>\n",
              "      <td>-0.092680</td>\n",
              "      <td>-0.037921</td>\n",
              "      <td>-0.064493</td>\n",
              "      <td>-0.085917</td>\n",
              "      <td>0.032686</td>\n",
              "      <td>-0.104749</td>\n",
              "      <td>0.044072</td>\n",
              "      <td>0.136332</td>\n",
              "      <td>...</td>\n",
              "      <td>0.034600</td>\n",
              "      <td>0.01502</td>\n",
              "      <td>-0.07150</td>\n",
              "      <td>0.05087</td>\n",
              "      <td>-0.082300</td>\n",
              "      <td>-0.01258</td>\n",
              "      <td>-0.01215</td>\n",
              "      <td>0.02357</td>\n",
              "      <td>0.011284</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.060472</td>\n",
              "      <td>-0.009816</td>\n",
              "      <td>-0.085757</td>\n",
              "      <td>-0.042129</td>\n",
              "      <td>-0.005332</td>\n",
              "      <td>-0.114610</td>\n",
              "      <td>-0.023706</td>\n",
              "      <td>-0.015580</td>\n",
              "      <td>0.004648</td>\n",
              "      <td>0.080403</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.010414</td>\n",
              "      <td>0.01288</td>\n",
              "      <td>-0.05704</td>\n",
              "      <td>0.01997</td>\n",
              "      <td>0.014015</td>\n",
              "      <td>-0.02370</td>\n",
              "      <td>0.06256</td>\n",
              "      <td>0.04718</td>\n",
              "      <td>0.013390</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.001879</td>\n",
              "      <td>-0.132213</td>\n",
              "      <td>0.030315</td>\n",
              "      <td>0.017521</td>\n",
              "      <td>0.093525</td>\n",
              "      <td>-0.132252</td>\n",
              "      <td>0.067915</td>\n",
              "      <td>0.093191</td>\n",
              "      <td>0.031767</td>\n",
              "      <td>0.070513</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.043850</td>\n",
              "      <td>-0.02454</td>\n",
              "      <td>-0.02739</td>\n",
              "      <td>0.04074</td>\n",
              "      <td>0.035600</td>\n",
              "      <td>-0.04605</td>\n",
              "      <td>-0.02040</td>\n",
              "      <td>0.04890</td>\n",
              "      <td>-0.019760</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-0.004742</td>\n",
              "      <td>-0.012900</td>\n",
              "      <td>-0.069494</td>\n",
              "      <td>0.027048</td>\n",
              "      <td>-0.110435</td>\n",
              "      <td>-0.009641</td>\n",
              "      <td>0.055032</td>\n",
              "      <td>-0.205043</td>\n",
              "      <td>0.036597</td>\n",
              "      <td>0.031489</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000988</td>\n",
              "      <td>-0.00882</td>\n",
              "      <td>-0.02419</td>\n",
              "      <td>0.05435</td>\n",
              "      <td>-0.088200</td>\n",
              "      <td>-0.04056</td>\n",
              "      <td>-0.06082</td>\n",
              "      <td>0.02296</td>\n",
              "      <td>0.020160</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 2305 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f4f91ec7-f056-442a-aea7-cded7346d671')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-5dfee7c8-4717-47e3-852a-653109753f5e\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5dfee7c8-4717-47e3-852a-653109753f5e')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-5dfee7c8-4717-47e3-852a-653109753f5e button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f4f91ec7-f056-442a-aea7-cded7346d671 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f4f91ec7-f056-442a-aea7-cded7346d671');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Concatenate test DataFrames\n",
        "test = pd.merge(\n",
        "    left=esm_test,\n",
        "    right=protbert_test,\n",
        "    on='id'\n",
        ")\n",
        "test.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "cellView": "form",
        "id": "ja3M81XaZubm",
        "outputId": "36a2d088-d703-4b06-b3e2-5253ea27b87a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        0_x       1_x       2_x       3_x       4_x       5_x       6_x  \\\n",
              "0 -0.013756 -0.056206 -0.010463  0.078954 -0.013414 -0.036181  0.101913   \n",
              "1  0.138220 -0.015468 -0.014704 -0.045518 -0.017561  0.011617 -0.093774   \n",
              "2  0.008007 -0.054583 -0.012078  0.037711 -0.046313 -0.094285  0.004338   \n",
              "3 -0.074227 -0.024113 -0.001488 -0.026625 -0.056001  0.005185 -0.002422   \n",
              "4  0.035093 -0.027917  0.017686  0.011974  0.123146 -0.051364 -0.024959   \n",
              "\n",
              "        7_x       8_x       9_x  ...    1015_y    1016_y   1017_y   1018_y  \\\n",
              "0 -0.078302 -0.029859  0.155479  ... -0.002998  0.026950 -0.05620  0.06690   \n",
              "1 -0.057526 -0.016540  0.202359  ...  0.035600 -0.014534 -0.08590  0.03397   \n",
              "2 -0.064758 -0.096635  0.059523  ... -0.038570 -0.009390 -0.03925  0.04712   \n",
              "3 -0.120850 -0.008313 -0.003280  ... -0.049500 -0.029420 -0.08580 -0.02002   \n",
              "4  0.176133 -0.043006 -0.027118  ...  0.029170 -0.052730 -0.06920  0.03854   \n",
              "\n",
              "    1019_y    1020_y   1021_y   1022_y    1023_y  Label_y  \n",
              "0  0.02548 -0.006428  0.04312  0.06300  0.008500        0  \n",
              "1  0.03726  0.012790 -0.02563  0.09100  0.027910        0  \n",
              "2  0.03078  0.004520 -0.02391  0.06647 -0.003840        1  \n",
              "3  0.03270 -0.029040 -0.06335  0.01161 -0.006042        0  \n",
              "4  0.11456 -0.035220 -0.01343  0.10626  0.055600        1  \n",
              "\n",
              "[5 rows x 2307 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-92ff39a1-f904-44a1-9d52-97055ef251ff\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0_x</th>\n",
              "      <th>1_x</th>\n",
              "      <th>2_x</th>\n",
              "      <th>3_x</th>\n",
              "      <th>4_x</th>\n",
              "      <th>5_x</th>\n",
              "      <th>6_x</th>\n",
              "      <th>7_x</th>\n",
              "      <th>8_x</th>\n",
              "      <th>9_x</th>\n",
              "      <th>...</th>\n",
              "      <th>1015_y</th>\n",
              "      <th>1016_y</th>\n",
              "      <th>1017_y</th>\n",
              "      <th>1018_y</th>\n",
              "      <th>1019_y</th>\n",
              "      <th>1020_y</th>\n",
              "      <th>1021_y</th>\n",
              "      <th>1022_y</th>\n",
              "      <th>1023_y</th>\n",
              "      <th>Label_y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.013756</td>\n",
              "      <td>-0.056206</td>\n",
              "      <td>-0.010463</td>\n",
              "      <td>0.078954</td>\n",
              "      <td>-0.013414</td>\n",
              "      <td>-0.036181</td>\n",
              "      <td>0.101913</td>\n",
              "      <td>-0.078302</td>\n",
              "      <td>-0.029859</td>\n",
              "      <td>0.155479</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.002998</td>\n",
              "      <td>0.026950</td>\n",
              "      <td>-0.05620</td>\n",
              "      <td>0.06690</td>\n",
              "      <td>0.02548</td>\n",
              "      <td>-0.006428</td>\n",
              "      <td>0.04312</td>\n",
              "      <td>0.06300</td>\n",
              "      <td>0.008500</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.138220</td>\n",
              "      <td>-0.015468</td>\n",
              "      <td>-0.014704</td>\n",
              "      <td>-0.045518</td>\n",
              "      <td>-0.017561</td>\n",
              "      <td>0.011617</td>\n",
              "      <td>-0.093774</td>\n",
              "      <td>-0.057526</td>\n",
              "      <td>-0.016540</td>\n",
              "      <td>0.202359</td>\n",
              "      <td>...</td>\n",
              "      <td>0.035600</td>\n",
              "      <td>-0.014534</td>\n",
              "      <td>-0.08590</td>\n",
              "      <td>0.03397</td>\n",
              "      <td>0.03726</td>\n",
              "      <td>0.012790</td>\n",
              "      <td>-0.02563</td>\n",
              "      <td>0.09100</td>\n",
              "      <td>0.027910</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.008007</td>\n",
              "      <td>-0.054583</td>\n",
              "      <td>-0.012078</td>\n",
              "      <td>0.037711</td>\n",
              "      <td>-0.046313</td>\n",
              "      <td>-0.094285</td>\n",
              "      <td>0.004338</td>\n",
              "      <td>-0.064758</td>\n",
              "      <td>-0.096635</td>\n",
              "      <td>0.059523</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.038570</td>\n",
              "      <td>-0.009390</td>\n",
              "      <td>-0.03925</td>\n",
              "      <td>0.04712</td>\n",
              "      <td>0.03078</td>\n",
              "      <td>0.004520</td>\n",
              "      <td>-0.02391</td>\n",
              "      <td>0.06647</td>\n",
              "      <td>-0.003840</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.074227</td>\n",
              "      <td>-0.024113</td>\n",
              "      <td>-0.001488</td>\n",
              "      <td>-0.026625</td>\n",
              "      <td>-0.056001</td>\n",
              "      <td>0.005185</td>\n",
              "      <td>-0.002422</td>\n",
              "      <td>-0.120850</td>\n",
              "      <td>-0.008313</td>\n",
              "      <td>-0.003280</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.049500</td>\n",
              "      <td>-0.029420</td>\n",
              "      <td>-0.08580</td>\n",
              "      <td>-0.02002</td>\n",
              "      <td>0.03270</td>\n",
              "      <td>-0.029040</td>\n",
              "      <td>-0.06335</td>\n",
              "      <td>0.01161</td>\n",
              "      <td>-0.006042</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.035093</td>\n",
              "      <td>-0.027917</td>\n",
              "      <td>0.017686</td>\n",
              "      <td>0.011974</td>\n",
              "      <td>0.123146</td>\n",
              "      <td>-0.051364</td>\n",
              "      <td>-0.024959</td>\n",
              "      <td>0.176133</td>\n",
              "      <td>-0.043006</td>\n",
              "      <td>-0.027118</td>\n",
              "      <td>...</td>\n",
              "      <td>0.029170</td>\n",
              "      <td>-0.052730</td>\n",
              "      <td>-0.06920</td>\n",
              "      <td>0.03854</td>\n",
              "      <td>0.11456</td>\n",
              "      <td>-0.035220</td>\n",
              "      <td>-0.01343</td>\n",
              "      <td>0.10626</td>\n",
              "      <td>0.055600</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 2307 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-92ff39a1-f904-44a1-9d52-97055ef251ff')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-1226ddbc-830f-4768-95aa-922a389bf507\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1226ddbc-830f-4768-95aa-922a389bf507')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-1226ddbc-830f-4768-95aa-922a389bf507 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-92ff39a1-f904-44a1-9d52-97055ef251ff button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-92ff39a1-f904-44a1-9d52-97055ef251ff');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test['Label'] = test['Label_x']\n",
        "test = test.drop(columns=['Label_x', 'Label_y', 'id'])\n",
        "test.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "N5u0kkibdTO2",
        "outputId": "b41b2f4e-dbc5-4c1d-d060-96db9409095e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        0_x       1_x       2_x       3_x       4_x       5_x       6_x  \\\n",
              "0 -0.013756 -0.056206 -0.010463  0.078954 -0.013414 -0.036181  0.101913   \n",
              "1  0.138220 -0.015468 -0.014704 -0.045518 -0.017561  0.011617 -0.093774   \n",
              "2  0.008007 -0.054583 -0.012078  0.037711 -0.046313 -0.094285  0.004338   \n",
              "3 -0.074227 -0.024113 -0.001488 -0.026625 -0.056001  0.005185 -0.002422   \n",
              "4  0.035093 -0.027917  0.017686  0.011974  0.123146 -0.051364 -0.024959   \n",
              "\n",
              "        7_x       8_x       9_x  ...    1015_y    1016_y   1017_y   1018_y  \\\n",
              "0 -0.078302 -0.029859  0.155479  ... -0.002998  0.026950 -0.05620  0.06690   \n",
              "1 -0.057526 -0.016540  0.202359  ...  0.035600 -0.014534 -0.08590  0.03397   \n",
              "2 -0.064758 -0.096635  0.059523  ... -0.038570 -0.009390 -0.03925  0.04712   \n",
              "3 -0.120850 -0.008313 -0.003280  ... -0.049500 -0.029420 -0.08580 -0.02002   \n",
              "4  0.176133 -0.043006 -0.027118  ...  0.029170 -0.052730 -0.06920  0.03854   \n",
              "\n",
              "    1019_y    1020_y   1021_y   1022_y    1023_y  Label  \n",
              "0  0.02548 -0.006428  0.04312  0.06300  0.008500      0  \n",
              "1  0.03726  0.012790 -0.02563  0.09100  0.027910      0  \n",
              "2  0.03078  0.004520 -0.02391  0.06647 -0.003840      1  \n",
              "3  0.03270 -0.029040 -0.06335  0.01161 -0.006042      0  \n",
              "4  0.11456 -0.035220 -0.01343  0.10626  0.055600      1  \n",
              "\n",
              "[5 rows x 2305 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-3537551e-0e3a-41bf-b6e4-d0ef8f95415d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0_x</th>\n",
              "      <th>1_x</th>\n",
              "      <th>2_x</th>\n",
              "      <th>3_x</th>\n",
              "      <th>4_x</th>\n",
              "      <th>5_x</th>\n",
              "      <th>6_x</th>\n",
              "      <th>7_x</th>\n",
              "      <th>8_x</th>\n",
              "      <th>9_x</th>\n",
              "      <th>...</th>\n",
              "      <th>1015_y</th>\n",
              "      <th>1016_y</th>\n",
              "      <th>1017_y</th>\n",
              "      <th>1018_y</th>\n",
              "      <th>1019_y</th>\n",
              "      <th>1020_y</th>\n",
              "      <th>1021_y</th>\n",
              "      <th>1022_y</th>\n",
              "      <th>1023_y</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.013756</td>\n",
              "      <td>-0.056206</td>\n",
              "      <td>-0.010463</td>\n",
              "      <td>0.078954</td>\n",
              "      <td>-0.013414</td>\n",
              "      <td>-0.036181</td>\n",
              "      <td>0.101913</td>\n",
              "      <td>-0.078302</td>\n",
              "      <td>-0.029859</td>\n",
              "      <td>0.155479</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.002998</td>\n",
              "      <td>0.026950</td>\n",
              "      <td>-0.05620</td>\n",
              "      <td>0.06690</td>\n",
              "      <td>0.02548</td>\n",
              "      <td>-0.006428</td>\n",
              "      <td>0.04312</td>\n",
              "      <td>0.06300</td>\n",
              "      <td>0.008500</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.138220</td>\n",
              "      <td>-0.015468</td>\n",
              "      <td>-0.014704</td>\n",
              "      <td>-0.045518</td>\n",
              "      <td>-0.017561</td>\n",
              "      <td>0.011617</td>\n",
              "      <td>-0.093774</td>\n",
              "      <td>-0.057526</td>\n",
              "      <td>-0.016540</td>\n",
              "      <td>0.202359</td>\n",
              "      <td>...</td>\n",
              "      <td>0.035600</td>\n",
              "      <td>-0.014534</td>\n",
              "      <td>-0.08590</td>\n",
              "      <td>0.03397</td>\n",
              "      <td>0.03726</td>\n",
              "      <td>0.012790</td>\n",
              "      <td>-0.02563</td>\n",
              "      <td>0.09100</td>\n",
              "      <td>0.027910</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.008007</td>\n",
              "      <td>-0.054583</td>\n",
              "      <td>-0.012078</td>\n",
              "      <td>0.037711</td>\n",
              "      <td>-0.046313</td>\n",
              "      <td>-0.094285</td>\n",
              "      <td>0.004338</td>\n",
              "      <td>-0.064758</td>\n",
              "      <td>-0.096635</td>\n",
              "      <td>0.059523</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.038570</td>\n",
              "      <td>-0.009390</td>\n",
              "      <td>-0.03925</td>\n",
              "      <td>0.04712</td>\n",
              "      <td>0.03078</td>\n",
              "      <td>0.004520</td>\n",
              "      <td>-0.02391</td>\n",
              "      <td>0.06647</td>\n",
              "      <td>-0.003840</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.074227</td>\n",
              "      <td>-0.024113</td>\n",
              "      <td>-0.001488</td>\n",
              "      <td>-0.026625</td>\n",
              "      <td>-0.056001</td>\n",
              "      <td>0.005185</td>\n",
              "      <td>-0.002422</td>\n",
              "      <td>-0.120850</td>\n",
              "      <td>-0.008313</td>\n",
              "      <td>-0.003280</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.049500</td>\n",
              "      <td>-0.029420</td>\n",
              "      <td>-0.08580</td>\n",
              "      <td>-0.02002</td>\n",
              "      <td>0.03270</td>\n",
              "      <td>-0.029040</td>\n",
              "      <td>-0.06335</td>\n",
              "      <td>0.01161</td>\n",
              "      <td>-0.006042</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.035093</td>\n",
              "      <td>-0.027917</td>\n",
              "      <td>0.017686</td>\n",
              "      <td>0.011974</td>\n",
              "      <td>0.123146</td>\n",
              "      <td>-0.051364</td>\n",
              "      <td>-0.024959</td>\n",
              "      <td>0.176133</td>\n",
              "      <td>-0.043006</td>\n",
              "      <td>-0.027118</td>\n",
              "      <td>...</td>\n",
              "      <td>0.029170</td>\n",
              "      <td>-0.052730</td>\n",
              "      <td>-0.06920</td>\n",
              "      <td>0.03854</td>\n",
              "      <td>0.11456</td>\n",
              "      <td>-0.035220</td>\n",
              "      <td>-0.01343</td>\n",
              "      <td>0.10626</td>\n",
              "      <td>0.055600</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 2305 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3537551e-0e3a-41bf-b6e4-d0ef8f95415d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-3bd8ce9b-dbec-4697-8932-3662109913b1\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3bd8ce9b-dbec-4697-8932-3662109913b1')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-3bd8ce9b-dbec-4697-8932-3662109913b1 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3537551e-0e3a-41bf-b6e4-d0ef8f95415d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3537551e-0e3a-41bf-b6e4-d0ef8f95415d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preprocess and prepare the data"
      ],
      "metadata": {
        "id": "mkBzz3oaOy-H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_train = train\n",
        "df_test = test"
      ],
      "metadata": {
        "id": "xO2B_cNfeyRn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert Label to integer\n",
        "df_train['Label'] = df_train['Label'].astype(int)\n",
        "df_test['Label'] = df_test['Label'].astype(int)\n",
        "\n",
        "# Split data into train and validation sets\n",
        "X_train = df_train.drop('Label', axis=1)\n",
        "y_train = df_train['Label']\n",
        "X_test = df_test.drop('Label', axis=1)\n",
        "y_test = df_test['Label']\n"
      ],
      "metadata": {
        "id": "W8VETsusb2l6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create new feature names\n",
        "feature_names = ['feat_'+str(i) for i in range(X_train.shape[1])]\n",
        "\n",
        "# Rename columns\n",
        "X_train.columns = feature_names\n",
        "X_test.columns = feature_names"
      ],
      "metadata": {
        "id": "QTn_AesRcGX4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Standardize the data\n",
        "scaler = StandardScaler()\n",
        "\n",
        "X_train_scaled = scaler.fit_transform(X_train)\n",
        "X_test_scaled = scaler.transform(X_test)\n",
        "\n",
        "# Convert to PyTorch tensors\n",
        "X_train_scaled = torch.from_numpy(X_train_scaled).float()\n",
        "X_test_scaled = torch.from_numpy(X_test_scaled).float()\n",
        "\n",
        "X_train = X_train_scaled\n",
        "X_test = X_test_scaled"
      ],
      "metadata": {
        "id": "LNdTmGZAIymv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert to numpy arrays\n",
        "y_train = np.array(y_train)\n",
        "y_test = np.array(y_test)\n",
        "\n",
        "# Create Tensors from Labels\n",
        "y_train = torch.from_numpy(y_train).long()\n",
        "y_test = torch.from_numpy(y_test).long()"
      ],
      "metadata": {
        "id": "YtHripjvJcAU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# LSTM"
      ],
      "metadata": {
        "id": "nj3_D7thQ_xR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Define a single-layer LSTM"
      ],
      "metadata": {
        "id": "oX9qsLggOr2G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LSTM(nn.Module):\n",
        "  def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
        "    super().__init__()\n",
        "    self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n",
        "    self.dropout = nn.Dropout(0.2)\n",
        "    self.fc = nn.Linear(hidden_size, num_classes)\n",
        "\n",
        "  def forward(self, x):\n",
        "    out, _ = self.lstm(x)\n",
        "    out = self.dropout(out)\n",
        "    out = out[:, -1]\n",
        "    out = self.fc(out)\n",
        "    return out"
      ],
      "metadata": {
        "id": "hVUVgE_sh2X6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train LSTM Model"
      ],
      "metadata": {
        "id": "xXaU-6fFPUdA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = LSTM(2304, 1024, 1, 2)\n",
        "\n",
        "# Reshape data\n",
        "# X_train = X_train.unsqueeze(1)\n",
        "# X_test = X_test.unsqueeze(1)\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters())"
      ],
      "metadata": {
        "id": "0hU7uu2gh49p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(100):\n",
        "  outputs = model(X_train)\n",
        "  loss = criterion(outputs, y_train)\n",
        "\n",
        "  optimizer.zero_grad()\n",
        "  loss.backward()\n",
        "  optimizer.step()\n",
        "\n",
        "  print(f\"Epoch {epoch+1}, Loss: {loss.item():.4f}\")\n",
        "\n",
        "with torch.no_grad():\n",
        "  outputs = model(X_test)\n",
        "\n",
        "_, y_pred = torch.max(outputs, 1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vgXQ0USIA2Ea",
        "outputId": "0f9d74f8-87d2-4af1-ce91-7014e70f20ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Loss: 0.6948\n",
            "Epoch 2, Loss: 1.5489\n",
            "Epoch 3, Loss: 0.6884\n",
            "Epoch 4, Loss: 0.7463\n",
            "Epoch 5, Loss: 0.6920\n",
            "Epoch 6, Loss: 0.6807\n",
            "Epoch 7, Loss: 0.6814\n",
            "Epoch 8, Loss: 0.6833\n",
            "Epoch 9, Loss: 0.6846\n",
            "Epoch 10, Loss: 0.6853\n",
            "Epoch 11, Loss: 0.6856\n",
            "Epoch 12, Loss: 0.6855\n",
            "Epoch 13, Loss: 0.6852\n",
            "Epoch 14, Loss: 0.6846\n",
            "Epoch 15, Loss: 0.6839\n",
            "Epoch 16, Loss: 0.6830\n",
            "Epoch 17, Loss: 0.6821\n",
            "Epoch 18, Loss: 0.6806\n",
            "Epoch 19, Loss: 0.6791\n",
            "Epoch 20, Loss: 0.6776\n",
            "Epoch 21, Loss: 0.6759\n",
            "Epoch 22, Loss: 0.6738\n",
            "Epoch 23, Loss: 0.6721\n",
            "Epoch 24, Loss: 0.6696\n",
            "Epoch 25, Loss: 0.6671\n",
            "Epoch 26, Loss: 0.6642\n",
            "Epoch 27, Loss: 0.6610\n",
            "Epoch 28, Loss: 0.6572\n",
            "Epoch 29, Loss: 0.6527\n",
            "Epoch 30, Loss: 0.6479\n",
            "Epoch 31, Loss: 0.6426\n",
            "Epoch 32, Loss: 0.6368\n",
            "Epoch 33, Loss: 0.6306\n",
            "Epoch 34, Loss: 0.6239\n",
            "Epoch 35, Loss: 0.6168\n",
            "Epoch 36, Loss: 0.6091\n",
            "Epoch 37, Loss: 0.6008\n",
            "Epoch 38, Loss: 0.5915\n",
            "Epoch 39, Loss: 0.5824\n",
            "Epoch 40, Loss: 0.5729\n",
            "Epoch 41, Loss: 0.5627\n",
            "Epoch 42, Loss: 0.5514\n",
            "Epoch 43, Loss: 0.5400\n",
            "Epoch 44, Loss: 0.5271\n",
            "Epoch 45, Loss: 0.5141\n",
            "Epoch 46, Loss: 0.5013\n",
            "Epoch 47, Loss: 0.4869\n",
            "Epoch 48, Loss: 0.4719\n",
            "Epoch 49, Loss: 0.4562\n",
            "Epoch 50, Loss: 0.4409\n",
            "Epoch 51, Loss: 0.4256\n",
            "Epoch 52, Loss: 0.4104\n",
            "Epoch 53, Loss: 0.3951\n",
            "Epoch 54, Loss: 0.3789\n",
            "Epoch 55, Loss: 0.3646\n",
            "Epoch 56, Loss: 0.3499\n",
            "Epoch 57, Loss: 0.3356\n",
            "Epoch 58, Loss: 0.3219\n",
            "Epoch 59, Loss: 0.3095\n",
            "Epoch 60, Loss: 0.2963\n",
            "Epoch 61, Loss: 0.2853\n",
            "Epoch 62, Loss: 0.2732\n",
            "Epoch 63, Loss: 0.2624\n",
            "Epoch 64, Loss: 0.2525\n",
            "Epoch 65, Loss: 0.2423\n",
            "Epoch 66, Loss: 0.2333\n",
            "Epoch 67, Loss: 0.2256\n",
            "Epoch 68, Loss: 0.2175\n",
            "Epoch 69, Loss: 0.2104\n",
            "Epoch 70, Loss: 0.2039\n",
            "Epoch 71, Loss: 0.1986\n",
            "Epoch 72, Loss: 0.1932\n",
            "Epoch 73, Loss: 0.1870\n",
            "Epoch 74, Loss: 0.1811\n",
            "Epoch 75, Loss: 0.1758\n",
            "Epoch 76, Loss: 0.1731\n",
            "Epoch 77, Loss: 0.1700\n",
            "Epoch 78, Loss: 0.1646\n",
            "Epoch 79, Loss: 0.1621\n",
            "Epoch 80, Loss: 0.1598\n",
            "Epoch 81, Loss: 0.1561\n",
            "Epoch 82, Loss: 0.1533\n",
            "Epoch 83, Loss: 0.1520\n",
            "Epoch 84, Loss: 0.1493\n",
            "Epoch 85, Loss: 0.1467\n",
            "Epoch 86, Loss: 0.1452\n",
            "Epoch 87, Loss: 0.1440\n",
            "Epoch 88, Loss: 0.1410\n",
            "Epoch 89, Loss: 0.1393\n",
            "Epoch 90, Loss: 0.1383\n",
            "Epoch 91, Loss: 0.1367\n",
            "Epoch 92, Loss: 0.1348\n",
            "Epoch 93, Loss: 0.1343\n",
            "Epoch 94, Loss: 0.1328\n",
            "Epoch 95, Loss: 0.1315\n",
            "Epoch 96, Loss: 0.1306\n",
            "Epoch 97, Loss: 0.1300\n",
            "Epoch 98, Loss: 0.1279\n",
            "Epoch 99, Loss: 0.1273\n",
            "Epoch 100, Loss: 0.1258\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation results"
      ],
      "metadata": {
        "id": "EkOZLLo99TIT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy = accuracy_score(y_test, y_pred)\n",
        "precision, recall, fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
        "\n",
        "print(\"Accuracy:\", accuracy)\n",
        "print(\"Precision:\", precision)\n",
        "print(\"Recall:\", recall)\n",
        "print(\"F1 Score:\", fscore)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5MIAieLjA5MY",
        "outputId": "5893da54-f14f-4fd7-d137-b3a7652fa483"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.9332506203473946\n",
            "Precision: 0.9346174485899291\n",
            "Recall: 0.9332506203473946\n",
            "F1 Score: 0.9331980990293953\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Embedding vector type: ProtT5\n",
        "* Embedding vector size: 2304\n",
        "* LSTM layer dimension: 1024\n",
        "* Model: Single-layer LSTM\n",
        "* Num epochs: 100\n",
        "* Optimizer: Adam\n",
        "* Loss function: Cross Entropy Loss\n",
        "* Dropout rate: 0.2"
      ],
      "metadata": {
        "id": "-qPs1ritLiSu"
      }
    }
  ]
}