std-scalar-10mb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scale Features and Build Model\n",
    "### Scales Raw Features\n",
    "\n",
    "Import CSV of Aggregated Darshan Logs <br>\n",
    "Apply Log10 and Percent Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./raws.csv\",lineterminator='\\n',sep = ',' ,error_bad_lines=False)\n",
    "#df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df.columns[0],axis = 1)\n",
    "df = df.drop(df.columns[0],axis = 1)\n",
    "f = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posix_read_time</th>\n",
       "      <th>posix_write_time</th>\n",
       "      <th>posix_meta_time</th>\n",
       "      <th>posix_bytes_read</th>\n",
       "      <th>posix_bytes_read_100</th>\n",
       "      <th>posix_bytes_read_1K</th>\n",
       "      <th>posix_bytes_read_10K</th>\n",
       "      <th>posix_bytes_read_100K</th>\n",
       "      <th>posix_bytes_read_1lM</th>\n",
       "      <th>posix_bytes_read_4M</th>\n",
       "      <th>...</th>\n",
       "      <th>posix_m_align</th>\n",
       "      <th>lustre_number_of_files</th>\n",
       "      <th>lustre_mdts</th>\n",
       "      <th>lustre_osts</th>\n",
       "      <th>lustre_stripe_size</th>\n",
       "      <th>lustre_stripe_offset</th>\n",
       "      <th>lustre_stripe_width</th>\n",
       "      <th>lustre_number_of_osts</th>\n",
       "      <th>jobid</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9042391.0</td>\n",
       "      <td>../results/USER375904U_sss2_id9042391_9-26-414...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>104.611641</td>\n",
       "      <td>10.024055</td>\n",
       "      <td>20.060841</td>\n",
       "      <td>2.390891e+10</td>\n",
       "      <td>147688.0</td>\n",
       "      <td>141853.0</td>\n",
       "      <td>33215.0</td>\n",
       "      <td>261267.0</td>\n",
       "      <td>51049.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>13704.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>9868242.0</td>\n",
       "      <td>../results/USER382862U_SpEC_id9868242_4-29-741...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>124.560730</td>\n",
       "      <td>42.051125</td>\n",
       "      <td>54.839272</td>\n",
       "      <td>5.019637e+10</td>\n",
       "      <td>332059.0</td>\n",
       "      <td>684893.0</td>\n",
       "      <td>130143.0</td>\n",
       "      <td>919688.0</td>\n",
       "      <td>2515.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6704.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>9029683.0</td>\n",
       "      <td>../results/USER264205U_SpEC_id9029683_9-20-580...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>25763.292969</td>\n",
       "      <td>582.297363</td>\n",
       "      <td>24.895737</td>\n",
       "      <td>5.488943e+12</td>\n",
       "      <td>30785.0</td>\n",
       "      <td>767.0</td>\n",
       "      <td>508381.0</td>\n",
       "      <td>2040.0</td>\n",
       "      <td>13683.0</td>\n",
       "      <td>626216.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2320.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54.993103</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>9165166.0</td>\n",
       "      <td>../results/USER341996U_CCTM_v52_Linux2_x86_64i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>154.534821</td>\n",
       "      <td>681.548279</td>\n",
       "      <td>658.484985</td>\n",
       "      <td>2.293203e+10</td>\n",
       "      <td>588029.0</td>\n",
       "      <td>2926599.0</td>\n",
       "      <td>425127.0</td>\n",
       "      <td>336133.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6328.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>8269019.0</td>\n",
       "      <td>../results/USER141481U_SpEC_id8269019_3-3-4542...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875282</th>\n",
       "      <td>138.354477</td>\n",
       "      <td>82.278084</td>\n",
       "      <td>194.485565</td>\n",
       "      <td>5.593977e+10</td>\n",
       "      <td>216146.0</td>\n",
       "      <td>535152.0</td>\n",
       "      <td>99814.0</td>\n",
       "      <td>889010.0</td>\n",
       "      <td>12485.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9920.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>7759072.0</td>\n",
       "      <td>../results/USER336298U_SpEC_id7759072_10-30-62...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875283</th>\n",
       "      <td>54.443073</td>\n",
       "      <td>231.440857</td>\n",
       "      <td>25.271391</td>\n",
       "      <td>1.465277e+09</td>\n",
       "      <td>3099.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180662.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8704.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>359.0</td>\n",
       "      <td>7275819.0</td>\n",
       "      <td>../results/USER331454U_pw.x_id7275819_8-2-4790...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875284</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8305062.0</td>\n",
       "      <td>../results/USER371284U_sss2_id8305062_3-14-211...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875285</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>31428.0</td>\n",
       "      <td>../results/USER303986U_testnodes_opt.x_id31428...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875286</th>\n",
       "      <td>227.063828</td>\n",
       "      <td>191.747269</td>\n",
       "      <td>172.671997</td>\n",
       "      <td>1.077412e+11</td>\n",
       "      <td>775359.0</td>\n",
       "      <td>1545229.0</td>\n",
       "      <td>305757.0</td>\n",
       "      <td>1897233.0</td>\n",
       "      <td>1068.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>15104.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>7525550.0</td>\n",
       "      <td>../results/USER382862U_SpEC_id7525550_9-24-380...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>875287 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        posix_read_time  posix_write_time  posix_meta_time  posix_bytes_read  \\\n",
       "0              0.000000          0.000000         0.000000      0.000000e+00   \n",
       "1            104.611641         10.024055        20.060841      2.390891e+10   \n",
       "2            124.560730         42.051125        54.839272      5.019637e+10   \n",
       "3          25763.292969        582.297363        24.895737      5.488943e+12   \n",
       "4            154.534821        681.548279       658.484985      2.293203e+10   \n",
       "...                 ...               ...              ...               ...   \n",
       "875282       138.354477         82.278084       194.485565      5.593977e+10   \n",
       "875283        54.443073        231.440857        25.271391      1.465277e+09   \n",
       "875284         0.000000          0.000000         0.000000      0.000000e+00   \n",
       "875285         0.000000          0.000000         0.000000      0.000000e+00   \n",
       "875286       227.063828        191.747269       172.671997      1.077412e+11   \n",
       "\n",
       "        posix_bytes_read_100  posix_bytes_read_1K  posix_bytes_read_10K  \\\n",
       "0                        0.0                  0.0                   0.0   \n",
       "1                   147688.0             141853.0               33215.0   \n",
       "2                   332059.0             684893.0              130143.0   \n",
       "3                    30785.0                767.0              508381.0   \n",
       "4                   588029.0            2926599.0              425127.0   \n",
       "...                      ...                  ...                   ...   \n",
       "875282              216146.0             535152.0               99814.0   \n",
       "875283                3099.0                  0.0              180662.0   \n",
       "875284                   0.0                  0.0                   0.0   \n",
       "875285                   0.0                  0.0                   0.0   \n",
       "875286              775359.0            1545229.0              305757.0   \n",
       "\n",
       "        posix_bytes_read_100K  posix_bytes_read_1lM  posix_bytes_read_4M  ...  \\\n",
       "0                         0.0                   0.0                  0.0  ...   \n",
       "1                    261267.0               51049.0                  0.0  ...   \n",
       "2                    919688.0                2515.0                  0.0  ...   \n",
       "3                      2040.0               13683.0             626216.0  ...   \n",
       "4                    336133.0                  98.0                  0.0  ...   \n",
       "...                       ...                   ...                  ...  ...   \n",
       "875282               889010.0               12485.0                  0.0  ...   \n",
       "875283                    0.0                   0.0                  0.0  ...   \n",
       "875284                    0.0                   0.0                  0.0  ...   \n",
       "875285                    0.0                   0.0                  0.0  ...   \n",
       "875286              1897233.0                1068.0                  0.0  ...   \n",
       "\n",
       "        posix_m_align  lustre_number_of_files  lustre_mdts  lustre_osts  \\\n",
       "0                 0.0                     4.0          1.0    36.000000   \n",
       "1             13704.0                   176.0          1.0   360.000000   \n",
       "2              6704.0                   224.0          1.0   360.000000   \n",
       "3              2320.0                   290.0          1.0    54.993103   \n",
       "4              6328.0                   201.0          1.0   360.000000   \n",
       "...               ...                     ...          ...          ...   \n",
       "875282         9920.0                   124.0          1.0   360.000000   \n",
       "875283         8704.0                  1088.0          1.0   360.000000   \n",
       "875284            0.0                     5.0          1.0    36.000000   \n",
       "875285            8.0                     2.0          1.0    36.000000   \n",
       "875286        15104.0                   128.0          1.0   360.000000   \n",
       "\n",
       "        lustre_stripe_size  lustre_stripe_offset  lustre_stripe_width  \\\n",
       "0                1048576.0                   0.0                  4.0   \n",
       "1                1048576.0                   0.0                176.0   \n",
       "2                1048576.0                   0.0                224.0   \n",
       "3                1048576.0                   0.0                290.0   \n",
       "4                1048576.0                   0.0                201.0   \n",
       "...                    ...                   ...                  ...   \n",
       "875282           1048576.0                   0.0                124.0   \n",
       "875283           1048576.0                   0.0               1088.0   \n",
       "875284           1048576.0                   0.0                  5.0   \n",
       "875285           1048576.0                   0.0                  2.0   \n",
       "875286           1048576.0                   0.0                128.0   \n",
       "\n",
       "        lustre_number_of_osts      jobid  \\\n",
       "0                         4.0  9042391.0   \n",
       "1                       147.0  9868242.0   \n",
       "2                       186.0  9029683.0   \n",
       "3                        52.0  9165166.0   \n",
       "4                       158.0  8269019.0   \n",
       "...                       ...        ...   \n",
       "875282                  110.0  7759072.0   \n",
       "875283                  359.0  7275819.0   \n",
       "875284                    5.0  8305062.0   \n",
       "875285                    1.0    31428.0   \n",
       "875286                  118.0  7525550.0   \n",
       "\n",
       "                                                     path  \n",
       "0       ../results/USER375904U_sss2_id9042391_9-26-414...  \n",
       "1       ../results/USER382862U_SpEC_id9868242_4-29-741...  \n",
       "2       ../results/USER264205U_SpEC_id9029683_9-20-580...  \n",
       "3       ../results/USER341996U_CCTM_v52_Linux2_x86_64i...  \n",
       "4       ../results/USER141481U_SpEC_id8269019_3-3-4542...  \n",
       "...                                                   ...  \n",
       "875282  ../results/USER336298U_SpEC_id7759072_10-30-62...  \n",
       "875283  ../results/USER331454U_pw.x_id7275819_8-2-4790...  \n",
       "875284  ../results/USER371284U_sss2_id8305062_3-14-211...  \n",
       "875285  ../results/USER303986U_testnodes_opt.x_id31428...  \n",
       "875286  ../results/USER382862U_SpEC_id7525550_9-24-380...  \n",
       "\n",
       "[875287 rows x 50 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['posix_read_time', 'posix_write_time', 'posix_meta_time',\n",
       "       'posix_bytes_read', 'posix_bytes_read_100', 'posix_bytes_read_1K',\n",
       "       'posix_bytes_read_10K', 'posix_bytes_read_100K', 'posix_bytes_read_1lM',\n",
       "       'posix_bytes_read_4M', 'posix_bytes_read_10M', 'posix_bytes_read_100M',\n",
       "       'posix_bytes_read_1G', 'posix_bytes_read_PLUS', 'posix_bytes_write',\n",
       "       'posix_bytes_write_100', 'posix_bytes_write_1K',\n",
       "       'posix_bytes_write_10K', 'posix_bytes_write_100K',\n",
       "       'posix_bytes_write_1M', 'posix_bytes_write_4M', 'posix_bytes_write_10M',\n",
       "       'posix_bytes_write_100M', 'posix_bytes_write_1G',\n",
       "       'posix_bytes_write_PLUS', 'posix_opens', 'posix_reads', 'posix_writes',\n",
       "       'posix_seeks', 'posix_stats', 'posix_mmaps', 'posix_fsyncs',\n",
       "       'posix_fdsyncs', 'posix_rename_sources', 'posix_rename_targets',\n",
       "       'posix_renamed_from', 'posix_renamed_mode', 'posix_number_of_files',\n",
       "       'nprocs', 'posix_f_align', 'posix_m_align', 'lustre_number_of_files',\n",
       "       'lustre_mdts', 'lustre_osts', 'lustre_stripe_size',\n",
       "       'lustre_stripe_offset', 'lustre_stripe_width', 'lustre_number_of_osts',\n",
       "       'jobid', 'path'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = df.dropna(axis=0, how='any')\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#files\n",
    "f['log10_p_files'] = df['posix_number_of_files'] \n",
    "f['log10_l_files'] = df['lustre_number_of_files']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0               0.0\n",
       "1          880136.0\n",
       "2         2379598.0\n",
       "3         8903411.0\n",
       "4         7846387.0\n",
       "            ...    \n",
       "875282    2234152.0\n",
       "875283     197651.0\n",
       "875284          0.0\n",
       "875285          0.0\n",
       "875286    6065006.0\n",
       "Name: log10_p_accesses, Length: 875287, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accesses\n",
    "df['p_accesses'] = df['posix_reads'] + df['posix_writes']\n",
    "f['log10_p_accesses'] = df['p_accesses']\n",
    "\n",
    "f['log10_p_accesses']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bytes\n",
    "f['log10_p_bytes'] = df['posix_bytes_read'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['log10_p_opens'] = df['posix_opens']\n",
    "f['log10_p_seeks'] = df['posix_seeks']\n",
    "f['log10_p_stats'] = df['posix_stats']\n",
    "f['log10_p_mode'] = df['posix_renamed_mode']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['log10_l_n_osts'] = df['lustre_number_of_osts']\n",
    "f['log10_l_stripe_w'] = df['lustre_stripe_width']\n",
    "f['log10_l_mdts'] = df['lustre_mdts']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['log10_p_nprocs'] = df['nprocs']\n",
    "f['log10_p_falign'] = df['posix_f_align']\n",
    "f['log10_p_malign'] = df['posix_m_align']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['perc_p_reads'] = df['posix_reads'] \n",
    "f['perc_p_writes'] = df['posix_writes'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['perc_p_bytes_read_100'] = df['posix_bytes_read_100']\n",
    "f['perc_p_bytes_read_1K'] = df['posix_bytes_read_1K']\n",
    "f['perc_p_bytes_read_10K'] = df['posix_bytes_read_10K'] \n",
    "f['perc_p_bytes_read_100K'] = df['posix_bytes_read_100K']\n",
    "f['perc_p_bytes_read_1M'] = df['posix_bytes_read_1lM']\n",
    "f['perc_p_bytes_read_4M'] = df['posix_bytes_read_4M']\n",
    "f['perc_p_bytes_read_10M'] = df['posix_bytes_read_10M']\n",
    "f['perc_p_bytes_read_100M'] = df['posix_bytes_read_100M']\n",
    "f['perc_p_bytes_read_1G'] = df['posix_bytes_read_1G']\n",
    "f['perc_p_bytes_read_PLUS'] = df['posix_bytes_read_PLUS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['perc_p_bytes_write_100'] = df['posix_bytes_write_100']\n",
    "f['perc_p_bytes_write_1K'] = df['posix_bytes_write_1K']\n",
    "f['perc_p_bytes_write_10K'] = df['posix_bytes_write_10K']\n",
    "f['perc_p_bytes_write_100K'] = df['posix_bytes_write_100K']\n",
    "f['perc_p_bytes_write_1M'] = df['posix_bytes_write_1M']\n",
    "f['perc_p_bytes_write_4M'] = df['posix_bytes_write_4M']\n",
    "f['perc_p_bytes_write_10M'] = df['posix_bytes_write_10M']\n",
    "f['perc_p_bytes_write_100M'] = df['posix_bytes_write_100M']\n",
    "f['perc_p_bytes_write_1G'] = df['posix_bytes_write_1G']\n",
    "f['perc_p_bytes_write_PLUS'] = df['posix_bytes_write_PLUS']\n",
    "\n",
    "\n",
    "f = f.replace(-np.inf, -1)\n",
    "f = f.replace(np.nan, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['time'] = df['posix_write_time'].astype('float') + df['posix_read_time'].astype('float') + df['posix_meta_time'].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['bytes'] = df['posix_bytes_read'].astype('float') + df['posix_bytes_write'].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df['bytes'] >9999999]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "f['throughput'] = df['bytes'].astype('float') / df['time']\n",
    "\n",
    "f = f[f['throughput'] >0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "log10_p_files              1.219270e+05\n",
       "log10_l_files              1.219270e+05\n",
       "log10_p_accesses           2.251942e+10\n",
       "log10_p_bytes              3.038456e+14\n",
       "log10_p_opens              5.531094e+08\n",
       "log10_p_seeks              1.445220e+10\n",
       "log10_p_stats              6.522921e+07\n",
       "log10_p_mode               5.337293e+07\n",
       "log10_l_n_osts             3.600000e+02\n",
       "log10_l_stripe_w           7.438575e+06\n",
       "log10_l_mdts               1.000000e+00\n",
       "log10_p_nprocs             3.200000e+05\n",
       "log10_p_falign             1.422540e+11\n",
       "log10_p_malign             1.085312e+06\n",
       "perc_p_reads               2.237846e+10\n",
       "perc_p_writes              1.302770e+10\n",
       "perc_p_bytes_read_100      5.221517e+08\n",
       "perc_p_bytes_read_1K       2.074657e+10\n",
       "perc_p_bytes_read_10K      1.536278e+09\n",
       "perc_p_bytes_read_100K     1.515506e+08\n",
       "perc_p_bytes_read_1M       4.044503e+08\n",
       "perc_p_bytes_read_4M       6.561462e+07\n",
       "perc_p_bytes_read_10M      2.083200e+06\n",
       "perc_p_bytes_read_100M     2.872090e+05\n",
       "perc_p_bytes_read_1G       1.792000e+06\n",
       "perc_p_bytes_write_100     1.302770e+10\n",
       "perc_p_bytes_write_1K      2.852127e+09\n",
       "perc_p_bytes_write_10K     3.867477e+08\n",
       "perc_p_bytes_write_100K    8.347452e+07\n",
       "perc_p_bytes_write_1M      1.357245e+07\n",
       "perc_p_bytes_write_4M      3.839488e+06\n",
       "perc_p_bytes_write_10M     6.190660e+05\n",
       "perc_p_bytes_write_100M    1.249280e+06\n",
       "perc_p_bytes_write_1G      1.937500e+04\n",
       "throughput                 2.344536e+09\n",
       "dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#delete columns with all zeros\n",
    "f = f.loc[:, (f != 0).any(axis=0)]\n",
    "\n",
    "#remove infinite values\n",
    "f = f.replace([np.inf, -np.inf], np.nan).dropna(axis=0)\n",
    "\n",
    "f.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>log10_p_files</th>\n",
       "      <th>log10_l_files</th>\n",
       "      <th>log10_p_accesses</th>\n",
       "      <th>log10_p_bytes</th>\n",
       "      <th>log10_p_opens</th>\n",
       "      <th>log10_p_seeks</th>\n",
       "      <th>log10_p_stats</th>\n",
       "      <th>log10_p_mode</th>\n",
       "      <th>log10_l_n_osts</th>\n",
       "      <th>log10_l_stripe_w</th>\n",
       "      <th>...</th>\n",
       "      <th>perc_p_bytes_read_1G</th>\n",
       "      <th>perc_p_bytes_write_100</th>\n",
       "      <th>perc_p_bytes_write_1K</th>\n",
       "      <th>perc_p_bytes_write_10K</th>\n",
       "      <th>perc_p_bytes_write_100K</th>\n",
       "      <th>perc_p_bytes_write_1M</th>\n",
       "      <th>perc_p_bytes_write_4M</th>\n",
       "      <th>perc_p_bytes_write_10M</th>\n",
       "      <th>perc_p_bytes_write_100M</th>\n",
       "      <th>perc_p_bytes_write_1G</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>799.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>880136.0</td>\n",
       "      <td>2.390891e+10</td>\n",
       "      <td>8858.0</td>\n",
       "      <td>319241.0</td>\n",
       "      <td>34901.0</td>\n",
       "      <td>37230.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>202780.0</td>\n",
       "      <td>16078.0</td>\n",
       "      <td>21989.0</td>\n",
       "      <td>3156.0</td>\n",
       "      <td>1061.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>360.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>2379598.0</td>\n",
       "      <td>5.019637e+10</td>\n",
       "      <td>62398.0</td>\n",
       "      <td>1107764.0</td>\n",
       "      <td>270222.0</td>\n",
       "      <td>27594.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18852.0</td>\n",
       "      <td>264906.0</td>\n",
       "      <td>22246.0</td>\n",
       "      <td>4282.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>290.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>8903411.0</td>\n",
       "      <td>5.488943e+12</td>\n",
       "      <td>8711.0</td>\n",
       "      <td>2010273.0</td>\n",
       "      <td>28432.0</td>\n",
       "      <td>115194.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7302089.0</td>\n",
       "      <td>52045.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>25830.0</td>\n",
       "      <td>756.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>319.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>7846387.0</td>\n",
       "      <td>2.293203e+10</td>\n",
       "      <td>23158.0</td>\n",
       "      <td>6015926.0</td>\n",
       "      <td>400399.0</td>\n",
       "      <td>31536.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>489873.0</td>\n",
       "      <td>2821931.0</td>\n",
       "      <td>240372.0</td>\n",
       "      <td>17983.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>428.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>6647935.0</td>\n",
       "      <td>5.209185e+10</td>\n",
       "      <td>69261.0</td>\n",
       "      <td>4608438.0</td>\n",
       "      <td>410846.0</td>\n",
       "      <td>38982.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>255023.0</td>\n",
       "      <td>1927630.0</td>\n",
       "      <td>186048.0</td>\n",
       "      <td>11509.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875280</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>57344.0</td>\n",
       "      <td>3.006477e+10</td>\n",
       "      <td>1808.0</td>\n",
       "      <td>59128.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28672.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875281</th>\n",
       "      <td>13.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>102439.0</td>\n",
       "      <td>7.467916e+08</td>\n",
       "      <td>193.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>385.0</td>\n",
       "      <td>4818.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>10927.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875282</th>\n",
       "      <td>624.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>2234152.0</td>\n",
       "      <td>5.593977e+10</td>\n",
       "      <td>35112.0</td>\n",
       "      <td>1035457.0</td>\n",
       "      <td>159180.0</td>\n",
       "      <td>16644.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>51807.0</td>\n",
       "      <td>397643.0</td>\n",
       "      <td>27690.0</td>\n",
       "      <td>3198.0</td>\n",
       "      <td>1207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875283</th>\n",
       "      <td>1088.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>197651.0</td>\n",
       "      <td>1.465277e+09</td>\n",
       "      <td>2112.0</td>\n",
       "      <td>12509.0</td>\n",
       "      <td>4198.0</td>\n",
       "      <td>473916.0</td>\n",
       "      <td>359.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13824.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875286</th>\n",
       "      <td>582.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>6065006.0</td>\n",
       "      <td>1.077412e+11</td>\n",
       "      <td>119432.0</td>\n",
       "      <td>2750807.0</td>\n",
       "      <td>578508.0</td>\n",
       "      <td>17520.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216962.0</td>\n",
       "      <td>1239000.0</td>\n",
       "      <td>77141.0</td>\n",
       "      <td>6985.0</td>\n",
       "      <td>272.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>643835 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        log10_p_files  log10_l_files  log10_p_accesses  log10_p_bytes  \\\n",
       "1               799.0          176.0          880136.0   2.390891e+10   \n",
       "2               360.0          224.0         2379598.0   5.019637e+10   \n",
       "3               290.0          290.0         8903411.0   5.488943e+12   \n",
       "4               319.0          201.0         7846387.0   2.293203e+10   \n",
       "6               428.0          190.0         6647935.0   5.209185e+10   \n",
       "...               ...            ...               ...            ...   \n",
       "875280            1.0            1.0           57344.0   3.006477e+10   \n",
       "875281           13.0            4.0          102439.0   7.467916e+08   \n",
       "875282          624.0          124.0         2234152.0   5.593977e+10   \n",
       "875283         1088.0         1088.0          197651.0   1.465277e+09   \n",
       "875286          582.0          128.0         6065006.0   1.077412e+11   \n",
       "\n",
       "        log10_p_opens  log10_p_seeks  log10_p_stats  log10_p_mode  \\\n",
       "1              8858.0       319241.0        34901.0       37230.0   \n",
       "2             62398.0      1107764.0       270222.0       27594.0   \n",
       "3              8711.0      2010273.0        28432.0      115194.0   \n",
       "4             23158.0      6015926.0       400399.0       31536.0   \n",
       "6             69261.0      4608438.0       410846.0       38982.0   \n",
       "...               ...            ...            ...           ...   \n",
       "875280         1808.0        59128.0           12.0           0.0   \n",
       "875281          193.0           91.0          385.0        4818.0   \n",
       "875282        35112.0      1035457.0       159180.0       16644.0   \n",
       "875283         2112.0        12509.0         4198.0      473916.0   \n",
       "875286       119432.0      2750807.0       578508.0       17520.0   \n",
       "\n",
       "        log10_l_n_osts  log10_l_stripe_w  ...  perc_p_bytes_read_1G  \\\n",
       "1                147.0             176.0  ...                   0.0   \n",
       "2                186.0             224.0  ...                   0.0   \n",
       "3                 52.0             290.0  ...                   0.0   \n",
       "4                158.0             201.0  ...                   0.0   \n",
       "6                158.0             190.0  ...                   0.0   \n",
       "...                ...               ...  ...                   ...   \n",
       "875280             1.0               8.0  ...                   0.0   \n",
       "875281             4.0               4.0  ...                   0.0   \n",
       "875282           110.0             124.0  ...                   0.0   \n",
       "875283           359.0            1088.0  ...                   0.0   \n",
       "875286           118.0             128.0  ...                   0.0   \n",
       "\n",
       "        perc_p_bytes_write_100  perc_p_bytes_write_1K  perc_p_bytes_write_10K  \\\n",
       "1                     202780.0                16078.0                 21989.0   \n",
       "2                      18852.0               264906.0                 22246.0   \n",
       "3                    7302089.0                52045.0                    73.0   \n",
       "4                     489873.0              2821931.0                240372.0   \n",
       "6                     255023.0              1927630.0                186048.0   \n",
       "...                        ...                    ...                     ...   \n",
       "875280                     0.0                    0.0                     0.0   \n",
       "875281                    33.0                   23.0                 10927.0   \n",
       "875282                 51807.0               397643.0                 27690.0   \n",
       "875283                    28.0                    0.0                    38.0   \n",
       "875286                216962.0              1239000.0                 77141.0   \n",
       "\n",
       "        perc_p_bytes_write_100K  perc_p_bytes_write_1M  perc_p_bytes_write_4M  \\\n",
       "1                        3156.0                 1061.0                    0.0   \n",
       "2                        4282.0                   14.0                    0.0   \n",
       "3                           9.0                25830.0                  756.0   \n",
       "4                       17983.0                  242.0                    0.0   \n",
       "6                       11509.0                  158.0                    0.0   \n",
       "...                         ...                    ...                    ...   \n",
       "875280                      0.0                28672.0                    0.0   \n",
       "875281                      0.0                    0.0                    0.0   \n",
       "875282                   3198.0                 1207.0                    0.0   \n",
       "875283                      0.0                13824.0                    0.0   \n",
       "875286                   6985.0                  272.0                    0.0   \n",
       "\n",
       "        perc_p_bytes_write_10M  perc_p_bytes_write_100M  perc_p_bytes_write_1G  \n",
       "1                          0.0                      0.0                    0.0  \n",
       "2                          0.0                      0.0                    0.0  \n",
       "3                          0.0                      0.0                    0.0  \n",
       "4                          0.0                      0.0                    0.0  \n",
       "6                          0.0                      0.0                    0.0  \n",
       "...                        ...                      ...                    ...  \n",
       "875280                     0.0                      0.0                    0.0  \n",
       "875281                     0.0                      0.0                    0.0  \n",
       "875282                     0.0                      0.0                    0.0  \n",
       "875283                     0.0                      0.0                    0.0  \n",
       "875286                     0.0                      0.0                    0.0  \n",
       "\n",
       "[643835 rows x 34 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = pd.DataFrame()\n",
    "t['throughput'] = f['throughput']\n",
    "f = f.drop(labels = 'throughput', axis = 1)\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.index.isin(t.index)]\n",
    "t = t.reset_index()\n",
    "f = f.reset_index()\n",
    "f = f.drop(f.columns[0] , axis =1)\n",
    "t = t.drop(t.columns[0] , axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = StandardScaler().fit_transform(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>throughput</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.803194e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.282342e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.083669e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.724841e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.581690e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643830</th>\n",
       "      <td>5.482767e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643831</th>\n",
       "      <td>2.136637e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643832</th>\n",
       "      <td>1.362498e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643833</th>\n",
       "      <td>1.942413e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643834</th>\n",
       "      <td>1.842942e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>643835 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          throughput\n",
       "0       1.803194e+08\n",
       "1       2.282342e+08\n",
       "2       2.083669e+08\n",
       "3       1.724841e+07\n",
       "4       4.581690e+07\n",
       "...              ...\n",
       "643830  5.482767e+08\n",
       "643831  2.136637e+07\n",
       "643832  1.362498e+08\n",
       "643833  1.942413e+07\n",
       "643834  1.842942e+08\n",
       "\n",
       "[643835 rows x 1 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "throughput    34.636498\n",
      "dtype: float64\n",
      "throughput    2.344536e+09\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(t.min())\n",
    "print(t.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "rseed = 0\n",
    "t_size = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "from sklearn.metrics import mean_absolute_percentage_error\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4443\n",
      "Mean True Value: \t 87399220\n",
      "Mean Absolute Error: \t 54618776\n",
      "Mean Squared Error:  2.4971233113764916e+16\n",
      "Root Mean Squared Error:  158022887.94274366\n",
      "MAPE :51.506406341620696\n",
      "R2: -1.058299261804346\n",
      "\n",
      "5461\n",
      "Mean True Value: \t 87069818\n",
      "Mean Absolute Error: \t 55237135\n",
      "Mean Squared Error:  8602123564594556.0\n",
      "Root Mean Squared Error:  92747633.74121496\n",
      "MAPE :34.35180550220375\n",
      "R2: 0.2727188760585141\n",
      "\n",
      "5096\n",
      "Mean True Value: \t 87821834\n",
      "Mean Absolute Error: \t 55380588\n",
      "Mean Squared Error:  8720098539358566.0\n",
      "Root Mean Squared Error:  93381467.85823494\n",
      "MAPE :109.28940318636863\n",
      "R2: 0.2741195303699203\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "for i in range(3):\n",
    "    \n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size=t_size, random_state=rseed)\n",
    "\n",
    "    reg = LinearRegression().fit(train_data, train_labels)\n",
    "\n",
    "    predicted_labels = reg.predict(test_data)\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4108\n",
      "Mean True Value: \t 87681038\n",
      "Mean Absolute Error: \t 17043438\n",
      "Mean Squared Error:  1254160897500540.8\n",
      "Root Mean Squared Error:  35414134.148677714\n",
      "MAPE :1.8567716043209894\n",
      "R2: 0.8951109097281286\n",
      "\n",
      "3471\n",
      "Mean True Value: \t 87980343\n",
      "Mean Absolute Error: \t 16889347\n",
      "Mean Squared Error:  1235690270358956.5\n",
      "Root Mean Squared Error:  35152386.41058323\n",
      "MAPE :1.3879568902693258\n",
      "R2: 0.8981300669270426\n",
      "\n",
      "9332\n",
      "Mean True Value: \t 88352544\n",
      "Mean Absolute Error: \t 17145459\n",
      "Mean Squared Error:  1233559755859044.2\n",
      "Root Mean Squared Error:  35122069.35616186\n",
      "MAPE :1.848308038571628\n",
      "R2: 0.8987565120459793\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#EXTREME GRADIENT BOOST\n",
    "\n",
    "for i in range(3):\n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size=t_size, random_state=rseed)\n",
    "    xgb_r = xg.XGBRegressor(n_estimators = 1000, seed = 123)\n",
    "    xgb_r.fit(train_data, train_labels)\n",
    "    predicted_labels = xgb_r.predict(test_data)\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1056\n",
      "Mean True Value: \t 87305207\n",
      "Mean Absolute Error: \t 33471938\n",
      "Mean Squared Error:  3066867907450926.0\n",
      "Root Mean Squared Error:  55379309.380407825\n",
      "MAPE :37.011378831900586\n",
      "R2: 0.7429626122139912\n",
      "\n",
      "8901\n",
      "Mean True Value: \t 86355413\n",
      "Mean Absolute Error: \t 33274122\n",
      "Mean Squared Error:  2943914107879881.0\n",
      "Root Mean Squared Error:  54257848.35284091\n",
      "MAPE :21.83074794738932\n",
      "R2: 0.7438499950730912\n",
      "\n",
      "6109\n",
      "Mean True Value: \t 87654101\n",
      "Mean Absolute Error: \t 33537624\n",
      "Mean Squared Error:  3104066352358050.5\n",
      "Root Mean Squared Error:  55714148.58326429\n",
      "MAPE :124.44988461046967\n",
      "R2: 0.7412106678920202\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "for i in range(3):\n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size= t_size, random_state=rseed)\n",
    "    \n",
    "    reg = DecisionTreeRegressor(max_depth = 4)   \n",
    "    \n",
    "\n",
    "    reg.fit(train_data, train_labels)\n",
    "    predicted_labels = reg.predict(test_data)\n",
    "\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}