Concrete Compressive Strength.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading data from provided csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing required packages\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cement (component 1)(kg in a m^3 mixture)</th>\n",
       "      <th>Blast Furnace Slag (component 2)(kg in a m^3 mixture)</th>\n",
       "      <th>Fly Ash (component 3)(kg in a m^3 mixture)</th>\n",
       "      <th>Water  (component 4)(kg in a m^3 mixture)</th>\n",
       "      <th>Superplasticizer (component 5)(kg in a m^3 mixture)</th>\n",
       "      <th>Coarse Aggregate  (component 6)(kg in a m^3 mixture)</th>\n",
       "      <th>Fine Aggregate (component 7)(kg in a m^3 mixture)</th>\n",
       "      <th>Age (day)</th>\n",
       "      <th>Concrete compressive strength(MPa, megapascals)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1040.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>79.986111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1055.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>61.887366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>270</td>\n",
       "      <td>40.269535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>365</td>\n",
       "      <td>41.052780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>198.6</td>\n",
       "      <td>132.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978.4</td>\n",
       "      <td>825.5</td>\n",
       "      <td>360</td>\n",
       "      <td>44.296075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cement (component 1)(kg in a m^3 mixture)  \\\n",
       "0                                      540.0   \n",
       "1                                      540.0   \n",
       "2                                      332.5   \n",
       "3                                      332.5   \n",
       "4                                      198.6   \n",
       "\n",
       "   Blast Furnace Slag (component 2)(kg in a m^3 mixture)  \\\n",
       "0                                                0.0       \n",
       "1                                                0.0       \n",
       "2                                              142.5       \n",
       "3                                              142.5       \n",
       "4                                              132.4       \n",
       "\n",
       "   Fly Ash (component 3)(kg in a m^3 mixture)  \\\n",
       "0                                         0.0   \n",
       "1                                         0.0   \n",
       "2                                         0.0   \n",
       "3                                         0.0   \n",
       "4                                         0.0   \n",
       "\n",
       "   Water  (component 4)(kg in a m^3 mixture)  \\\n",
       "0                                      162.0   \n",
       "1                                      162.0   \n",
       "2                                      228.0   \n",
       "3                                      228.0   \n",
       "4                                      192.0   \n",
       "\n",
       "   Superplasticizer (component 5)(kg in a m^3 mixture)  \\\n",
       "0                                                2.5     \n",
       "1                                                2.5     \n",
       "2                                                0.0     \n",
       "3                                                0.0     \n",
       "4                                                0.0     \n",
       "\n",
       "   Coarse Aggregate  (component 6)(kg in a m^3 mixture)  \\\n",
       "0                                             1040.0      \n",
       "1                                             1055.0      \n",
       "2                                              932.0      \n",
       "3                                              932.0      \n",
       "4                                              978.4      \n",
       "\n",
       "   Fine Aggregate (component 7)(kg in a m^3 mixture)  Age (day)  \\\n",
       "0                                              676.0         28   \n",
       "1                                              676.0         28   \n",
       "2                                              594.0        270   \n",
       "3                                              594.0        365   \n",
       "4                                              825.5        360   \n",
       "\n",
       "   Concrete compressive strength(MPa, megapascals)   \n",
       "0                                         79.986111  \n",
       "1                                         61.887366  \n",
       "2                                         40.269535  \n",
       "3                                         41.052780  \n",
       "4                                         44.296075  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Reading data from Excel file available in local system\n",
    "df_concrete = pd.read_excel('data/Concrete/concrete_data.xls')\n",
    "\n",
    "df_concrete.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cement (component 1)(kg in a m^3 mixture)</th>\n",
       "      <th>Blast Furnace Slag (component 2)(kg in a m^3 mixture)</th>\n",
       "      <th>Fly Ash (component 3)(kg in a m^3 mixture)</th>\n",
       "      <th>Water  (component 4)(kg in a m^3 mixture)</th>\n",
       "      <th>Superplasticizer (component 5)(kg in a m^3 mixture)</th>\n",
       "      <th>Coarse Aggregate  (component 6)(kg in a m^3 mixture)</th>\n",
       "      <th>Fine Aggregate (component 7)(kg in a m^3 mixture)</th>\n",
       "      <th>Age (day)</th>\n",
       "      <th>Concrete compressive strength(MPa, megapascals)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "      <td>1030.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>281.165631</td>\n",
       "      <td>73.895485</td>\n",
       "      <td>54.187136</td>\n",
       "      <td>181.566359</td>\n",
       "      <td>6.203112</td>\n",
       "      <td>972.918592</td>\n",
       "      <td>773.578883</td>\n",
       "      <td>45.662136</td>\n",
       "      <td>35.817836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>104.507142</td>\n",
       "      <td>86.279104</td>\n",
       "      <td>63.996469</td>\n",
       "      <td>21.355567</td>\n",
       "      <td>5.973492</td>\n",
       "      <td>77.753818</td>\n",
       "      <td>80.175427</td>\n",
       "      <td>63.169912</td>\n",
       "      <td>16.705679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>102.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>121.750000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>801.000000</td>\n",
       "      <td>594.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.331808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>192.375000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>164.900000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>932.000000</td>\n",
       "      <td>730.950000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>23.707115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>272.900000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>185.000000</td>\n",
       "      <td>6.350000</td>\n",
       "      <td>968.000000</td>\n",
       "      <td>779.510000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>34.442774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>350.000000</td>\n",
       "      <td>142.950000</td>\n",
       "      <td>118.270000</td>\n",
       "      <td>192.000000</td>\n",
       "      <td>10.160000</td>\n",
       "      <td>1029.400000</td>\n",
       "      <td>824.000000</td>\n",
       "      <td>56.000000</td>\n",
       "      <td>46.136287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>540.000000</td>\n",
       "      <td>359.400000</td>\n",
       "      <td>200.100000</td>\n",
       "      <td>247.000000</td>\n",
       "      <td>32.200000</td>\n",
       "      <td>1145.000000</td>\n",
       "      <td>992.600000</td>\n",
       "      <td>365.000000</td>\n",
       "      <td>82.599225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Cement (component 1)(kg in a m^3 mixture)  \\\n",
       "count                                1030.000000   \n",
       "mean                                  281.165631   \n",
       "std                                   104.507142   \n",
       "min                                   102.000000   \n",
       "25%                                   192.375000   \n",
       "50%                                   272.900000   \n",
       "75%                                   350.000000   \n",
       "max                                   540.000000   \n",
       "\n",
       "       Blast Furnace Slag (component 2)(kg in a m^3 mixture)  \\\n",
       "count                                        1030.000000       \n",
       "mean                                           73.895485       \n",
       "std                                            86.279104       \n",
       "min                                             0.000000       \n",
       "25%                                             0.000000       \n",
       "50%                                            22.000000       \n",
       "75%                                           142.950000       \n",
       "max                                           359.400000       \n",
       "\n",
       "       Fly Ash (component 3)(kg in a m^3 mixture)  \\\n",
       "count                                 1030.000000   \n",
       "mean                                    54.187136   \n",
       "std                                     63.996469   \n",
       "min                                      0.000000   \n",
       "25%                                      0.000000   \n",
       "50%                                      0.000000   \n",
       "75%                                    118.270000   \n",
       "max                                    200.100000   \n",
       "\n",
       "       Water  (component 4)(kg in a m^3 mixture)  \\\n",
       "count                                1030.000000   \n",
       "mean                                  181.566359   \n",
       "std                                    21.355567   \n",
       "min                                   121.750000   \n",
       "25%                                   164.900000   \n",
       "50%                                   185.000000   \n",
       "75%                                   192.000000   \n",
       "max                                   247.000000   \n",
       "\n",
       "       Superplasticizer (component 5)(kg in a m^3 mixture)  \\\n",
       "count                                        1030.000000     \n",
       "mean                                            6.203112     \n",
       "std                                             5.973492     \n",
       "min                                             0.000000     \n",
       "25%                                             0.000000     \n",
       "50%                                             6.350000     \n",
       "75%                                            10.160000     \n",
       "max                                            32.200000     \n",
       "\n",
       "       Coarse Aggregate  (component 6)(kg in a m^3 mixture)  \\\n",
       "count                                        1030.000000      \n",
       "mean                                          972.918592      \n",
       "std                                            77.753818      \n",
       "min                                           801.000000      \n",
       "25%                                           932.000000      \n",
       "50%                                           968.000000      \n",
       "75%                                          1029.400000      \n",
       "max                                          1145.000000      \n",
       "\n",
       "       Fine Aggregate (component 7)(kg in a m^3 mixture)    Age (day)  \\\n",
       "count                                        1030.000000  1030.000000   \n",
       "mean                                          773.578883    45.662136   \n",
       "std                                            80.175427    63.169912   \n",
       "min                                           594.000000     1.000000   \n",
       "25%                                           730.950000     7.000000   \n",
       "50%                                           779.510000    28.000000   \n",
       "75%                                           824.000000    56.000000   \n",
       "max                                           992.600000   365.000000   \n",
       "\n",
       "       Concrete compressive strength(MPa, megapascals)   \n",
       "count                                       1030.000000  \n",
       "mean                                          35.817836  \n",
       "std                                           16.705679  \n",
       "min                                            2.331808  \n",
       "25%                                           23.707115  \n",
       "50%                                           34.442774  \n",
       "75%                                           46.136287  \n",
       "max                                           82.599225  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# describe the data\n",
    "df_concrete.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#renaming columns\n",
    "df_concrete = df_concrete.rename(columns={'Cement (component 1)(kg in a m^3 mixture)':\"cement\",\n",
    "       'Blast Furnace Slag (component 2)(kg in a m^3 mixture)':\"furnace_slag\",\n",
    "       'Fly Ash (component 3)(kg in a m^3 mixture)':\"fly_ash\",\n",
    "       'Water  (component 4)(kg in a m^3 mixture)':\"water\",\n",
    "       'Superplasticizer (component 5)(kg in a m^3 mixture)':\"super_plasticizer\",\n",
    "       'Coarse Aggregate  (component 6)(kg in a m^3 mixture)':\"coarse_agg\",\n",
    "       'Fine Aggregate (component 7)(kg in a m^3 mixture)':\"fine_agg\", 'Age (day)':\"age\",\n",
    "       'Concrete compressive strength(MPa, megapascals) ':\"compressive_strength\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cement</th>\n",
       "      <th>furnace_slag</th>\n",
       "      <th>fly_ash</th>\n",
       "      <th>water</th>\n",
       "      <th>super_plasticizer</th>\n",
       "      <th>coarse_agg</th>\n",
       "      <th>fine_agg</th>\n",
       "      <th>age</th>\n",
       "      <th>compressive_strength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1040.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>79.986111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1055.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>61.887366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>270</td>\n",
       "      <td>40.269535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>365</td>\n",
       "      <td>41.052780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>198.6</td>\n",
       "      <td>132.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978.4</td>\n",
       "      <td>825.5</td>\n",
       "      <td>360</td>\n",
       "      <td>44.296075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cement  furnace_slag  fly_ash  water  super_plasticizer  coarse_agg  \\\n",
       "0   540.0           0.0      0.0  162.0                2.5      1040.0   \n",
       "1   540.0           0.0      0.0  162.0                2.5      1055.0   \n",
       "2   332.5         142.5      0.0  228.0                0.0       932.0   \n",
       "3   332.5         142.5      0.0  228.0                0.0       932.0   \n",
       "4   198.6         132.4      0.0  192.0                0.0       978.4   \n",
       "\n",
       "   fine_agg  age  compressive_strength  \n",
       "0     676.0   28             79.986111  \n",
       "1     676.0   28             61.887366  \n",
       "2     594.0  270             40.269535  \n",
       "3     594.0  365             41.052780  \n",
       "4     825.5  360             44.296075  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_concrete.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1030 entries, 0 to 1029\n",
      "Data columns (total 9 columns):\n",
      "cement                  1030 non-null float64\n",
      "furnace_slag            1030 non-null float64\n",
      "fly_ash                 1030 non-null float64\n",
      "water                   1030 non-null float64\n",
      "super_plasticizer       1030 non-null float64\n",
      "coarse_agg              1030 non-null float64\n",
      "fine_agg                1030 non-null float64\n",
      "age                     1030 non-null int64\n",
      "compressive_strength    1030 non-null float64\n",
      "dtypes: float64(8), int64(1)\n",
      "memory usage: 72.5 KB\n"
     ]
    }
   ],
   "source": [
    "# information of the data\n",
    "df_concrete.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cement                  0\n",
       "furnace_slag            0\n",
       "fly_ash                 0\n",
       "water                   0\n",
       "super_plasticizer       0\n",
       "coarse_agg              0\n",
       "fine_agg                0\n",
       "age                     0\n",
       "compressive_strength    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking if there is any NULL data \n",
    "df_concrete.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1030, 9)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Data size\n",
    "df_concrete.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1500x1500 with 81 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pandas.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "sm = scatter_matrix(df_concrete, figsize=(15,15), diagonal = 'kde')\n",
    "#Changing label rotation\n",
    "[s.xaxis.label.set_rotation(45) for s in sm.reshape(-1)]\n",
    "[s.yaxis.label.set_rotation(45) for s in sm.reshape(-1)]\n",
    "#Changing offset for label\n",
    "[s.get_yaxis().set_label_coords(-0.5,0.5) for s in sm.reshape(-1)]\n",
    "#Hiding ticks\n",
    "[s.set_xticks(()) for s in sm.reshape(-1)]\n",
    "[s.set_yticks(()) for s in sm.reshape(-1)]\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAAFhCAYAAADjkKoFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd7xcVb3+8c+TEAyQ0NFLb1KkQwKCoBSRK+gVC4JdQUFUBFQs9ycIolwrcikKolLEAoIgqAgo0qWFDlGUiyABFBCkl5A8vz/2OjI5nJZk9uw5Z543r/1iZs+e/V37nJP5zlp7FdkmIiKiV41rugARERFNSiKMiIielkQYERE9LYkwIiJ6WhJhRET0tCTCiIjoaUmEERExKkg6QdIDkm4d5HVJOkrSHZJulrTJSM6bRBgREaPFScDrh3h9R2CNsu0FHDuSkyYRRkTEqGD7UuDhIQ7ZGfihK1cBi0tadrjzJhFGRMRYsTxwT8vzGWXfkBaorTjRtWY+dGfH59Xbb+rnOh0SgM1mLtjxmAc/c3PHYwIsoGb+OS84rpm46y007Bf9trvmib92PCbAGRNXbyTupveepfk9x9x83iy4zOofpmrS7HO87ePnItxA5R02fhJhRETUZ/asER9akt7cJL7+ZgArtjxfAbhvuDelaTQiIurj2SPf5t85wPtK79HNgUdt3z/cm1IjjIiI+sxuS4IDQNJPgW2ApSXNAA4GJgDYPg44F9gJuAN4Cth9JOdNIoyIiNq4PTW9ci6/c5jXDXxsbs+bRBgREfVpY42wLkmEERFRn1kzmy7BsJIIIyKiPm1sGq1LEuEYIGkjYDnb5zZdloiIOYyCptEMnxgbNqLqKRUR0VXs2SPempJE2AaS3ldmOr9J0imSlpH0c0nXlm3Lctwhkk6WdIGkuyS9VdLXJd0i6TxJE8pxUyRdIuk6Sef3zZUn6WJJX5N0jaQ/S3q1pAWBQ4HdJN0oabfmfhIREf3Mnj3yrSFpGp1PktYFPg9safshSUsCxwBH2L5c0krA+cAryltWB7YF1gGuBN5m+zOSzgLeIOnXwNHAzrYfLIntMGCP8v4FbG8maSfgYNvbS/oCMNX2Ph267IiIkck9wp6wHXCG7YcAbD8saXtgHenf094tKmlyefwb2zMl3QKMB84r+28BVgHWAtYDflvePx5onRnhzPL/68rxIyJpL8ocft85/Mt86H1DDseJiGiP9BrtCeLFk7qOA7aw/fQcB1aJ7VkA27MlzSwDQAFmU/0+BNxme4tB4j1b/j+Lufj9tc7h18Sk2xHRo9JZpidcCOwqaSmA0jR6AfDvZsrSq3OkbgeWkbRFee+E0vw6lMeBycMcExHReZ2da3SeJBHOJ9u3Ud3Du0TSTcC3gH2BqaUDzXRg77k433PALsDXyvluBF41zNsuomqKTWeZiOgu6SzTG2yfDJzcb/eLEpLtQ/o9nzTQa7ZvBF4zwPu3aXn8EOUeoe2HgU3noegREbWyR74MU1OSCCMioj6znm+6BMNKIoyIiPpk+ERERPS0uVihvilJhBERUZ/UCCMioqeNgnGESYQREVGf1AgjIqKnPZ9eo9GF9pv6uY7HPHLaVzseE+CwKQd1POZC41/S8ZgA/3jqkUbivm/pqY3E9YtmNqzfY88+1fGYAB9e8MFG4l7fhnNkHGFERPS23COMiIielnuEERHR01IjjIiInpYaYURE9LTMNRoRET0tTaMREdHTkggjIqKnjYJ7hGN6hXpJ+0r6o6QfN12WuSXpEEkHNF2OiIj5khXqG/dRYEfbfx3uQEkCZI+Cry8REaPFKOgsM2ZrhJKOA1YDzpH0aGvtStKtklYp2x8lfYdqNqEVJT0h6TBJN0m6StLLynv+S9LVkm6Q9LuW/ZMknSjpFkk3S3pb2b+DpCslXS/pdEmThijrVyVNL+//5gCv7ynp2lKmn0tauOxfvZTxWkmHSnqinT/DiIj55tkj3xoyZhOh7b2B+4BtgSOGOHQt4Ie2N7Z9N7AIcJXtDYFLgT3LcZcDm9veGDgV+EzZfxDwqO31bW8A/F7S0sCBwPa2NwGmAZ8cKLikJYG3AOuW9395gMPOtL1pKdMfgQ+W/UcCR9retFzroCTtJWmapGnTH79zqEMjItpnFDSNjtlEOBfutn1Vy/PngF+Vx9cBq5THKwDnS7oF+DSwbtm/PfDtvjfbfgTYHFgHuELSjcD7gZUHif8Y8AzwfUlvBQaa1Xc9SZeV2O9uib0FcHp5/JOhLtL28ban2p66zuTVhjo0IqJ9kgi7xvPMea0TWx4/2e/Ymbb7prWfxQv3UY8GjrG9PvDhlnMIXjQNvoDf2t6obOvY/iADsP08sBnwc+DNwHkDHHYSsE+J/cV+5Y+I6F72yLeG9EoivAvYBEDSJsCq83COxYB7y+P3t+y/ANin74mkJYCrgC0lvbzsW1jSmgOdtNw7XMz2ucD+wEYDHDYZuF/SBKoaYZ+rgLeVx++Y6yuKiKhbaoRd4+fAkqWZ8iPAn+fhHIcAp0u6DHioZf+XgSVKB5ybgG1tPwh8APippJupEtbag5x3MvCrctwlwCcGOOYg4Grgt8CfWvbvD3xS0jXAssCj83BdERH1mfX8yLcRkPR6SbdLukPSixZXlbSSpItKx8abJe003DnH9PAJ26u0PN1hkMPW6/eeSS2PzwDOKI/PBs4eIMYTzFlD7Nv/e2DTEZTxfqqm0f77D2l5fCxw7ABvv5eqA48lvYOqU05ERPdoY01P0niqPhmvA2YA10o6x/b0lsMOBH5m+1hJ6wDn8kJfjwGN6UTYA6YAx5QxkP8C9mi4PBERc2rvvb/NgDts3wkg6VRgZ6A1ERpYtDxejGF61EMSYUdJOosX35/8rO3z5+V8ti8DNpzvgkVE1KW99/6WB+5peT4DeGW/Yw4BLpD0carhcNsPd9Ikwg6y/ZamyxAR0VFzkQgl7QXs1bLreNvHtx4ywNv6VznfCZxk+3BJWwCnSFpvqFnDkggjIqI2njVr5MdWSe/4IQ6ZAazY8nwFXtz0+UHg9eV8V0qaCCwNPDDYSXul12hERDShvcMnrgXWkLSqpAWpho2d0++YvwGvBZD0Cqpx1w8OddLUCCMioj5tnEPU9vOS9gHOB8YDJ9i+TdKhwDTb5wCfAr4n6RNUzaYfaJkkZUBJhD1os5kLdjzmYVMO6nhMgM9f96WOx5wx9TPDH1SDqS9Zq5G4H7/vokbiLjtpyY7HXHuxFYc/qAaPzBzF8+nPbu+MMWXykXP77ftCy+PpwJZzc84kwoiIqE9WqI+IiJ6WRBgRET1tLnqNNiWJMCIi6tPme4R1SCKMiIj6NLjy/EglEUZERH1SI4yIiF7mdJaJiIielhphRET0tFHQazRzjbaZpH0l/VHSvZKOqTnWB+qOERExX9o712gtUiNsv48COwJbA1MbLktERLNGQdNoaoRtJOk4YDWq2dCXKPsmS/qrpAnl+aKS7up7PsA59pR0raSbJP1c0sJl/9sl3Vr2X9ryluUknSfpL5K+PkTZ9pI0TdK0i5/8S7suOSJiaJ498q0hSYRtZHtvqrWxtgUeKfseBy4G3lAOewfwc9szBznNmbY3tb0h8EeqtbUAvgD8Z9n/ppbjNwJ2A9YHdpM04KzAto+3PdX21G0WWWNeLzEiYu7M9si3hiQRdsb3gd3L492BE4c4dj1Jl0m6BXg3sG7ZfwVwkqQ9qZYf6XOh7UdtPwNMB1Zub9EjIuadn5814q0puUfYAbavkLSKpK2B8bZvHeLwk4A3275J0geAbco59pb0Sqqa5Y2SNirHP9vy3lnkdxoR3ST3CKPFD4GfMnRtEGAycH+5h/juvp2SVrd9dVl36yGgmYXRIiLmRu4RRosfU3Wg+ekwxx0EXA38FvhTy/5vSLpF0q3ApcBNtZQyIqKdRsE9wjSjtZntVcrDk8rWZyvgDNv/Gub9xwLHDrD/rQMcPkcM22+cm7JGRNTNo6BpNImwAyQdTTW2cKemyxIR0VFJhAFg++P990n6NrBlv91H2h7uHmJExOjRYG/QkUoibIjtjzVdhoiI2qVGGBERvcxOIoyIiF6WGmF0o4OfubnjMRca/5KOxwSYMfUzHY957LRBp3yt1VOf2LORuGdevH4jcfd/bvGOxzx3QjMf6hMmvKyRuG2RRBgREb0swyciIqK3PZ9EGBERPSw1woiI6G1JhBER0dOam0t7xJIIIyKiNmkajYiInuZ0lomIiJ42CppGsx5hl5O0v6SFmy5HRMS8GAXr8iYRjgL7A3OVCCWNr6ksERFzZ/ZcbA1JIuwQSZ+RtG95fISk35fHr5X0I0nHSpom6TZJXyyv7QssB1wk6aKybwdJV0q6XtLpkiaV/XdJ+oKky4G3N3KRERH9tLtGKOn1km6XdIekzw1yzK6SppfP058Md84kws65FHh1eTwVmCRpAtXK9ZcBn7c9FdgA2FrSBraPAu4DtrW9raSlgQOB7W1vAkwDPtkS4xnbW9k+tUPXFBExtDbWCEtr17epFjpfB3inpHX6HbMG8N/AlrbXpWpVG1I6y3TOdcAUSZOBZ4HrqRLiq4F9gV0l7UX1O1mW6pfcf3bszcv+KyQBLAhc2fL6aYMFL+feC2DJhZdn0sQl23BJERFDm/18W0+3GXCH7TsBJJ0K7AxMbzlmT+Dbth8BsP3AcCdNIuwQ2zMl3QXsDvyBKsltC6wOPA0cAGxq+xFJJwETBziNgN/afucgYZ4cIv7xwPEAKy+1Qff3Z46IMaHNnWCWB+5peT4DeGW/Y9YEkHQFMB44xPZ5Q500TaOddSlVwruUqjl0b+BGYFGqJPaopJdRVfv7PA5MLo+vAraU9HIASQtLWrNDZY+ImHvWiDdJe5W+En3bXv3OpoEi9Hu+ALAGsA3wTuD7koZcsys1ws66DPg8cKXtJyU9A1xm+yZJNwC3AXcCV7S853jgN5LuL/cJPwD8VFLfAn8HAn/u3CVERIzc3NQIW1uuBjEDWLHl+QpU/Sj6H3OV7ZnAXyXdTpUYrx3spEmEHWT7QmBCy/M1Wx5/YJD3HA0c3fL898CmAxy3ShuLGhHRFp49UCVunl0LrCFpVeBe4B3Au/od8wuqmuBJpYPhmlQVjEElEUZERG3aeY/Q9vOS9gHOp7r/d4Lt2yQdCkyzfU55bQdJ04FZwKdt/3Oo8yYRRkREbWbPamuNENvnAuf22/eFlsemGlb2SUYoiTAiImrT5qbRWiQRRkREbTwKBmslEUZERG1SI4yIiJ6WRBhdaQF1/tf+j6ce6XhMgKkvWavjMZ/6xJ4djwmw8BHfayTuqlM/20jcEyc+3fGYT8x6ruMxAXZiqUbitkO7O8vUIYkwIiJqYycRRkRED2tywd2RSiKMiIjazE6NMCIielmaRiMioqel12hERPS09BqNiIielnuEERHR03KPMCIietpomGt0XNMFGA0knSRpl3l43zaSXtXyfG9J7xvi+OUknTGv5YyI6DazrRFvTem5GqEkAbI7MsxzG+AJ4A8Ato8b6mDb9wFznXBbdfj6IiKGNBqaRrumRihpEUm/lnSTpFsl7SbpLklLl9enSrq4PD5E0imSfi/pL5L2bDnPpyVdK+lmSV8s+1aR9EdJ3wGuB1YcpAxPSDpc0vWSLpS0zADHfKGc/1ZJx5fEg6R9JU0vcU+VtAqwN/AJSTdKenUp9wHl+JdL+l253uslrV7KeWt5/fvlfTdKelDSwfN7fRERnTZrtka8NaVrEiHweuA+2xvaXg84b5jjNwDeAGwBfKE0K+4ArAFsBmwETJH0mnL8WsAPbW9s++5BzrkIcL3tTYBLgIMHOOYY25uWMi4EvLHs/xywse0NgL1t3wUcBxxheyPbl/U7z4+Bb9veEHgVcH/ri7Y/ZHsjYGfgn8BJ83N9kvaSNE3StMeeeWiQy4+IaC9bI96a0k2J8BZge0lfk/Rq248Oc/zZtp+2/RBwEVVy2KFsN1DVjNamShwAd9u+aphzzgZOK49/BGw1wDHbSrpa0i3AdsC6Zf/NwI8lvQd4fqggkiYDy9s+C8D2M7afGuC4icDpwD4luc3z9dk+3vZU21MXnbj0UMWLiGib3COcC7b/LGkKsBPwFUkXUCWUvmQ9sf9bBngu4Cu2v9v6QmmmfHJeitXvPBOB7wBTbd8j6ZCWcr0BeA3wJuAgSesyuJH+xo8DzrT9u5b3tfP6IiJqNQo6jXZPjVDScsBTtn8EfBPYBLgLmFIOeVu/t+wsaaKkpag6pVwLnA/sIWlSOefykl46F8UYxwudVd4FXN7v9b6k91CJsUuJMw5Y0fZFwGeAxYFJwOPA5P5BbD8GzJD05vL+l0hauPUYSR8DJtv+asvu+b2+iIiOSo1w7qwPfEPSbGAm8BGqe3A/kPT/gKv7HX8N8GtgJeBLpcflfZJeAVxZ+rA8AbwHmDXCMjwJrCvpOuBRYLfWF23/S9L3qJpx76JKvgDjgR9JWoyq1nZEOfaXwBmSdgY+3i/We4HvSjq0XO/bqZpm+xwAzJR0Y3l+nO3j5vP6IiI6atYo6DXaNYnQ9vlUNZ7+1hzkLX+2vdcA5zkSOHKA49cbYTkOAg7qt+8DLY8PBA4c4K0vup9o+89UnXr6XNby2l+o7jEOWE7bqw5Svvm6voiITvKI7wQ1p2sSYUREjD2zR8FNwlGZCG0fMj/vl3Q18JJ+u99re9L8nDciIuY0OzXC7mT7lU2XISKiF6RpNCIietpomOsxiTAiImozKzXC6EYLjuv8r/19S0/teEyAj993Ucdjnnnx+h2PCbDq1M82EveYaV9rJO6OG3+k4zEv/setHY8J8OQyazcSt/+Yr3mRGmFERPS03COMiIie1uCiEiOWRBgREbXJ8ImIiOhpo2H+xyTCiIiozWylRhgRET1sFMyw1j3LMEVExNgzey62kZD0ekm3S7pD0ueGOG4XSZY07Nit1AgjIqI27ew1Kmk88G3gdcAM4FpJ59ie3u+4ycC+vHj5vgGlRhgREbWZjUa8jcBmwB2277T9HHAqsPMAx30J+DrwzEhOmkTYBpJSs46IGMAsjXwbgeWBe1qezyj7/k3SxsCKtn810jL2bCKU9D5JN0u6SdIpklaWdGHZd6Gklcpx/yXpakk3SPqdpJeV/YdIOl7SBcAPJa0r6RpJN5ZzrFGOe0/L/u+Wqv1gZTpW0jRJt0n6Ysv+nST9SdLlko6S9KuyfxlJv5V0fTn33ZKWrvUHFxExF+bmHqGkvcpnYN/Wf/H1gdLlv/vjSBoHHAF8am7K2JOJUNK6wOeB7WxvCOwHHAP80PYGwI+Bo8rhlwOb296Yqhr+mZZTTQF2tv0uYG/gSNsbAVOBGZJeAewGbFn2zwLePUTRPm97KtWq9ltL2kDSROC7wI62twKWaTn+YOD3tjcBzgJWmscfSURELTw3m3287akt2/H9TjcDWLHl+QrAfS3PJwPrARdLugvYHDhnuA4zvdqktx1whu2HAGw/LGkL4K3l9VOo2peh+kGfJmlZYEHgry3nOcf20+XxlcDnJa0AnGn7L5JeS5Usr1U1lmYh4IEhyrVr+Qa0ALAssA7Vl5U7bffF/SnQ9y1pK+At5RrOk/TIYCcu590L4D8mrcziC710iGJERLRHm6dYuxZYQ9KqwL3AO4B39b1o+1Hg361iki4GDrA9baiT9mSNkKp6Pdzwlr7XjwaOsb0+8GFgYssxT/77YPsnwJuAp4HzJW1X4pxse6OyrWX7kAELVP1iDwBeW2qlvy6xhvozGvGfWOs3rSTBiOiUdg6fsP08sA9wPvBH4Ge2b5N0qKQ3zWsZezURXkhV+1oKQNKSwB+ovl1A1Xx5eXm8GNU3D4D3D3ZCSatR1dyOAs6hat68ENhF0kv74khaeZBTLEqVWB8t9yF3LPv/BKwmaZXyfLeW91wO7FrOvQOwxJBXHRHRYe0eR2j7XNtr2l7d9mFl3xdsnzPAsdsMVxuEHm0aLd8gDgMukTQLuIFqzMkJkj4NPAjsXg4/BDhd0r3AVcCqg5x2N+A9kmYCfwcOLU2uBwIXlJu4M4GPAXcPUKabJN0A3AbcCVxR9j8t6aPAeZIeAq5pedsXgZ9K2g24BLgfeHyefigRETUYYW/QRvVkIgSwfTJwcr/d2w1w3NnA2QPsP6Tf868AXxnguNOA00ZYpg8M8tJFttdWdaPx20DfN5xHgf+0/Xy5x7mt7WdHEisiohOyMG+0y56S3k/VWecGql6kUPUS/VmpbT4H7NlQ+SIiBjQa5hpNImyApKuBl/Tb/V7btwx0vO0jqMbG9N//F2Dj9pcwIqI9sjBvDMj2K5suQ0REJ6RpNCIieloW5o2IiJ6WptGIiOhpaRqNrrTeQst2PKYb6ju27KQlOx5z/+cW73hMgBMnPj38QTXYceOPNBL3Nzcc2/GY22z4oY7HBBinUVCtGkR6jUZERE+bPQpSYRJhRETUJk2jERHR09JrNCIielp6jUZERE/LPcKIiOhp3Z8GkwgjIqJG6SwTERE9LU2jERHR00ZDr9FxTRdgtJG0r6Q/SnpE0ueaLk9ERDebjUe8NSU1wrn3UWBH239tuiAREd2u+xtGUyOcK5KOA1YDzpH0CUnHlP0nSTpK0h8k3Slpl5b3fFrStZJulvTFYc7/C0nXSbpN0l4t+z8o6c+SLpb0vZa4q0u6qpz/UElP1HPlERHzZvZcbE1JIpwLtvcG7gO2BR7p9/KywFbAG4GvAkjaAVgD2AzYCJgi6TVDhNjD9hRgKrCvpKUkLQccBGwOvA5Yu+X4I4EjbW9ayjUoSXtJmiZp2p1P3DWSy42ImG+ei/+akkTYPr+wPdv2dOBlZd8OZbsBuJ4qia0xxDn2lXQTcBWwIi8k0UtsP2x7JnB6y/FbtDz/yVCFs3287am2p642aZW5u7KIiHn0PB7x1pTcI2yfZ1seq+X/X7H93eHeLGkbYHtgC9tPSboYmNhyroiIUSf3CON8YA9JkwAkLS/ppYMcuxjwSEmCa1M1hQJcA2wtaQlJCwBva3nPVS3P39H+4kdEzJ/0Gu1xti+Q9ArgSlULaz4BvAd4YIDDzwP2lnQzcDtVksP2vZL+B7ia6j7gdODR8p79gR9J+hTw65b9ERFdITPLjEG2VykPTyobtj/Q75hJLY+PpOrUMtx5nwV2HOTln9g+vtQIzwIuKPvvBTa3bUnvAKaN9DoiIjqhyU4wI5VEODocIml7qnuGFwC/KPunAMeoqm7+C9ijofJFRAwoNcJ4EUlLARcO8NJrbf9zoPfYPmCQ/ZcBG7axeBERbTUrNcLoryS7jZouR0REJ8x2EmFERPSw7k+DSYQREVGjLMMUERE9Lb1Goytd80TnF8547NmnOh4TYO3FVux4zHMnNPMP/4lZzzUS9+J/3NpI3G02/FDHY1580/c7HhPgK1MOaiRuO6TXaERE9LRZoyAVZoq1iIioTbuXYZL0ekm3S7pjoMXRJX1S0vSy9N2FklYe7pxJhBERURvbI96GI2k88G2qWbjWAd4paZ1+h90ATLW9AXAG8PXhzptEGBERtWnzpNubAXfYvtP2c8CpwM6tB9i+yHZfp4SrgBWGO2kSYURE1GZumkZbFxAv2179Trc8cE/L8xll32A+CPxmuDKms0xERNRmbjrL2D4eOH6IQwZan3XAqqSk9wBTga2Hi5tEGBERtRnJvb+5MANoHRO1AtXydHMoixR8Hti6rOwzpDSNRkREbdrca/RaYA1Jq0pakGpB8nNaD5C0MfBd4E22B1r79UVSI4yIiNq0c2YZ289L2gc4HxgPnGD7NkmHAtNsnwN8A5gEnF4WRP+b7TcNdd4kwi4k6RdU1f+JwJFlUd4PAp+lagb4C/Cs7X0kLQMcB6xU3r6/7SuaKHdERH/tnmvU9rnAuf32faHl8fZze84kwu60h+2HJS0EXCvp18BBwCbA48DvgZvKsUcCR9i+XNJKVN+UXtFEoSMi+mvzPcJaJBF2p30lvaU8XhF4L3CJ7YcBJJ0OrFle3x5YpzQBACwqabLtx1tPWLoh7wWw5MLLM2nikjVfQkTE6JhiLYmwy0jahiq5bWH7KUkXA7czeC1vXDn26aHO29oteeWlNuj+r2gRMSaMhoV502u0+ywGPFKS4NrA5sDCwNaSlpC0APC2luMvAPbpeyJpo46WNiJiCJ6LrSlJhN3nPGABSTcDX6KaIuhe4H+Aq4HfAdOBR8vx+wJTywSz04G9O1/kiIiBtXmKtVqkabTLlMGfO/bfL2la6T26AHAWVU0Q2w8Bu3W2lBERI5MV6qOdDimzJUykSoK/aLg8ERHDmuV0lok2sX1A02WIiJhb7RxQX5ckwoiIqE3GEUZERE/LPcKIiOhpqRFGRERPS40wIiJ6WnqNRlc6Y+LqHY/54QUf7HhMgEdmPtHxmBMmvKzjMQF2YqlG4j65zNqNxB2ngRYrr9dXphzU8ZgA/33dlxqJ2w7pNRoRET1tNMw1mkQYERG1SY0wIiJ6WmqEERHR01IjjIiInpZeoxER0dOcRBgREb0sA+ojIqKnjYYp1sb8CvWSzpW0eM0x9pe0cJ0xWmL9v5bHq0i6tRNxIyLmxWhYob5rE2FZiX2+2d7J9r/aca4h7A8MmAgljW9zrP83/CEREd1h1uzZI96aMqJEKOl9km6WdJOkUyStLOnCsu9CSSuV406SdKykiyTdKWlrSSdI+qOkk1rO94SkwyVdX96/TNl/saT/kXQJsJ+kZST9XNK1ZduyHLe1pBvLdoOkyZKWlXRp2XerpFeXY++StLSkr0n6aEsZDpH0qfL40+X8N0v64hA/h0Uk/br8HG6VtJukfYHlgIskXdRyfYdKuhrYQtIUSZdIuk7S+ZKWbbner0m6RtKfW8q8sKSflfKcJulqSVMlfRVYqFzjj0uxxkv6nqTbJF0gaaGR/E4jIjrBc/FfU4ZNhJLWBT4PbGd7Q2A/4Bjgh7Y3AH4MHNXyliWA7YBPAL8EjgDWBdaXtFE5ZhHgetubAJcAB7e8f3HbW9s+HDgSOML2psDbgO+XYw4APmZ7I+DVwNPAu4Dzy74NgRv7XcqpwG4tz3cFTpe0A7AGsBmwETBF0gwyP7kAABz4SURBVGsG+XG8HrjP9oa21wPOs30UcB+wre1tW67vVtuvBK4GjgZ2sT0FOAE4rOWcC9jejKpW2fdz+CjwSPn5fgmYAmD7c8DTtjey/e5y7BrAt22vC/yr/JxeRNJekqZJmnbWk3cNcnkREe1le8RbU0bS/LgdcIbthwBsPyxpC+Ct5fVTgK+3HP9L25Z0C/AP27cASLoNWIUqQc0GTivH/wg4s+X9p7U83h5YRy9MrruopMnAFcC3Sq3oTNszJF0LnCBpAvAL23MkQts3SHqppOWAZagSzd9KjW4H4IZy6CSq5HLpAD+LW4BvSvoa8Cvblw3yM5sF/Lw8XgtYD/htuY7xwP0tx/Zd+3Xl5wOwFdWXAGzfKunmQeIA/LXlWlvPMQfbxwPHA1y7/Fu6/+51RIwJY6XXqGDYK2l9/dny/9ktj/ueDxav9f1PtjweB2xh++l+x39V0q+BnYCrJG1v+9JSk3sDcIqkb9j+Yb/3nQHsAvwHVQ0Rquv7iu3vDn55pZD2nyVNKXG/IukC24cOcOgztme1nP8221sMctq+n9EsXvj5zM20+q0/41lAmkYjomuMlV6jFwK7SloKQNKSwB+Ad5TX3w1cPg9xdymP3zXE+y8A9ul70te0Kml127fY/howDVhb0srAA7a/B/wA2GSA851ayr0LVVIEOB/YQ9Kkcu7lJb10oMKU2uRTtn8EfLMlxuPA5EGu4XZgmVKLRtKE0tw8lMupmm6RtA6wfstrM0utNyKi6822R7w1Zdgaoe3bJB0GXCJpFlUT4r5UzZCfBh4Edp/LuE8C60q6DniUOe/dtdoX+HZpGlyAqrlyb2B/SdtS1YCmA7+hSnCfljQTeAJ43yDXMhm41/b9Zd8Fkl4BXFmaLp8A3gM8MEB51ge+IWk2MBP4SNl/PPAbSfe33Cfsi/mcpF2AoyQtVq7jf4Hbhvj5fAc4uVz3DcDN5efUF+tmSddT3buNiOhao2GKNTVRbZX0hO1JHQ88SqgacjHB9jOSVqeqla9p+7l2nL+Je4Qfnt3MwrxPzXp2+IPabMeFO7/wMcBqs5qZH+P02fcPf1ANmliYd7vxzSy63NTCvBOWXm2+f8iLLrLaiD9vHnvyzs7/UsnMMt1qYarhGBOo7hd+pF1JMCKik7IM0yC6vTZY7odeOMBLr7X9z7rj234cmFp3nIiIumUZplGqJLuNhj0wIiKGlBphRET0tNEwfCKJMCIiajN7FPQaTSKMiIjajIYaYSPDJ2J0krRXmaotccdY3F661sSN/rp2GaboSnsl7piN20vXmrgxhyTCiIjoaUmEERHR05IIY240dY8hccdmzMQd+3FHhXSWiYiInpYaYURE9LQkwoiI6GlJhBER0dOSCGNIkt4+kn0xbySNk7RrQ7ElacUmYvcaSYs0XYYYXDrLxJAkXW97k+H2jSWSlgdWpmUKQtuX1hjvUtuvqev8w8S+zvaUDsZbcoDdj9ue2akydJKkVwHfBybZXknShsCHbX+05rgvAd4GrMKcf8eH1hl3tMpcozEgSTsCOwHLSzqq5aVFgec7EP9xeNFCZo8C04BP2b6zprhfA3YDpgOzym4DtSVC4LeSDgBOA57s22n74Rpj9rlK0qa2r+1ALIDrgRWBR6gWnV4cuF/SA8Cetq+rK3BDf1NHAP8JnANg+yZJnfjSczbVtV0HPNuBeKNaEmEM5j6qD4g3Uf1j6vM48IkOxP9WKcNPqD4w3wH8B3A7cAKwTU1x3wysZbuTHx57lP9/rGWfgdU6EHtbYG9Jd1ElYQG2vUFN8c4DzrJ9PoCkHYDXAz8DvgO8sqa40NDflO17JLXumjXYsW20gu3XdyDOmJCm0RiSpAlNNFtJutr2K/vtu8r25pJusr1hTXF/A7zd9hN1nL/bSFp5oP22764p3jTbUwfaJ+lG27UtiN3E35SkM6gS8DHA5sC+wFTb72h3rH5xjweOtn1LnXHGitQIYzibSTqEF+6Z9dUY6q6tzC6dSM4oz3dpea3t394kHV3O+xRwo6QLaWlSsr1vu2O2xF4Y+CSwku29JK1BVSv9VV0x+9i+W9JWwBq2T5S0DDCpxpAPS/oscGp5vhvwiKTxQN0L13X0b6rYGzgSWB6YAVzAnDX/tpJ0C9W1LADsLulOqr/jumv6o1pqhDEkSX+iagq9jpYmHdv/rDnualQfIFtQ/cO+qpTjXmCK7cvbHO/9Q71u++R2xusX+zSqn+/7bK8naSHgyjprRy2xDwamUiXeNSUtB5xue8ua4i0NHAxsRfXhfDnwRar7WSvZvqOOuCV2R/+mmjBYDb9PXTX90S6JMIY0UHNSr5C0BLCi7ZtrjtPXNHiD7Y3Lvtqaf/vFvhHYGLi+JfbNqTm0R7+OZn0eBabZPrvGuKfYfu9w+6KSptEYzkWSvgGcyZxNhdfXGVTSROCDwLrAxJa4ewz6pvbEvZiqg9ACwI3Ag5Iusf3JGsM+V2qBLmVYnc719HvOtiX1xa51vJukXzJ4z83v2n6mxthNJKWJwNrA6eX524DbgA9K2tb2/jXFXbf1SWl67tgwmdEmiTCG01cbbO3gYGC7muOeAvyJquv5ocC7gT/WHBNgMduPSfoQcKLtgyXVWiMEDqHqTbmipB8DWwK71xyzz88kfRdYXNKeVD1Yv19jvDuBZYCflue7Af8A1gS+B9RZY2kiKb0c2M728wCSjqW6T/g6oO0dWST9N/D/gIUkPda3G3iOrEAxqDSNRlfqaybsa6aTNAE433atCbh0NtgBOBn4vO1rO9FUKGkpql6FAq6y/VCd8frFfh3VNYvqZ/zbGmO9aPKAvn2SbrO97mDvbUPs3wM7tCSlBWhJSrbXqSHm7cBmth8tzxcDrra9dmtTeA1xv2L7v+s491iUGmEMSdLLgP8BlrO9o6R1gC1s/6Dm0H1DNv4laT3g71SzZNTtUOB84PKSBFcD/lJnQEkX2n4t8OsB9tVK0kHASa3JT9JetuuqPSwjaSXbfyuxVgKWLq89V1PMPssDi1A1h1IeL2d7lqS6mqK/TtUL+WKqLxqvAf6nNEH/rqaYAKdL6j/706PA3X1fBOIFqRHGkMq4uhOpakcblm/RN9hev+a4HwJ+DmxQ4k8CvmD7uDrjdlK5D7owcBHVYO6+UdeLAr+x/YoOlOEB4CHgY7YvKvtqm0JP0k7AccD/UV3vqsBHgYupZpb53zriltgfBA4ssf6dlKiaaQ+x/ema4i5H1eT7J6rkO6POKftKzKuATYCbqa51feAmYClgb9sX1Bl/tEkijCFJutb2pv16NNY68LlJneykI2k/YH9gOaou/H2J8DHge7aPaXfMAcpwA7Az1X2zM2x/o84muxLzJVT36gT8qc4OMgPEXhbYrMS+xvZ9Ncf7ELAfsAJV56vNqYbG1N3EfyrwJdu3lefrAJ8GvgScOVb//c6rNI3GcJ4s96/6ehVuzgtNS20nacjemba/VVfsomOddGwfCRwp6eO2j64jxgjL8TdJWwPHSjodWKjmkGsAa1F90dhAErZ/WHPMPs8A95fYL5f08pprZ/sBm1Ld991W0tpU4ybrtnZfEgSwPV3Sxrbv1JzTvQVJhDG8T1JNGLy6pCuoevztMvRb5svkGs89Ei+3/XZJO9s+WdJPqO4Z1sb20eU+6DrMWQvtRHKYVmI9QzUTyceosZt9GcC/DdW1ngvsSDWovvZrHax2Rr09oJ+x/YwkJL3E9p8krVVjvD63lx6qrTP4/LnUxsfkSh/zI02jMaxyX3Atquak25uYe7RTJF1jezNJl1Ldu/o7VRNabVPKDZYcbNf5haMRpVfuhlT3mTcsnbG+b/u/OhS7r3a2UV/tzPZuNcY8i2oozP5UCfcRYILtneqKWeIuRPX32zqDz3eoasQLu0fm0h2p1AhjSGUg7k68sK7ZDqUpq9YmSklfB74MPE01xm5DYH/bP6ozLnB8mVHmQKqa8CTgoJpj7sILyWH3vuRQZ0BJP7O9q16Ym3IONQ4Xedr2bEnPS1oUeIDOrLIBDdTObL+lPDxE0kXAYlR/z7Wy/TRweNn6SxLsJ4kwhvNLqm+Rt1D/pMitdrD9GUlvoZqs+O1UvStrTYS2+xLQpQzwAS3p/TXMO/pMA8lhv/L/N9Ycp79pkhanGjx/HdWH8jUdij2jxP4F1RqQj1Aty9QRti/pVCxJW1JN1NB/gelOfekYVZIIYzgrNDTv5ITy/52An9p+uEtu8u9HNdi+na7tdHKwfX95OA64v6/nZmlSe1mNcftWZj9O0nnAoq1zuUpat7WTR5tjD1k7k7SE7UfqiN2AHzDAZPkxsCTCGM5vJO3QwLijX6pa+eJp4KOqlgfqWDf7IdSRjSdT1XgvpvpgniM51Ox04FUtz2eVfZvWHdj2XQPsPoVq/FvdsQeqnV3Yidgd8qjt3zRdiNEiiTCGcxVwlqRxVL3N+tY1W7TOoLY/J+lrwGNl5o+nqMa7AdW0YHVOBTZU0Wo454lUnRqOpmoSvbFMO3ZkDbH6W8D2v2d0sf2cpAU7EHcwTVb7u6LJoU0amSx/tEoijOEcTrV+2y3ucBfj1mYq208CT7a8/DWgiUTY9g9L27+XdAlVLWxbqsVc16VaO69uD0p6k+1zACTtTDXTTFOa7MY+lrrQNzVZ/qiURBjD+Qtwa6eT4AjU8u1d0njbQ91TuaKGmBdSTb11JXAZsKntB9odZxB7Az+WdAzVz/Qe4H0dih01sb1t02UYTZIIYzj3AxeXOUdbm1jqnuFlOHUl5jsknUG1BNP0FwW196kh5s1Ug9jXo5q151+Srixd4Gtl+/+AzSVNohpX/HjdMYdR98TbQxkzTaMNTpY/KmVAfQypDPZ+EdudmCZqUHVNDC1pMvAOqkHQ44ATgFNtPzbkG9sTe1KJewDwH7ZfUmOs99j+0WBT2tX1RUdV1993A6vZPrSsPvEftjsyhELSVsAatk8sHbAm2f5reW1J2w93ohx1a2qy/NEqNcIYUl/Ck7RIuU/XLe6q46SlRvQ94HuSXkO1MsERpZb4Jdt3tDumpH2AV1PVCu+mSr6XtTtOP30r0Q80pV2d346/QzUedTuquVwfp1plpPZequVL3VSqWZJOpBqi8yOqhZAZK0mwWNr2z1Qt1Ivt5yVlGMUgkghjSJK2oBqTNAlYSdKGwIdbxoPVFXdh4FPASrb3lLQGsJbtXwHYfmtNcccDb6Cqma1C1Vnox1SJ6lyqldTbbSHgW8B17tBacba/Wx7+zvYc9z3LYOy6vNL2JmXVC2w/0sFeqm8BNgauL7HvKy0AY1FHJ8sf7ZIIYzj/S7USwzkAtm8qNaW6nUg1GHiL8nwG1fi2X9Uc9y9UM9h8w/YfWvafUdd12/5GHecdoaN58di5gfa1y8zyZaPvA3oZOjdj0XO2Lakv9iLDvWEU6/Rk+aNaEmEMy/Y9/WZ16UQTy+q2d5P0zlKGp9WZqWU2GGxCYtv7diB+R5Sa/quoVoxvvU+4KDC+xtBHAWcBL5V0GNWH84E1xmv1M0nfBRaXtCewB1Uz+JhSxvxOBLamRybLn19JhDGceyS9CnBpwtqXmtbn6+e5Mt1X37f31Wnptdpuko5qefyi18dSEiwWpGruXoA57xM+Ro01B9s/lnQd8FqqD+g32+7E3xO2vynpdVTXuBbwhYYmZahVmbf2cNtbALVMVzfWpNdoDEnS0lQDu7en+uC6ANi37o4F5QPrQKqliS6g6tDwAdsX1xRvBvB5YAmqpXLmUMNE211B0sq27y6Px1H1oqyth2z5QjPD9rOStgE2AH5o+191xexFkr5INSznzC4cA9x1kghjSJJOplr+6JHyfAngcNt7dCD2UlSLp4pqDbnaZjyRNJ1qHcBzqGZ3mcMY61H4b6oWHt6bqrn7OqqJqL9V131LSTdS9dxchWpe1V9SdYKqdX2+EvutVDMSvZTqb6oj0wU2QdLjVD2Dn6eao3fMXms7JBHGkCTdYHvj4fbVEPctwO9tP1qeLw5sY/sXNcXbF/gI1Vyf97a+RPUBMiaXr5F0o6tFat9NNXzjs1S9V2tZcaRv/Kekz1CtTXh0J/6eSuw7gP/qVFNsjB7jmi5AdL1xpRYIVIOO6cy95YP7kiBAaTobcHB/O9g+yvYrgBNsr9ayrTpWk2AxQdIE4M3A2aVDRZ3fjmeWDlDv44UewBOGOL6d/tErSbBM2zfsvqiks0wM53DgD2VAuYFdgcM6EHegL2m1/73a/kjdMbrMd6kmJ7gJuFTSylSdSeqyO1VT7GG2/yppVWpebLnFNEmnUS3M2zpd4Jkdil87SROBhYGlyxfYvp5fiwLLNVawLpem0RhWmadwO6p/VBcONAdnDTFPAP4FfJsqAX8cWML2B+qO3eskLVDHwP4yfvBk2+9p97lHGP/EAXa7E/e7O0XSfsD+VEnvXl5IhI8B37N9TFNl62ZJhNGVymDng5izt+qXu2yatzFB0huoln2a2LfP9qE1xTqf6j5dk5Nrj3mSPm776KbLMVqkaTS6Ukl4n2u6HGOdpOOomtK2Bb5PNYawzgmw7wKukHQOLetL1rmaiaTP2P66pKMZ4P7nGBwjCvB3SZNtPy7pQKqZgr7sLMw7oCTC6Epl6q3P8OKaShYWba9X2d5A0s22vyjpcKpVzetyX9nGMfCE33X4LPB14P8YYIzoGHWQ7dPLahv/CXwTOJYXFuyNFkmE0a1+DJwGvJGqc8X7gQcbLdHY1Lfm4VOSlgP+CaxaV7CGlu/6R+kEtDsDjBEdo/qmQXwDcKztsyUd0mB5uloSYXSrpWz/QNJ+ti8BLpF0SdOFGoN+VcZofoNqVQZTNZHWoqGa/rFUg/dXA6a1Fofqesfi8Jh7y7yq2wNfk/QSMlxuUOksE11J0lW2Ny+dK46iak47w/bqDRdtzCoflhNbx2/WEOMCqpr+AbTU9G1/tq6YLbGP7ZXhMWUZs9cDt9j+i6RlgfVtX1BeX6JvtqhIIowuJemNVIvTrki1LNCiwBdtn9NowcaIMt3YoOoaWyfpOttTyj3JDcq+S2xvXUe8GFjfDD9Nl6NbpGk0ulLfArxUi4n2yn2dTvqvIV4z9XWY6VsK6P4ybOM+YIWaYsXgOrGk2aiRRBhdqdxL2pNqcuZ//52OpcHPTbK9e0OhvyxpMeBTvFDT/0RDZellaQpskUQY3epsqqbR39GZhYB7Ulnh42BgK6oPx8uBQ23/s454qelHN0oijG61cCc6UASnApcCbyvP303VmWX7OoJJWoGqJrgVMJsq8e5ne0Yd8WJQaRptke600a1+Jan2NeqCJW1/yfZfy/ZlYPEa451ItebjssDyVOsRDjQHaMwnSVtJ2r08XqZMcN7ntQ0Vqyul12h0pZaFRZ+l6mCRhUVrIOmbVGPrflZ27QKsa7uWJa/61j8cbl/MH0kHUy2AvJbtNctkCafb3rLhonWlJMLoOpIErGj7b02XZaxr+cLRdx92PC/MAdr2Lx6SfgecBPy07HonsLvt1FDaSNKNwMbA9X2LHrcOWYk5pWk0uo6rb2dnNV2OXmB7su1xtieUbVzZN9n2opLWbXPIPajWtPw7cD9VDbSpHqxj2XPl35Hh36u5xCCSCKNbXSVp06YLEZzS5vN9CXi/7WVsv5QqMR7S5hgBPytTrC0uaU+q3tffa7hMXStNo9GVJE0H1gTupmqq67tHmKadDpJ0Q1/TWl3na3eMqEh6HbAD1b+d823/tuEida0Mn4hutWPTBQig/QOvx7XOcylpSfI51HaSPkHVOSbJbwTyBxjdKk0VY9PhwB8knUH1O94VOKzZIo1JiwLnS3qYaqzoGbb/0XCZulaaRqMrSbqF6oNSVMv1rArcbrvdnTd6Vumdu4Lte4Y45irbm7c57jrAdlS/2wttT2/n+eMFkjYAdqOaMGGG7VomShjtUiOMrmR7/dbnkjYBPtxQccYk25b0C2DKEMe0NQmWc04Hkvw64wGqHrr/BF7acFm6VnqNxqhg+3ogvUjbL71zxyBJH5F0MXAhsDSwZzqaDS41wuhKkj7Z8nQcVa3lwYaKM5ZtC+wt6S7SO3csWRnY3/aNTRdkNMg9wugqkk6x/V5J/wKOKLufB+4Cfm77mcYKNwZJWnmg/bbv7nRZYv5JWtT2Y6U37ovYfrjTZRoNUiOMbjOlfDj/jWqVglYLA0mEbWT7bklbAWvYPrGsAzmp6XLFPPsJ8EbgOl7obNbHwGpNFKrbpUYYXUXSvsBHqHqJ3tf6ElWTXf4ht1EmZ45IIowuJelY2x9puhxjXSZnHpskbQncaPtJSe8BNgH+NxPZDyy9RqMrJQl2TCZnHpuOBZ6StCHwGaqpCts9b+yYkUQY0dsyOfPY9Hz5grMzcKTtI4HJDZepa6VpNKLHtUzODHBB5qcc/SRdApxHtcTVa6iGHt3Yf6KKqKTXaETcAixE1Tx6S8NlifbYDXgX8EHbf5e0EvCNhsvUtVIjjOhhkj4EfAH4PVXP3K2BQ22f0GjBYr6Ue73P2J4laU1gbeA3tmc2XLSulEQY0cMk3Q68yvY/y/OlgD/YXqvZksX8kHQd8GpgCeAqYBrwlO13N1qwLpXOMhG9bQbweMvzx4FBV6OIUUO2nwLeChxt+y1AVm4ZRO4RRvS2e4GrJZ1NdY9wZ+CavrlebX+rycLFPJOkLYB3Ax8s+8Y3WJ6ulkQY0dv+r2x9zi7/T1f70W1/4L+Bs2zfJmk14KKGy9S1co8wImKMkrSI7SebLke3S40woodJuogyq0wr29s1UJxok9Is+gOqCdRXKjPMfNj2R5stWXdKIozobQe0PJ4IvI1q2asY3f4X+E/gHADbN0l6TbNF6l5JhBE9zPZ1/XZdUWYliVHO9j1S6ypMzGqqLN0uiTCih/VbwHUc1ZJM/9FQcaJ97pH0KsCSFgT2Bf7YcJm6VhJhRG9rXcB1JnAXL3S3j9Frb+BIYHmqsaIXAB9rtERdLIkword9FjjP9mOSDqJat+6phssU80HSeOC9mUVm5DKzTERvO7Akwa2A1wEnUa1lF6OU7VlUEyPECCURRvS2vg4UbwCOs302sGCD5Yn2uELSMZJeLWmTvq3pQnWrDKiP6GGSfkU1zdr2wBTgaeAa2xs2WrCYL2V8aH/O+NCBJRFG9DBJCwOvB26x/RdJywLr276g4aJFdEwSYUTEGFOW0zoY2IqqV/DlVOtM/rPRgnWp3COMiBh7TgUepJopaJfy+LRGS9TFUiOMiBhjJF1ne0q/fdNsT22qTN0sNcKIiLHnIknvkDSubLsCv266UN0qNcKIiDFG0uPAIsDssmsc0Lcck20v2kjBulQSYURE9LRMsRYRMQZJ2gBYhZbPedtnNlagLpZEGBExxkg6AdgAuI0XmkcNJBEOIE2jERFjjKTpttdpuhyjRXqNRkSMPVdKSiIcodQIIyLGGEmvAX4J/B14lmq9SdveoNGCdakkwoiIMUbSHcAngVt44R4htu9urFBdLJ1lIiLGnr/ZPqfpQowWqRFGRIwxkr4DLE7VPPps3/4MnxhYaoQREWPPQlQJcIeWfRk+MYjUCCMioqdl+ERExBgjaQVJZ0l6QNI/JP1c0gpNl6tbJRFGRIw9JwLnAMsBy1PdKzyx0RJ1sTSNRkSMMZJutL3RcPuikhphRMTY85Ck90gaX7b3AP9sulDdKjXCiIgxRtJKwDHAFlS9Rf8A7Gv7b40WrEslEUZEjDGSTgb2t/1Ieb4k8E3bezRbsu6UptGIiLFng74kCGD7YWDjBsvT1ZIIIyLGnnGSluh7UmqEmUBlEPnBRESMPYcDf5B0BtU9wl2Bw5otUvfKPcKIiDGorEe4HdUSTBfant5wkbpWEmFERPS03COMiIielkQYERE9LYkwIiJ6WhJhRET0tCTCiIjoaf8f7loF43Nd+gMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.heatmap(df_concrete.corr().abs())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Little correlation of ~0.6 between Superplasticizer and Water (which is negative as evident from scatter matrix).  Let's remove \"super_plasticizer\" from dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rather than splitting training data further to validation set, we will perform cross validation in all our training models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X= df_concrete.drop(['compressive_strength','super_plasticizer'], axis=1) \n",
    "y= df_concrete['compressive_strength']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = .2, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cement</th>\n",
       "      <th>furnace_slag</th>\n",
       "      <th>fly_ash</th>\n",
       "      <th>water</th>\n",
       "      <th>coarse_agg</th>\n",
       "      <th>fine_agg</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.024580</td>\n",
       "      <td>-0.836936</td>\n",
       "      <td>-0.849283</td>\n",
       "      <td>0.452843</td>\n",
       "      <td>-0.071972</td>\n",
       "      <td>1.010350</td>\n",
       "      <td>-0.274315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.760705</td>\n",
       "      <td>-0.614363</td>\n",
       "      <td>1.345880</td>\n",
       "      <td>-1.313732</td>\n",
       "      <td>-0.408315</td>\n",
       "      <td>0.307891</td>\n",
       "      <td>-0.687211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.064802</td>\n",
       "      <td>1.293906</td>\n",
       "      <td>-0.849283</td>\n",
       "      <td>0.199793</td>\n",
       "      <td>0.502398</td>\n",
       "      <td>-0.799438</td>\n",
       "      <td>-0.274315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.827741</td>\n",
       "      <td>1.353650</td>\n",
       "      <td>-0.849283</td>\n",
       "      <td>0.452843</td>\n",
       "      <td>-1.934796</td>\n",
       "      <td>-0.254075</td>\n",
       "      <td>-0.274315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.641604</td>\n",
       "      <td>-0.836936</td>\n",
       "      <td>0.713642</td>\n",
       "      <td>-1.058295</td>\n",
       "      <td>0.391146</td>\n",
       "      <td>1.620722</td>\n",
       "      <td>-0.687211</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     cement  furnace_slag   fly_ash     water  coarse_agg  fine_agg       age\n",
       "0 -0.024580     -0.836936 -0.849283  0.452843   -0.071972  1.010350 -0.274315\n",
       "1  0.760705     -0.614363  1.345880 -1.313732   -0.408315  0.307891 -0.687211\n",
       "2 -0.064802      1.293906 -0.849283  0.199793    0.502398 -0.799438 -0.274315\n",
       "3  0.827741      1.353650 -0.849283  0.452843   -1.934796 -0.254075 -0.274315\n",
       "4 -0.641604     -0.836936  0.713642 -1.058295    0.391146  1.620722 -0.687211"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Scaling the features\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler().fit(x_train)\n",
    "x_train_scaled = pd.DataFrame(scaler.transform(x_train),\n",
    "                              columns = x_train.columns)\n",
    "x_train_scaled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cement</th>\n",
       "      <th>furnace_slag</th>\n",
       "      <th>fly_ash</th>\n",
       "      <th>water</th>\n",
       "      <th>coarse_agg</th>\n",
       "      <th>fine_agg</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.168485</td>\n",
       "      <td>-0.836936</td>\n",
       "      <td>0.980643</td>\n",
       "      <td>-0.320630</td>\n",
       "      <td>0.636936</td>\n",
       "      <td>-0.299416</td>\n",
       "      <td>-0.687211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.397591</td>\n",
       "      <td>-0.836936</td>\n",
       "      <td>0.615872</td>\n",
       "      <td>0.245629</td>\n",
       "      <td>-0.305990</td>\n",
       "      <td>0.895530</td>\n",
       "      <td>-0.687211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.336581</td>\n",
       "      <td>0.721074</td>\n",
       "      <td>0.754276</td>\n",
       "      <td>0.882551</td>\n",
       "      <td>-0.744659</td>\n",
       "      <td>-0.305163</td>\n",
       "      <td>-0.274315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.728464</td>\n",
       "      <td>0.752703</td>\n",
       "      <td>-0.849283</td>\n",
       "      <td>0.199793</td>\n",
       "      <td>1.327733</td>\n",
       "      <td>-0.224700</td>\n",
       "      <td>-0.621148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.125895</td>\n",
       "      <td>1.587937</td>\n",
       "      <td>1.828505</td>\n",
       "      <td>1.646475</td>\n",
       "      <td>-1.960668</td>\n",
       "      <td>-1.773940</td>\n",
       "      <td>-0.274315</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     cement  furnace_slag   fly_ash     water  coarse_agg  fine_agg       age\n",
       "0  0.168485     -0.836936  0.980643 -0.320630    0.636936 -0.299416 -0.687211\n",
       "1 -0.397591     -0.836936  0.615872  0.245629   -0.305990  0.895530 -0.687211\n",
       "2 -1.336581      0.721074  0.754276  0.882551   -0.744659 -0.305163 -0.274315\n",
       "3 -0.728464      0.752703 -0.849283  0.199793    1.327733 -0.224700 -0.621148\n",
       "4 -1.125895      1.587937  1.828505  1.646475   -1.960668 -1.773940 -0.274315"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test_scaled = pd.DataFrame(scaler.transform(x_test),\n",
    "                              columns = x_test.columns)\n",
    "x_test_scaled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We will save the model performance metrics in a DataFrame\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.model_selection import KFold, cross_val_score\n",
    "import numpy as np\n",
    "Model = []\n",
    "RMSE = []\n",
    "R_sq = []\n",
    "cv = KFold(5, random_state = 1)\n",
    "\n",
    "#Creating a Function to append the cross validation scores of the algorithms\n",
    "def input_scores(name, model, x, y):\n",
    "    Model.append(name)\n",
    "    RMSE.append(np.sqrt((-1) * cross_val_score(model, x, y, cv=cv, \n",
    "                                               scoring='neg_mean_squared_error').mean()))\n",
    "    R_sq.append(cross_val_score(model, x, y, cv=cv, scoring='r2').mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import (RandomForestRegressor, GradientBoostingRegressor, \n",
    "                              AdaBoostRegressor)\n",
    "\n",
    "names = ['Linear Regression', 'Ridge Regression', 'Lasso Regression',\n",
    "         'K Neighbors Regressor', 'Decision Tree Regressor', \n",
    "         'Random Forest Regressor', 'Gradient Boosting Regressor',\n",
    "         'Adaboost Regressor']\n",
    "models = [LinearRegression(), Ridge(), Lasso(),\n",
    "          KNeighborsRegressor(), DecisionTreeRegressor(),\n",
    "          RandomForestRegressor(), GradientBoostingRegressor(), \n",
    "          AdaBoostRegressor()]\n",
    "\n",
    "#Running all algorithms\n",
    "for name, model in zip(names, models):\n",
    "    input_scores(name, model, x_train_scaled, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOLLOWING ARE THE TRAINING SCORES: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>R Squared</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>10.340493</td>\n",
       "      <td>0.620461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ridge Regression</td>\n",
       "      <td>10.339082</td>\n",
       "      <td>0.620511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lasso Regression</td>\n",
       "      <td>10.870840</td>\n",
       "      <td>0.580307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>K Neighbors Regressor</td>\n",
       "      <td>9.029101</td>\n",
       "      <td>0.709607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree Regressor</td>\n",
       "      <td>7.187398</td>\n",
       "      <td>0.825132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>5.672585</td>\n",
       "      <td>0.888144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gradient Boosting Regressor</td>\n",
       "      <td>5.201671</td>\n",
       "      <td>0.903235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Adaboost Regressor</td>\n",
       "      <td>7.777754</td>\n",
       "      <td>0.780076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Model       RMSE  R Squared\n",
       "0            Linear Regression  10.340493   0.620461\n",
       "1             Ridge Regression  10.339082   0.620511\n",
       "2             Lasso Regression  10.870840   0.580307\n",
       "3        K Neighbors Regressor   9.029101   0.709607\n",
       "4      Decision Tree Regressor   7.187398   0.825132\n",
       "5      Random Forest Regressor   5.672585   0.888144\n",
       "6  Gradient Boosting Regressor   5.201671   0.903235\n",
       "7           Adaboost Regressor   7.777754   0.780076"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluation = pd.DataFrame({'Model': Model,\n",
    "                           'RMSE': RMSE,\n",
    "                           'R Squared': R_sq})\n",
    "print(\"FOLLOWING ARE THE TRAINING SCORES: \")\n",
    "evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gradient Boosting Regressor has the lowest RMSE, highest R-Squared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "                          learning_rate=0.1, loss='ls', max_depth=3,\n",
       "                          max_features=None, max_leaf_nodes=None,\n",
       "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                          min_samples_leaf=1, min_samples_split=2,\n",
       "                          min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                          n_iter_no_change=None, presort='auto',\n",
       "                          random_state=None, subsample=1.0, tol=0.0001,\n",
       "                          validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tuning this base model\n",
    "GradientBoostingRegressor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "                          learning_rate=0.2, loss='ls', max_depth=10,\n",
      "                          max_features=None, max_leaf_nodes=None,\n",
      "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                          min_samples_leaf=1, min_samples_split=100,\n",
      "                          min_weight_fraction_leaf=0.0, n_estimators=200,\n",
      "                          n_iter_no_change=None, presort='auto', random_state=1,\n",
      "                          subsample=1.0, tol=0.0001, validation_fraction=0.1,\n",
      "                          verbose=0, warm_start=False)\n",
      "R Squared: 0.9312383384864246\n"
     ]
    }
   ],
   "source": [
    "#tuning for number of trees\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "param_grid = {'n_estimators':range(20,1001,10),\n",
    "              'max_depth':[10], #range(5,16,2), \n",
    "              'min_samples_split':[100], #range(200,1001,200), \n",
    "              'learning_rate':[0.2]}\n",
    "clf = GridSearchCV(GradientBoostingRegressor(random_state=1), \n",
    "                   param_grid = param_grid, scoring='r2', \n",
    "                   cv=cv).fit(x_train_scaled, y_train)\n",
    "print(clf.best_estimator_) \n",
    "print(\"R Squared:\",clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "                          learning_rate=0.2, loss='ls', max_depth=16,\n",
      "                          max_features=None, max_leaf_nodes=None,\n",
      "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                          min_samples_leaf=1, min_samples_split=180,\n",
      "                          min_weight_fraction_leaf=0.0, n_estimators=230,\n",
      "                          n_iter_no_change=None, presort='auto', random_state=1,\n",
      "                          subsample=1.0, tol=0.0001, validation_fraction=0.1,\n",
      "                          verbose=0, warm_start=False)\n",
      "R Squared: 0.9368563421781125\n"
     ]
    }
   ],
   "source": [
    "#tuning the tree specific parameters\n",
    "param_grid = {'n_estimators': [230],\n",
    "              'max_depth': range(10,31,2), \n",
    "              'min_samples_split': range(50,501,10), \n",
    "              'learning_rate':[0.2]}\n",
    "clf = GridSearchCV(GradientBoostingRegressor(random_state=1), \n",
    "                   param_grid = param_grid, scoring='r2', \n",
    "                   cv=cv).fit(x_train_scaled, y_train)\n",
    "print(clf.best_estimator_) \n",
    "print(\"R Squared:\",clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R Squared Train average score: 0.9362243763227699\n",
      "R Squared Test score:  0.92318096500655\n"
     ]
    }
   ],
   "source": [
    "#now increasing number of trees and decreasing learning rate proportionally\n",
    "clf = GradientBoostingRegressor(random_state=1, max_depth=20, \n",
    "                                min_samples_split=170, n_estimators=230*2, \n",
    "                                learning_rate=0.2/2)\n",
    "print(\"R Squared Train average score:\",cross_val_score(clf, x_train_scaled, y_train, cv=cv, scoring='r2').mean())\n",
    "\n",
    "clf.fit(x_train_scaled, y_train)\n",
    "print(\"R Squared Test score: \", clf.score(x_test_scaled, y_test.values.reshape(-1,1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Since score improved, the best model is GradientBoostingRegressor with learning_rate= 0.2/2, max_depth= 20, min_samples_split= 170, n_estimators= 230*2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding model to pickle file and reading model from pickle again for finding score of test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 92.32 %\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "pkl_filename = 'concrete_model_analysis.data'\n",
    "with open(pkl_filename, 'wb') as file:\n",
    "    pickle.dump(clf, file)\n",
    "\n",
    "with open(pkl_filename, 'rb') as file:\n",
    "    pickle_model = pickle.load(file)\n",
    "\n",
    "score = pickle_model.score(x_test_scaled, y_test.values.reshape(-1,1))\n",
    "print('Test score: {0:.2f} %'.format(100 * score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}