regressions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a34bcc78",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0a900d98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import yearly dataframes\n",
    "df_2017 = pd.read_csv('data\\IPEDS_2017.csv')\n",
    "df_2018 = pd.read_csv('data\\IPEDS_2018.csv')\n",
    "df_2019 = pd.read_csv('data\\IPEDS_2019.csv')\n",
    "\n",
    "# list of dataframes\n",
    "df_years_list = [df_2017, df_2018, df_2019]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4c6ecca4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_IPEDS_regression(df):\n",
    "    '''\n",
    "    Runs a regression for a given dataframe\n",
    "    \n",
    "    Args:\n",
    "        df(pd.DataFrame): dataframe\n",
    "    \n",
    "    Returns:\n",
    "        x: array of values\n",
    "        y: array of values\n",
    "        y_pred: array of predicted values\n",
    "    '''\n",
    "\n",
    "    x_feat_list = list([df.columns[7]])\n",
    "    y_feat = df.columns[50]\n",
    "\n",
    "    # initialize regression object\n",
    "    reg = LinearRegression()\n",
    "\n",
    "    # get target variable\n",
    "    # (note: since we index with list -> garauanteed 2d x array, no reshape needed)\n",
    "    x = df.loc[:, x_feat_list].values\n",
    "    y = df.loc[:, y_feat].values\n",
    "\n",
    "    # fit regression\n",
    "    reg.fit(x, y)\n",
    "\n",
    "    # compute / store r2\n",
    "    y_pred = reg.predict(x)\n",
    "    \n",
    "    return x,y,y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3ab19a6d",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 50 is out of bounds for axis 0 with size 45",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn [4], line 7\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m df \u001b[38;5;129;01min\u001b[39;00m df_years_list:\n\u001b[0;32m      5\u001b[0m     year \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m----> 7\u001b[0m     x,y,y_pred \u001b[38;5;241m=\u001b[39m \u001b[43mfit_IPEDS_regression\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      9\u001b[0m     \u001b[38;5;66;03m# compute mean squared error\u001b[39;00m\n\u001b[0;32m     10\u001b[0m     mse \u001b[38;5;241m=\u001b[39m ((y_pred \u001b[38;5;241m-\u001b[39m y) \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m \u001b[38;5;241m2\u001b[39m)\u001b[38;5;241m.\u001b[39mmean()\n",
      "Cell \u001b[1;32mIn [3], line 15\u001b[0m, in \u001b[0;36mfit_IPEDS_regression\u001b[1;34m(df)\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;124;03mRuns a regression for a given dataframe\u001b[39;00m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;124;03m\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[38;5;124;03m    y_pred: array of predicted values\u001b[39;00m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[0;32m     14\u001b[0m x_feat_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m([df\u001b[38;5;241m.\u001b[39mcolumns[\u001b[38;5;241m7\u001b[39m]])\n\u001b[1;32m---> 15\u001b[0m y_feat \u001b[38;5;241m=\u001b[39m \u001b[43mdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m50\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;66;03m# initialize regression object\u001b[39;00m\n\u001b[0;32m     18\u001b[0m reg \u001b[38;5;241m=\u001b[39m LinearRegression()\n",
      "File \u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py:5358\u001b[0m, in \u001b[0;36mIndex.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   5355\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_integer(key) \u001b[38;5;129;01mor\u001b[39;00m is_float(key):\n\u001b[0;32m   5356\u001b[0m     \u001b[38;5;66;03m# GH#44051 exclude bool, which would return a 2d ndarray\u001b[39;00m\n\u001b[0;32m   5357\u001b[0m     key \u001b[38;5;241m=\u001b[39m com\u001b[38;5;241m.\u001b[39mcast_scalar_indexer(key, warn_float\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m-> 5358\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mgetitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   5360\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(key, \u001b[38;5;28mslice\u001b[39m):\n\u001b[0;32m   5361\u001b[0m     \u001b[38;5;66;03m# This case is separated from the conditional above to avoid\u001b[39;00m\n\u001b[0;32m   5362\u001b[0m     \u001b[38;5;66;03m# pessimization com.is_bool_indexer and ndim checks.\u001b[39;00m\n\u001b[0;32m   5363\u001b[0m     result \u001b[38;5;241m=\u001b[39m getitem(key)\n",
      "\u001b[1;31mIndexError\u001b[0m: index 50 is out of bounds for axis 0 with size 45"
     ]
    }
   ],
   "source": [
    "year = 2016\n",
    "\n",
    "for df in df_years_list:\n",
    "    \n",
    "    year += 1\n",
    "    \n",
    "    x,y,y_pred = fit_IPEDS_regression(df)\n",
    "    \n",
    "    # compute mean squared error\n",
    "    mse = ((y_pred - y) ** 2).mean()\n",
    "    r2 = r2_score(y_true=y, y_pred=y_pred)\n",
    "\n",
    "    # plot it\n",
    "    plt.scatter(x, y, label='observed', color='b')\n",
    "    plt.plot(x, y_pred, label='model (regression)', color='black')\n",
    "    plt.gca().set_title(f'dataset = {year}, mse={mse:.3f}, r^2={r2:.3f}')\n",
    "    plt.legend()\n",
    "    plt.xlabel('Percentage of Pell Grant Recipients')\n",
    "    plt.ylabel('75th Percentile Composite ACT')\n",
    "    plt.gcf().set_size_inches(15, 5)\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}