Chi SQ.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import stats as spt\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.formula.api import ols\n",
    "from scipy import stats as sps\n",
    "import seaborn as sns\n",
    "import statsmodels.stats.proportion as ssp\n",
    "import geopandas as gpd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = sns.load_dataset('tips')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total_bill</th>\n",
       "      <th>tip</th>\n",
       "      <th>sex</th>\n",
       "      <th>smoker</th>\n",
       "      <th>day</th>\n",
       "      <th>time</th>\n",
       "      <th>size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>12.03</td>\n",
       "      <td>1.50</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Fri</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>27.18</td>\n",
       "      <td>2.00</td>\n",
       "      <td>Female</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Sat</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>22.23</td>\n",
       "      <td>5.00</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>16.43</td>\n",
       "      <td>2.30</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>Thur</td>\n",
       "      <td>Lunch</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>20.08</td>\n",
       "      <td>3.15</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>Sat</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>11.24</td>\n",
       "      <td>1.76</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Sat</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>10.65</td>\n",
       "      <td>1.50</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>Thur</td>\n",
       "      <td>Lunch</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>25.56</td>\n",
       "      <td>4.34</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>Sun</td>\n",
       "      <td>Dinner</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     total_bill   tip     sex smoker   day    time  size\n",
       "97        12.03  1.50    Male    Yes   Fri  Dinner     2\n",
       "240       27.18  2.00  Female    Yes   Sat  Dinner     2\n",
       "46        22.23  5.00    Male     No   Sun  Dinner     2\n",
       "144       16.43  2.30  Female     No  Thur   Lunch     2\n",
       "65        20.08  3.15    Male     No   Sat  Dinner     3\n",
       "58        11.24  1.76    Male    Yes   Sat  Dinner     2\n",
       "117       10.65  1.50  Female     No  Thur   Lunch     2\n",
       "54        25.56  4.34    Male     No   Sun  Dinner     4"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>time</th>\n",
       "      <th>Lunch</th>\n",
       "      <th>Dinner</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Male</th>\n",
       "      <td>33</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Female</th>\n",
       "      <td>35</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "time    Lunch   Dinner\n",
       "sex                   \n",
       "Male        33     124\n",
       "Female      35      52"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "var1='sex'\n",
    "var2='time'\n",
    "# crosstab creates a summary table that represents the frequencies (observed) values of each category.\n",
    "# dfCrossTab = pd.crosstab(df['time'],df['smoker'])\n",
    "dfCrossTab = pd.crosstab(df[var1],df[var2])\n",
    "\n",
    "dfCrossTab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 33 124]\n",
      " [ 35  52]]\n"
     ]
    }
   ],
   "source": [
    "# you can see the observed values as an array\n",
    "observed_values = dfCrossTab.values\n",
    "print(observed_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The SciPy way! for 2X2 contingency tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[1;31mSignature:\u001b[0m \u001b[0msps\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchi2_contingency\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobserved\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcorrection\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlambda_\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
       "\u001b[1;31mDocstring:\u001b[0m\n",
       "Chi-square test of independence of variables in a contingency table.\n",
       "\n",
       "This function computes the chi-square statistic and p-value for the\n",
       "hypothesis test of independence of the observed frequencies in the\n",
       "contingency table [1]_ `observed`.  The expected frequencies are computed\n",
       "based on the marginal sums under the assumption of independence; see\n",
       "`scipy.stats.contingency.expected_freq`.  The number of degrees of\n",
       "freedom is (expressed using numpy functions and attributes)::\n",
       "\n",
       "    dof = observed.size - sum(observed.shape) + observed.ndim - 1\n",
       "\n",
       "\n",
       "Parameters\n",
       "----------\n",
       "observed : array_like\n",
       "    The contingency table. The table contains the observed frequencies\n",
       "    (i.e. number of occurrences) in each category.  In the two-dimensional\n",
       "    case, the table is often described as an \"R x C table\".\n",
       "correction : bool, optional\n",
       "    If True, *and* the degrees of freedom is 1, apply Yates' correction\n",
       "    for continuity.  The effect of the correction is to adjust each\n",
       "    observed value by 0.5 towards the corresponding expected value.\n",
       "lambda_ : float or str, optional.\n",
       "    By default, the statistic computed in this test is Pearson's\n",
       "    chi-squared statistic [2]_.  `lambda_` allows a statistic from the\n",
       "    Cressie-Read power divergence family [3]_ to be used instead.  See\n",
       "    `power_divergence` for details.\n",
       "\n",
       "Returns\n",
       "-------\n",
       "chi2 : float\n",
       "    The test statistic.\n",
       "p : float\n",
       "    The p-value of the test\n",
       "dof : int\n",
       "    Degrees of freedom\n",
       "expected : ndarray, same shape as `observed`\n",
       "    The expected frequencies, based on the marginal sums of the table.\n",
       "\n",
       "See Also\n",
       "--------\n",
       "contingency.expected_freq\n",
       "fisher_exact\n",
       "chisquare\n",
       "power_divergence\n",
       "\n",
       "Notes\n",
       "-----\n",
       "An often quoted guideline for the validity of this calculation is that\n",
       "the test should be used only if the observed and expected frequencies\n",
       "in each cell are at least 5.\n",
       "\n",
       "This is a test for the independence of different categories of a\n",
       "population. The test is only meaningful when the dimension of\n",
       "`observed` is two or more.  Applying the test to a one-dimensional\n",
       "table will always result in `expected` equal to `observed` and a\n",
       "chi-square statistic equal to 0.\n",
       "\n",
       "This function does not handle masked arrays, because the calculation\n",
       "does not make sense with missing values.\n",
       "\n",
       "Like stats.chisquare, this function computes a chi-square statistic;\n",
       "the convenience this function provides is to figure out the expected\n",
       "frequencies and degrees of freedom from the given contingency table.\n",
       "If these were already known, and if the Yates' correction was not\n",
       "required, one could use stats.chisquare.  That is, if one calls::\n",
       "\n",
       "    chi2, p, dof, ex = chi2_contingency(obs, correction=False)\n",
       "\n",
       "then the following is true::\n",
       "\n",
       "    (chi2, p) == stats.chisquare(obs.ravel(), f_exp=ex.ravel(),\n",
       "                                 ddof=obs.size - 1 - dof)\n",
       "\n",
       "The `lambda_` argument was added in version 0.13.0 of scipy.\n",
       "\n",
       "References\n",
       "----------\n",
       ".. [1] \"Contingency table\",\n",
       "       https://en.wikipedia.org/wiki/Contingency_table\n",
       ".. [2] \"Pearson's chi-squared test\",\n",
       "       https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test\n",
       ".. [3] Cressie, N. and Read, T. R. C., \"Multinomial Goodness-of-Fit\n",
       "       Tests\", J. Royal Stat. Soc. Series B, Vol. 46, No. 3 (1984),\n",
       "       pp. 440-464.\n",
       "\n",
       "Examples\n",
       "--------\n",
       "A two-way example (2 x 3):\n",
       "\n",
       ">>> from scipy.stats import chi2_contingency\n",
       ">>> obs = np.array([[10, 10, 20], [20, 20, 20]])\n",
       ">>> chi2_contingency(obs)\n",
       "(2.7777777777777777,\n",
       " 0.24935220877729619,\n",
       " 2,\n",
       " array([[ 12.,  12.,  16.],\n",
       "        [ 18.,  18.,  24.]]))\n",
       "\n",
       "Perform the test using the log-likelihood ratio (i.e. the \"G-test\")\n",
       "instead of Pearson's chi-squared statistic.\n",
       "\n",
       ">>> g, p, dof, expctd = chi2_contingency(obs, lambda_=\"log-likelihood\")\n",
       ">>> g, p\n",
       "(2.7688587616781319, 0.25046668010954165)\n",
       "\n",
       "A four-way example (2 x 2 x 2 x 2):\n",
       "\n",
       ">>> obs = np.array(\n",
       "...     [[[[12, 17],\n",
       "...        [11, 16]],\n",
       "...       [[11, 12],\n",
       "...        [15, 16]]],\n",
       "...      [[[23, 15],\n",
       "...        [30, 22]],\n",
       "...       [[14, 17],\n",
       "...        [15, 16]]]])\n",
       ">>> chi2_contingency(obs)\n",
       "(8.7584514426741897,\n",
       " 0.64417725029295503,\n",
       " 11,\n",
       " array([[[[ 14.15462386,  14.15462386],\n",
       "          [ 16.49423111,  16.49423111]],\n",
       "         [[ 11.2461395 ,  11.2461395 ],\n",
       "          [ 13.10500554,  13.10500554]]],\n",
       "        [[[ 19.5591166 ,  19.5591166 ],\n",
       "          [ 22.79202844,  22.79202844]],\n",
       "         [[ 15.54012004,  15.54012004],\n",
       "          [ 18.10873492,  18.10873492]]]]))\n",
       "\u001b[1;31mFile:\u001b[0m      c:\\users\\mehdi\\.conda\\envs\\urbandataanalysis\\lib\\site-packages\\scipy\\stats\\contingency.py\n",
       "\u001b[1;31mType:\u001b[0m      function\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# see how the fucntion works\n",
    "sps.chi2_contingency?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "what is correction?\n",
    "The Yates correction is made to account for the fact that both Pearson’s chi-square test and McNemar’s chi-square test are biased upwards for a 2 x 2 contingency table. An upwards bias tends to make results larger than they should be. If you are creating a 2 x 2 contingency table that uses either of these two tests, the Yates correction is usually recommended, especially if the expected cell frequencies are below 10 (some authors put that figure at 5)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9.343808982970623, 0.002237400118075248, 1, array([[ 43.75409836, 113.24590164],\n",
      "       [ 24.24590164,  62.75409836]]))\n"
     ]
    }
   ],
   "source": [
    "# Returns\n",
    "# -------\n",
    "# [0]chi2 : float\n",
    "#     The test statistic.\n",
    "# [1]p : float\n",
    "#     The p-value of the test\n",
    "# [2]dof : int\n",
    "#     Degrees of freedom ((#rows -1)*(#cols-1))\n",
    "# [3]expected : ndarray, same shape as `observed`\n",
    "#     The expected frequencies, based on the marginal sums of the table.\n",
    "\n",
    "results = sps.chi2_contingency(dfCrossTab)\n",
    "print (results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P-Value is:  0.002237400118075248\n"
     ]
    }
   ],
   "source": [
    "pValue=results[1]\n",
    "print ('P-Value is: ',pValue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "degree of freedom is:  1\n"
     ]
    }
   ],
   "source": [
    "# degree of freedom\n",
    "print ('degree of freedom is: ', results[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[59 97]\n",
      " [33 53]]\n"
     ]
    }
   ],
   "source": [
    "# get the expected values from the results list\n",
    "# print(results[3])\n",
    "print(results[3].astype('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[60 97]\n",
      " [33 54]]\n"
     ]
    }
   ],
   "source": [
    "# you can see the observed values as an array\n",
    "observed_values = dfCrossTab.values\n",
    "print(observed_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a Chi-square test, a p-value that is less than or equal to your significance level indicates there is sufficient evidence to conclude that the observed distribution is not the same as the expected distribution. You can conclude that a relationship exists between the categorical variables.\n",
    "\n",
    "For example, if the p-value is 0.01 it means that you are taking 1% risk to say that difference between observed and expected values is a real difference and represents the population. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is basically the risk level that you can set for yourself 0.01 is 1%\n",
    "significanceLevel = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reject the null hypothesis; there is a relationship between two categorical variables\n"
     ]
    }
   ],
   "source": [
    "if pValue<=significanceLevel:\n",
    "    print('Reject the null hypothesis; there is a relationship between two categorical variables')\n",
    "else:\n",
    "    print('Accept the null hypothesis; there is no relationship betwee two categorical variables')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}