2. METIS_Krystkowiak_Krystian_Project_2_Regression - cleaning&preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e0e75c6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "import patsy\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import RidgeCV\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c77f0de5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import of IMDB data scraped in previous notebook\n",
    "df = pd.read_pickle(\"./shows.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5885b2bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['link', 'imdb_rating', 'title', 's01_episodes', 'avg_runtime', 'genres',\n",
       "       'rel_date', 'certification', 'origin', 'company', 'creators',\n",
       "       'creators_a', 'stars', 'stars_a'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "5b816c39",
   "metadata": {},
   "source": [
    "### data cleaning, processing, and initial exploration"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3c9bb8af",
   "metadata": {},
   "source": [
    "GENRES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "20ff705b",
   "metadata": {},
   "outputs": [],
   "source": [
    "genres_l = set(df.genres.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "661ff520",
   "metadata": {},
   "outputs": [],
   "source": [
    "def m(x):\n",
    "    if i in x:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "for i in genres_l:\n",
    "    df[i] = df.genres.apply(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "67a7fbfd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "imdb_rating    1.000000\n",
       "Thriller       0.034059\n",
       "Music          0.014045\n",
       "Comedy         0.022244\n",
       "Biography      0.067982\n",
       "Adventure     -0.001700\n",
       "Horror        -0.100276\n",
       "Documentary    0.027008\n",
       "Western        0.036265\n",
       "Sci-Fi        -0.105571\n",
       "Romance        0.045513\n",
       "Short         -0.053239\n",
       "Mystery       -0.006094\n",
       "Action         0.000980\n",
       "Family        -0.048417\n",
       "Game-Show     -0.053298\n",
       "News           0.069006\n",
       "Talk-Show     -0.009340\n",
       "War            0.014827\n",
       "Crime          0.038813\n",
       "History        0.051376\n",
       "Drama          0.033611\n",
       "Reality-TV    -0.082501\n",
       "Animation      0.038486\n",
       "Musical       -0.054243\n",
       "Sport          0.071994\n",
       "Fantasy        0.004190\n",
       "Name: imdb_rating, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['imdb_rating'] + list(genres_l)].corr().iloc[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "61cf8f94",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.rename(columns={\"Talk-Show\": \"Talk_Show\", \"Sci-Fi\": \"Sci_Fi\", \"Game-Show\": \"Game_Show\", \"Reality-TV\":\"Reality_TV\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cb942f72",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['genres_n'] = (df.genres.apply(lambda x: len(x)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b773b118",
   "metadata": {},
   "source": [
    "RELEASE DATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fb423a73",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a column for the release month of the TV show\n",
    "df['rel_month']= pd.DatetimeIndex(df['rel_date']).month "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32ca51cc",
   "metadata": {},
   "source": [
    "CERTIFICATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9cb1d1b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        456\n",
       "TV-MA    446\n",
       "TV-14    314\n",
       "TV-PG    139\n",
       "Name: certification, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#after exploration looks like over 457 of the shows have no certification info, still will keep that column to anlyse shows that have certification info\n",
    "df.loc[df.certification.apply(len) > 7, 'certification'] = \"0\"\n",
    "df.loc[df.certification == '', 'certification'] = \"0\"\n",
    "\n",
    "#Unification of the certification categories\n",
    "df.certification.replace({'TV-Y7': 'TV-PG', 'TV-G': 'TV-PG', 'TV-Y': 'TV-PG', 'PG': 'TV-PG', 'PG-13': 'TV-14'}, inplace=True)\n",
    "\n",
    "df.certification.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a1acc63c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df['TV_PG'] = (df['certification'] == 'TV-PG').apply(int)\n",
    "df['TV_14'] = (df['certification'] == 'TV-14').apply(int)\n",
    "df['TV_MA'] = (df['certification'] == 'TV-MA').apply(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "735b12aa",
   "metadata": {},
   "source": [
    "ORIGIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4dee95a9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "United States             517\n",
       "India                     158\n",
       "United Kingdom            133\n",
       "Japan                      85\n",
       "South Korea                83\n",
       "Turkey                     63\n",
       "Spain                      28\n",
       "Sweden                     20\n",
       "Canada                     19\n",
       "Italy                      17\n",
       "Australia                  17\n",
       "China                      17\n",
       "France                     17\n",
       "Norway                     15\n",
       "Thailand                   14\n",
       "Iran                       14\n",
       "Germany                    13\n",
       "Serbia                     12\n",
       "Mexico                     12\n",
       "Denmark                    11\n",
       "Brazil                     10\n",
       "Pakistan                    7\n",
       "Egypt                       5\n",
       "Taiwan                      5\n",
       "Ireland                     5\n",
       "South Africa                4\n",
       "Russia                      4\n",
       "Israel                      4\n",
       "Belgium                     3\n",
       "New Zealand                 3\n",
       "Bangladesh                  3\n",
       "Colombia                    3\n",
       "Poland                      3\n",
       "Netherlands                 2\n",
       "Indonesia                   2\n",
       "Finland                     2\n",
       "Iceland                     2\n",
       "Portugal                    2\n",
       "Singapore                   2\n",
       "Chile                       2\n",
       "Argentina                   2\n",
       "Tunisia                     2\n",
       "Bosnia and Herzegovina      1\n",
       "Saudi Arabia                1\n",
       "Korea                       1\n",
       "Croatia                     1\n",
       "Estonia                     1\n",
       "Latvia                      1\n",
       "Jordan                      1\n",
       "Romania                     1\n",
       "Philippines                 1\n",
       "Austria                     1\n",
       "Sri Lanka                   1\n",
       "Luxembourg                  1\n",
       "United Arab Emirates        1\n",
       "Name: origin, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#I simplyfy that column to boolean 'made_in_usa'\n",
    "df.origin.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "160386d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['made_in_usa'] = (df.origin == 'United States').apply(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "501430ca",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>made_in_usa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>imdb_rating</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.169572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>made_in_usa</th>\n",
       "      <td>-0.169572</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             imdb_rating  made_in_usa\n",
       "imdb_rating     1.000000    -0.169572\n",
       "made_in_usa    -0.169572     1.000000"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['imdb_rating', 'made_in_usa']].corr()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b57fb574",
   "metadata": {},
   "source": [
    "### PRODUCING COMPANY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9d665033",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Netflix                                   68\n",
       "Amazon Studios                            31\n",
       "Studio Dragon                             23\n",
       "Universal Television                      21\n",
       "CBS Television Studios                    20\n",
       "3 Arts Entertainment                      20\n",
       "ABC Signature                             17\n",
       "ALTBalaji                                 17\n",
       "Warner Bros. Television                   16\n",
       "Netflix Studios                           15\n",
       "Applause Entertainment Ltd.               15\n",
       "Lionsgate Television                      14\n",
       "British Broadcasting Corporation (BBC)    14\n",
       "BBC Studios                               14\n",
       "20th Television                           11\n",
       "JTBC Studios                              11\n",
       "Home Box Office (HBO)                     10\n",
       "Sony Pictures Television                  10\n",
       "MX Player                                 10\n",
       "20th Century Fox Television                9\n",
       "Ay Yapim                                   9\n",
       "Berlanti Productions                       8\n",
       "Paramount Television                       8\n",
       "Fox Entertainment                          8\n",
       "The Viral Fever                            8\n",
       "Contagious Online Media Network            8\n",
       "ITV - Independent Television               7\n",
       "Zee5                                       7\n",
       "Movistar+                                  7\n",
       "Broadway Video                             7\n",
       "Aniplex                                    7\n",
       "Netflix Animation                          7\n",
       "GMMTV                                      7\n",
       "Studio S                                   6\n",
       "CBS Studios                                6\n",
       "Warner Bros. Animation                     6\n",
       "Jar Pictures                               6\n",
       "Boardwalk Pictures                         6\n",
       "Amblin Television                          6\n",
       "Anonymous Content                          6\n",
       "Firefly Productions                        6\n",
       "Mappa                                      6\n",
       "FX Productions                             6\n",
       "DC Entertainment                           6\n",
       "tvN Production                             5\n",
       "C More                                     5\n",
       "Imagine Entertainment                      5\n",
       "Nickelodeon Animation Studios              5\n",
       "Two Brothers Pictures                      5\n",
       "Endeavor Content                           5\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "comp = pd.DataFrame(Counter(df.company.sum()), index=[0]).T\n",
    "comp[0].sort_values(ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a810d1fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>title</th>\n",
       "      <th>s01_episodes</th>\n",
       "      <th>avg_runtime</th>\n",
       "      <th>genres</th>\n",
       "      <th>rel_date</th>\n",
       "      <th>certification</th>\n",
       "      <th>origin</th>\n",
       "      <th>company</th>\n",
       "      <th>...</th>\n",
       "      <th>Musical</th>\n",
       "      <th>Sport</th>\n",
       "      <th>Fantasy</th>\n",
       "      <th>genres_n</th>\n",
       "      <th>rel_month</th>\n",
       "      <th>TV_PG</th>\n",
       "      <th>TV_14</th>\n",
       "      <th>TV_MA</th>\n",
       "      <th>made_in_usa</th>\n",
       "      <th>big_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.imdb.com/title/tt8115560</td>\n",
       "      <td>6.6</td>\n",
       "      <td>Tidying Up with Marie Kondo</td>\n",
       "      <td>8</td>\n",
       "      <td>40</td>\n",
       "      <td>[Reality-TV]</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>TV-PG</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Netflix, The Jackal Group]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>https://www.imdb.com/title/tt1312171</td>\n",
       "      <td>8.0</td>\n",
       "      <td>The Umbrella Academy</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>[Action, Adventure, Comedy]</td>\n",
       "      <td>2019-02-15</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Dark Horse Entertainment, Netflix, Universal ...</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>https://www.imdb.com/title/tt9561862</td>\n",
       "      <td>8.4</td>\n",
       "      <td>Love, Death &amp; Robots</td>\n",
       "      <td>18</td>\n",
       "      <td>15</td>\n",
       "      <td>[Animation, Short, Action]</td>\n",
       "      <td>2019-03-15</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Blur Studio, Netflix Studios, Netflix]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>https://www.imdb.com/title/tt9184994</td>\n",
       "      <td>7.9</td>\n",
       "      <td>How to Sell Drugs Online (Fast)</td>\n",
       "      <td>6</td>\n",
       "      <td>30</td>\n",
       "      <td>[Comedy, Crime, Drama]</td>\n",
       "      <td>2019-05-31</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>Germany</td>\n",
       "      <td>[Bildundtonfabrik, Netflix]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>https://www.imdb.com/title/tt7087260</td>\n",
       "      <td>7.4</td>\n",
       "      <td>Tales of the City</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>[Drama]</td>\n",
       "      <td>2019-06-07</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Netflix, Sweetpants Productions, Working Titl...</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163</th>\n",
       "      <td>https://www.imdb.com/title/tt14163838</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Feels Like Ishq</td>\n",
       "      <td>6</td>\n",
       "      <td>32</td>\n",
       "      <td>[Comedy, Drama, Romance]</td>\n",
       "      <td>2021-07-23</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>India</td>\n",
       "      <td>[Awesomeness TV, Mutant Films, Netflix]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187</th>\n",
       "      <td>https://www.imdb.com/title/tt11834150</td>\n",
       "      <td>7.2</td>\n",
       "      <td>The Chair</td>\n",
       "      <td>6</td>\n",
       "      <td>30</td>\n",
       "      <td>[Comedy, Drama]</td>\n",
       "      <td>2021-08-20</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>United States</td>\n",
       "      <td>[BLB Media, Netflix Studios, Netflix]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1201</th>\n",
       "      <td>https://www.imdb.com/title/tt5540990</td>\n",
       "      <td>6.4</td>\n",
       "      <td>On the Verge</td>\n",
       "      <td>12</td>\n",
       "      <td>35</td>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>2021-09-06</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>France</td>\n",
       "      <td>[The Film TV, Canal+, Netflix]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1233</th>\n",
       "      <td>https://www.imdb.com/title/tt13278100</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Braqueurs</td>\n",
       "      <td>6</td>\n",
       "      <td>44</td>\n",
       "      <td>[Action, Crime, Drama]</td>\n",
       "      <td>2021-09-24</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>France</td>\n",
       "      <td>[Labyrinthe Films, Netflix]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1315</th>\n",
       "      <td>https://www.imdb.com/title/tt1267295</td>\n",
       "      <td>6.7</td>\n",
       "      <td>Cowboy Bebop</td>\n",
       "      <td>10</td>\n",
       "      <td>58</td>\n",
       "      <td>[Action, Adventure, Comedy]</td>\n",
       "      <td>2021-11-19</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Midnight Radio, Netflix, Sunrise]</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       link  imdb_rating  \\\n",
       "2      https://www.imdb.com/title/tt8115560          6.6   \n",
       "51     https://www.imdb.com/title/tt1312171          8.0   \n",
       "86     https://www.imdb.com/title/tt9561862          8.4   \n",
       "114    https://www.imdb.com/title/tt9184994          7.9   \n",
       "123    https://www.imdb.com/title/tt7087260          7.4   \n",
       "...                                     ...          ...   \n",
       "1163  https://www.imdb.com/title/tt14163838          6.4   \n",
       "1187  https://www.imdb.com/title/tt11834150          7.2   \n",
       "1201   https://www.imdb.com/title/tt5540990          6.4   \n",
       "1233  https://www.imdb.com/title/tt13278100          7.0   \n",
       "1315   https://www.imdb.com/title/tt1267295          6.7   \n",
       "\n",
       "                                title  s01_episodes  avg_runtime  \\\n",
       "2         Tidying Up with Marie Kondo             8           40   \n",
       "51               The Umbrella Academy            10           60   \n",
       "86               Love, Death & Robots            18           15   \n",
       "114   How to Sell Drugs Online (Fast)             6           30   \n",
       "123                 Tales of the City            10           60   \n",
       "...                               ...           ...          ...   \n",
       "1163                  Feels Like Ishq             6           32   \n",
       "1187                        The Chair             6           30   \n",
       "1201                     On the Verge            12           35   \n",
       "1233                        Braqueurs             6           44   \n",
       "1315                     Cowboy Bebop            10           58   \n",
       "\n",
       "                           genres   rel_date certification         origin  \\\n",
       "2                    [Reality-TV] 2019-01-01         TV-PG  United States   \n",
       "51    [Action, Adventure, Comedy] 2019-02-15         TV-14  United States   \n",
       "86     [Animation, Short, Action] 2019-03-15         TV-MA  United States   \n",
       "114        [Comedy, Crime, Drama] 2019-05-31         TV-MA        Germany   \n",
       "123                       [Drama] 2019-06-07         TV-MA  United States   \n",
       "...                           ...        ...           ...            ...   \n",
       "1163     [Comedy, Drama, Romance] 2021-07-23         TV-MA          India   \n",
       "1187              [Comedy, Drama] 2021-08-20         TV-MA  United States   \n",
       "1201                     [Comedy] 2021-09-06         TV-14         France   \n",
       "1233       [Action, Crime, Drama] 2021-09-24         TV-MA         France   \n",
       "1315  [Action, Adventure, Comedy] 2021-11-19         TV-MA  United States   \n",
       "\n",
       "                                                company  ... Musical Sport  \\\n",
       "2                           [Netflix, The Jackal Group]  ...       0     0   \n",
       "51    [Dark Horse Entertainment, Netflix, Universal ...  ...       0     0   \n",
       "86              [Blur Studio, Netflix Studios, Netflix]  ...       0     0   \n",
       "114                         [Bildundtonfabrik, Netflix]  ...       0     0   \n",
       "123   [Netflix, Sweetpants Productions, Working Titl...  ...       0     0   \n",
       "...                                                 ...  ...     ...   ...   \n",
       "1163            [Awesomeness TV, Mutant Films, Netflix]  ...       0     0   \n",
       "1187              [BLB Media, Netflix Studios, Netflix]  ...       0     0   \n",
       "1201                     [The Film TV, Canal+, Netflix]  ...       0     0   \n",
       "1233                        [Labyrinthe Films, Netflix]  ...       0     0   \n",
       "1315                 [Midnight Radio, Netflix, Sunrise]  ...       0     0   \n",
       "\n",
       "     Fantasy genres_n  rel_month  TV_PG  TV_14  TV_MA  made_in_usa  big_c  \n",
       "2          0        1          1      1      0      0            1      1  \n",
       "51         0        3          2      0      1      0            1      1  \n",
       "86         0        3          3      0      0      1            1      1  \n",
       "114        0        3          5      0      0      1            0      1  \n",
       "123        0        1          6      0      0      1            1      1  \n",
       "...      ...      ...        ...    ...    ...    ...          ...    ...  \n",
       "1163       0        3          7      0      0      1            0      1  \n",
       "1187       0        2          8      0      0      1            1      1  \n",
       "1201       0        1          9      0      1      0            0      1  \n",
       "1233       0        3          9      0      0      1            0      1  \n",
       "1315       0        3         11      0      0      1            1      1  \n",
       "\n",
       "[68 rows x 47 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create list of big companies based on the number of shows produced in the observed time period (minimum of 10 shows)\n",
    "company_l = ['Netflix', 'Netflix Studios', 'Amazon Studios', 'Studio Dragon', 'Universal Television', 'CBS Television Studios', '3 Arts Entertainment', 'ABC Signature', 'ALTBalaji', 'Warner Bros. Television', ' Applause Entertainment Ltd.', 'Lionsgate Television', 'British Broadcasting Corporation (BBC)', 'BBC Studios', '20th Television', 'JTBC Studios', 'Home Box Office (HBO)', 'Sony Pictures Television', 'MX Player', '20th Century Fox Television', 'Paramount Television', 'Fox Entertainment', 'Netflix Animation', 'Warner Bros. Animation']\n",
    "\n",
    "def c(x):\n",
    "    for i in company_l:\n",
    "        if i in x:\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "    \n",
    "\n",
    "df['big_c'] = df.company.apply(c)\n",
    "\n",
    "df[df.big_c==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "716b37cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['company_n'] = (df.company.apply(lambda x: len(x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dd910fc",
   "metadata": {},
   "source": [
    "AWARDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "884e2952",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>title</th>\n",
       "      <th>s01_episodes</th>\n",
       "      <th>avg_runtime</th>\n",
       "      <th>genres</th>\n",
       "      <th>rel_date</th>\n",
       "      <th>certification</th>\n",
       "      <th>origin</th>\n",
       "      <th>company</th>\n",
       "      <th>...</th>\n",
       "      <th>rel_month</th>\n",
       "      <th>TV_PG</th>\n",
       "      <th>TV_14</th>\n",
       "      <th>TV_MA</th>\n",
       "      <th>made_in_usa</th>\n",
       "      <th>big_c</th>\n",
       "      <th>company_n</th>\n",
       "      <th>creators_a_n</th>\n",
       "      <th>stars_a_n</th>\n",
       "      <th>awards_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.imdb.com/title/tt7670568</td>\n",
       "      <td>5.8</td>\n",
       "      <td>The Masked Singer</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>[Game-Show, Music, Reality-TV]</td>\n",
       "      <td>2019-01-02</td>\n",
       "      <td>TV-PG</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Smart Dog Media, Fox Alternative Entertainmen...</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.imdb.com/title/tt8001250</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Siempre Bruja</td>\n",
       "      <td>11</td>\n",
       "      <td>40</td>\n",
       "      <td>[Drama, Fantasy]</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>Colombia</td>\n",
       "      <td>[Caracol]</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.imdb.com/title/tt8115560</td>\n",
       "      <td>6.6</td>\n",
       "      <td>Tidying Up with Marie Kondo</td>\n",
       "      <td>8</td>\n",
       "      <td>40</td>\n",
       "      <td>[Reality-TV]</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>TV-PG</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Netflix, The Jackal Group]</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.imdb.com/title/tt8324422</td>\n",
       "      <td>8.1</td>\n",
       "      <td>PEN15</td>\n",
       "      <td>10</td>\n",
       "      <td>30</td>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>2019-02-08</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Awesomeness TV, Odenkirk Provissiero Entertai...</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.imdb.com/title/tt8888322</td>\n",
       "      <td>6.6</td>\n",
       "      <td>Bloom</td>\n",
       "      <td>6</td>\n",
       "      <td>60</td>\n",
       "      <td>[Drama, Mystery, Sci-Fi]</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>Australia</td>\n",
       "      <td>[Playmaker Media]</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   link  imdb_rating  \\\n",
       "0  https://www.imdb.com/title/tt7670568          5.8   \n",
       "1  https://www.imdb.com/title/tt8001250          6.2   \n",
       "2  https://www.imdb.com/title/tt8115560          6.6   \n",
       "3  https://www.imdb.com/title/tt8324422          8.1   \n",
       "4  https://www.imdb.com/title/tt8888322          6.6   \n",
       "\n",
       "                         title  s01_episodes  avg_runtime  \\\n",
       "0            The Masked Singer            10           60   \n",
       "1                Siempre Bruja            11           40   \n",
       "2  Tidying Up with Marie Kondo             8           40   \n",
       "3                        PEN15            10           30   \n",
       "4                        Bloom             6           60   \n",
       "\n",
       "                           genres   rel_date certification         origin  \\\n",
       "0  [Game-Show, Music, Reality-TV] 2019-01-02         TV-PG  United States   \n",
       "1                [Drama, Fantasy] 2019-01-01         TV-14       Colombia   \n",
       "2                    [Reality-TV] 2019-01-01         TV-PG  United States   \n",
       "3                        [Comedy] 2019-02-08         TV-MA  United States   \n",
       "4        [Drama, Mystery, Sci-Fi] 2019-01-01             0      Australia   \n",
       "\n",
       "                                             company  ... rel_month TV_PG  \\\n",
       "0  [Smart Dog Media, Fox Alternative Entertainmen...  ...         1     1   \n",
       "1                                          [Caracol]  ...         1     0   \n",
       "2                        [Netflix, The Jackal Group]  ...         1     1   \n",
       "3  [Awesomeness TV, Odenkirk Provissiero Entertai...  ...         2     0   \n",
       "4                                  [Playmaker Media]  ...         1     0   \n",
       "\n",
       "  TV_14 TV_MA  made_in_usa  big_c  company_n  creators_a_n  stars_a_n  \\\n",
       "0     0     0            1      0          3             0          9   \n",
       "1     1     0            0      0          1             0          0   \n",
       "2     0     0            1      1          2             1          1   \n",
       "3     0     1            1      0          3             6          7   \n",
       "4     0     0            0      0          1             5         12   \n",
       "\n",
       "   awards_n  \n",
       "0         9  \n",
       "1         0  \n",
       "2         2  \n",
       "3        13  \n",
       "4        17  \n",
       "\n",
       "[5 rows x 51 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculating the number of awards for each creator and star, and summing them to create a new feature representing the total awards of the show's cast and crew\n",
    "df['creators_a_n']= df.creators_a.apply(sum) \n",
    "df['stars_a_n']= df.stars_a.apply(sum)\n",
    "df['awards_n'] = df['creators_a_n'] + df['stars_a_n']\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9bbe0235",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>s01_episodes</th>\n",
       "      <th>avg_runtime</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>Music</th>\n",
       "      <th>Comedy</th>\n",
       "      <th>Biography</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Horror</th>\n",
       "      <th>Documentary</th>\n",
       "      <th>...</th>\n",
       "      <th>rel_month</th>\n",
       "      <th>TV_PG</th>\n",
       "      <th>TV_14</th>\n",
       "      <th>TV_MA</th>\n",
       "      <th>made_in_usa</th>\n",
       "      <th>big_c</th>\n",
       "      <th>company_n</th>\n",
       "      <th>creators_a_n</th>\n",
       "      <th>stars_a_n</th>\n",
       "      <th>awards_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>imdb_rating</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.036943</td>\n",
       "      <td>0.076961</td>\n",
       "      <td>0.034059</td>\n",
       "      <td>0.014045</td>\n",
       "      <td>0.022244</td>\n",
       "      <td>0.067982</td>\n",
       "      <td>-0.0017</td>\n",
       "      <td>-0.100276</td>\n",
       "      <td>0.027008</td>\n",
       "      <td>...</td>\n",
       "      <td>0.028357</td>\n",
       "      <td>-0.105924</td>\n",
       "      <td>-0.025684</td>\n",
       "      <td>-0.031788</td>\n",
       "      <td>-0.169572</td>\n",
       "      <td>-0.034374</td>\n",
       "      <td>-0.038182</td>\n",
       "      <td>0.001824</td>\n",
       "      <td>0.058789</td>\n",
       "      <td>0.050013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             imdb_rating  s01_episodes  avg_runtime  Thriller     Music  \\\n",
       "imdb_rating          1.0      0.036943     0.076961  0.034059  0.014045   \n",
       "\n",
       "               Comedy  Biography  Adventure    Horror  Documentary  ...  \\\n",
       "imdb_rating  0.022244   0.067982    -0.0017 -0.100276     0.027008  ...   \n",
       "\n",
       "             rel_month     TV_PG     TV_14     TV_MA  made_in_usa     big_c  \\\n",
       "imdb_rating   0.028357 -0.105924 -0.025684 -0.031788    -0.169572 -0.034374   \n",
       "\n",
       "             company_n  creators_a_n  stars_a_n  awards_n  \n",
       "imdb_rating  -0.038182      0.001824   0.058789  0.050013  \n",
       "\n",
       "[1 rows x 40 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.corr().iloc[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "83edd97a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#It does not seem interesting to explore particular names as in explored time period only 12 artists manageged to work in more than 3 shows\n",
    "(pd.Series((df.creators+df.stars).sum()).value_counts() > 3).sum() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "657f8003",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['link', 'imdb_rating', 'title', 's01_episodes', 'avg_runtime', 'genres',\n",
       "       'rel_date', 'certification', 'origin', 'company', 'creators',\n",
       "       'creators_a', 'stars', 'stars_a', 'Thriller', 'Music', 'Comedy',\n",
       "       'Biography', 'Adventure', 'Horror', 'Documentary', 'Western', 'Sci_Fi',\n",
       "       'Romance', 'Short', 'Mystery', 'Action', 'Family', 'Game_Show', 'News',\n",
       "       'Talk_Show', 'War', 'Crime', 'History', 'Drama', 'Reality_TV',\n",
       "       'Animation', 'Musical', 'Sport', 'Fantasy', 'genres_n', 'rel_month',\n",
       "       'TV_PG', 'TV_14', 'TV_MA', 'made_in_usa', 'big_c', 'company_n',\n",
       "       'creators_a_n', 'stars_a_n', 'awards_n'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f7b5960",
   "metadata": {},
   "source": [
    "### pickling shows list to save progress and to follow processing in another notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a78f97ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.to_pickle(\"shows2.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (classification)",
   "language": "python",
   "name": "classification"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}