2.0. DataManipulation_Functions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b45b3a83",
   "metadata": {},
   "source": [
    "# Functions for primary data set manipulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b7f793fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29d83134",
   "metadata": {},
   "source": [
    "## Data loading and cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2b1b7f2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_scraped_data(folder):\n",
    "    \"\"\"\n",
    "    Loads all the files from the folder and\n",
    "    concatenate them in one dataframe.\n",
    "    \n",
    "    Args: folder - string, a path of a folder with files\n",
    "    Returns: df_scraped - dataframe\n",
    "    \"\"\"\n",
    "    \n",
    "    # Data loading\n",
    "    \n",
    "    # Get a list of files for loading\n",
    "    files = [file for file in  os.listdir(folder) if file.endswith('_cleaned.csv')]\n",
    "    \n",
    "    # Load the first file\n",
    "    df_scraped = pd.read_csv(folder+'/'+files[0], sep='|')\n",
    "    # Add a key (like '0-15'), so we can identify which file each row comes from\n",
    "    key = files[0][:files[0].rfind('_output')]\n",
    "    df_scraped['file_key'] = key\n",
    "    \n",
    "    # Load the remaining files\n",
    "    for file in files[1:]:\n",
    "        df = pd.read_csv(folder+'/'+file, sep='|')\n",
    "        key = file[:file.rfind('_output')]\n",
    "        df['file_key'] = key\n",
    "        \n",
    "        df_scraped = pd.concat([df_scraped, df])\n",
    "        \n",
    "    # Give a meaningful name to the first column\n",
    "    df_scraped.rename(columns={'Unnamed: 0': 'id_in_source_file'}, inplace=True)\n",
    "    \n",
    "    # Sort values by time descending\n",
    "    df_scraped.sort_values(by='date', ascending=False, inplace=True)\n",
    "    \n",
    "    # Add a unique index\n",
    "    df_scraped.reset_index(inplace=True, drop=True)\n",
    "    \n",
    "    # Replace np.Nan in the column \"body\" with empty string\n",
    "    # otherwise tokenization can't be applied\n",
    "    df_scraped.loc[df_scraped['body'].isnull(), 'body'] = ''\n",
    "        \n",
    "    return df_scraped     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "81d6baef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_columns(df_scraped):\n",
    "    \"\"\"\n",
    "    Performs the following manipulations with dataframe:\n",
    "    - Adds columns useful for data grouping\n",
    "    - Adds a level (whether a row represents an individual\n",
    "      news story or a whole newscast)\n",
    "    \n",
    "    Args: df_scraped - a dataframe with scraped data\n",
    "    Returns: df_scraped - the initial dataframe with added columns\n",
    "    \"\"\"  \n",
    "    \n",
    "    # Columns useful for data grouping\n",
    "    \n",
    "    # Create a column with a date-time in date-time format\n",
    "    df_scraped['datetime'] = pd.to_datetime(df_scraped['date'])\n",
    "    # Create a column with date \n",
    "    df_scraped['dat'] = df_scraped['datetime'].dt.date\n",
    "    # Create a column with year\n",
    "    df_scraped['year'] = df_scraped['datetime'].dt.year\n",
    "    # Create a column with year_month\n",
    "    df_scraped['year_month'] = df_scraped['datetime'].dt.to_period('M')\n",
    "    # Create a column with hour\n",
    "    df_scraped['hour'] = df_scraped['datetime'].dt.hour\n",
    "    # Create a column with a day of week\n",
    "    df_scraped['weekday'] = df_scraped['datetime'].dt.dayofweek\n",
    "    \n",
    "    \n",
    "    # Level (whether a row represents an individual \n",
    "    \n",
    "    # news story or a whole newscast) - new column 'whole newscast' (bool)\n",
    "    df_scraped['whole newscast'] = df_scraped['title'].str \\\n",
    "        .contains(\"Выпуск программы «Время»|Выпуск программы «Воскресное Время»|Выпуск программы «Воскресное время»|Выпуск новостей\")\n",
    "    # Adding time of a newscast extracted from its title\n",
    "    df_scraped['newscast title time'] = df_scraped['title'].str.extract('^[а-яА-Я«» ]*([0-9]*):[0-9][0-9].*')\n",
    "    df_scraped['newscast title time1'] = df_scraped['title'].str.extract('^[а-яА-Я«» ]*([0-9]*) час от .*')\n",
    "    df_scraped['newscast title time'] = np.where(df_scraped['newscast title time1'].isnull(),\n",
    "                                                 df_scraped['newscast title time'],\n",
    "                                                 df_scraped['newscast title time1'])\n",
    "    df_scraped['newscast title time'] = pd.to_numeric(df_scraped['newscast title time'], \n",
    "                                                      errors='coerce').astype('Int64')\n",
    "    # If there are no time in a title, it is an individual news story\n",
    "    df_scraped.loc[(df_scraped['whole newscast'])&(df_scraped['newscast title time'].isnull()),\n",
    "                   'whole newscast'] = False\n",
    "    # We don't need the column 'newscast title time' anymore\n",
    "    df_scraped.drop(columns=['newscast title time', 'newscast title time1'], inplace=True)\n",
    "    \n",
    "    return df_scraped  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bf208d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_duplicates(df_scraped):\n",
    "    \"\"\"\n",
    "    Performs the following manipulations:\n",
    "    - Deletes duplicates rows  \n",
    "    - Deletes the first and the last days, because the loading\n",
    "      for them is not full\n",
    "    \n",
    "    Args: df_scraped - a dataframe with scraped data\n",
    "          This function uses added columns, so before \n",
    "          use the function \"add_columns\"\n",
    "    Returns: df_scraped - the initial dataframe with deleted rows\n",
    "    \"\"\"\n",
    "    # Because the loading for the first and last day is not complete, \n",
    "    # we must delete them in order to use only full data in further analysis\n",
    "    max_date = df_scraped['dat'].max()\n",
    "    min_date = df_scraped['dat'].min()\n",
    "    df_scraped.drop(df_scraped[df_scraped['dat'].isin([max_date, min_date])].index, \n",
    "                    inplace=True)\n",
    "    \n",
    "    # Delete duplicate rows \n",
    "    cols_subset = list(df_scraped.columns[1:7]) # all columns before file_key except 'id_in_source_file'\n",
    "    row_duplicates = df_scraped.duplicated(subset=cols_subset, keep='first')\n",
    "    df_scraped.drop(df_scraped[row_duplicates].index, inplace=True)\n",
    "    \n",
    "    return df_scraped    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "65775c3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_clean_scraped_data(folder):\n",
    "    \"\"\"\n",
    "    Loads all the files from the folder and\n",
    "    concatenate them in one dataframe.\n",
    "    \n",
    "    Performs the following manipulations:\n",
    "    - Adds columns useful for data grouping\n",
    "    - Adds a level (whether a row represents an individual\n",
    "      news story or a whole newscast)\n",
    "    - Deletes duplicates rows  \n",
    "    - Deletes the first and the last days, because the loading\n",
    "      for them is not full\n",
    "    \n",
    "    Args: folder - string, a path of a folder with files\n",
    "    Returns: df_scraped - datafrme\n",
    "    \"\"\"\n",
    "    df_scraped = load_scraped_data(folder)\n",
    "    df_scraped = add_columns(df_scraped)\n",
    "    df_scraped = delete_duplicates(df_scraped)\n",
    "    df_scraped.reset_index(inplace=True, drop=True)\n",
    "    \n",
    "    # Update inaccurate video duration values\n",
    "    if len(df_scraped) >= 337000:\n",
    "        # https://www.1tv.ru/news/2007-06-20/209380-militsionery_ranenye_v_rezultate_napadeniya_na_bazu_omona_nahodyatsya_v_gospitale\n",
    "        df_scraped.iloc[348394, 7] = 8\n",
    "        \n",
    "        # https://www.1tv.ru/news/2007-07-06/204193-v_krasnoyarskom_krae_ischut_propavshih_turistov_iz_moskvy\n",
    "        df_scraped.iloc[347534, 7] = 24\n",
    "        \n",
    "        # https://www.1tv.ru/news/2007-10-16/203095-tsik_zaregistriroval_federalnye_spiski_esche_3_politicheskih_partiy\n",
    "        df_scraped.iloc[342323, 7] = 51\n",
    "        \n",
    "        # https://www.1tv.ru/news/2008-02-03/196382-brachnye_muzy_nikolya_sarkozi_nakonets_zhenilsya_na_karle_bruni\n",
    "        df_scraped.iloc[337016, 7] = 181\n",
    "    \n",
    "    return df_scraped"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}