1_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a764e5be",
   "metadata": {},
   "source": [
    "# Data Selection and Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "648e5597",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "os.environ['USE_PYGEOS'] = '0'\n",
    "\n",
    "import geopandas as gpd\n",
    "import trackintel as ti"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11154224",
   "metadata": {},
   "source": [
    "## Load data\n",
    "Stages from original data, example data format:\n",
    "\n",
    "|index | id | user_id | started_at    | finished_at | mode |geom | length |\n",
    "|----|----|----|----|----|----|----|----|\n",
    "| 0 | 29 | 16 | 2016-04-14 18:17:23.000+00:00 | 2016-04-14 18:20:57.000+00:00 | Mode::Walk | \"LINESTRING (8.0125 47.5064, 8.0129 47.50666)\" | 235.018 |\n",
    "| 1 | 32 | 17 | 2017-04-30 11:05:40.000+00:00 | 2017-04-30 11:19:45.000+00:00 | Mode::Walk | \"LINESTRING (6.6961 46.8441, 6.6962 46.8446)\"  | 1035.4314 |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cedad155",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load stages from CSV\n",
    "tl = ti.io.file.read_triplegs_csv(os.path.join(\"data\", \"triplegs.csv\"), index_col=\"index\", geom_col=\"geom\", crs=4326)\n",
    "tl[\"dur_s\"] = (tl[\"finished_at\"] - tl[\"started_at\"]).dt.total_seconds()\n",
    "\n",
    "n_tl_raw = len(tl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf1e1ca5",
   "metadata": {},
   "source": [
    "## Step 1: Select stages of users with high tracking quality\n",
    "TODO: need staypoint file to calculate user tracking quality\n",
    "\n",
    "sp from original data, example data format:\n",
    "\n",
    "|index | id | user_id | started_at    | finished_at |\n",
    "|----|----|----|----|----|\n",
    "| 0 | 1 | 2 | 2016-03-20 09:16:29+00:00 | 2016-03-20 09:46:29+00:00 | \n",
    "| 1 | 3 | 3 | 2017-03-19 09:52:15+00:00 | 2017-03-19 13:21:58+00:00 | \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8514cadc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_user(tl, quality_filter = 0.5):\n",
    "    sp = ti.io.file.read_staypoints_csv(\n",
    "        os.path.join(\"data\", \"staypoints.csv\"), index_col=\"index\", geom_col=\"geom\", crs=4326\n",
    "    )\n",
    "\n",
    "    # calculate temporal tracking quality per user\n",
    "    tl_sp = pd.concat([tl, sp])\n",
    "    user_tracking_quality = ti.analysis.tracking_quality.temporal_tracking_quality(tl_sp, granularity=\"all\")\n",
    "    user_tracking_quality.rename(columns={\"quality\": \"user_t_quality\"}, inplace=True)\n",
    "    user_tracking_quality\n",
    "\n",
    "    tl = tl.merge(user_tracking_quality, on=\"user_id\", how=\"left\")\n",
    "\n",
    "    tl = tl[tl[\"user_t_quality\"] >= quality_filter]\n",
    "\n",
    "    return tl, len(tl)\n",
    "\n",
    "\n",
    "tl, n_tl_user_selection = filter_user(tl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "372e3491",
   "metadata": {},
   "source": [
    "## Step 2: Select stages within Switzerland\n",
    "TODO: need area shp file as input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "29e27ab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_area(tl):\n",
    "    CH = gpd.read_file(os.path.join(\"data\", \"swiss\", \"swiss.shp\"))\n",
    "    CH_WGS84 = CH.to_crs(4326)\n",
    "\n",
    "    # simplify boundaries with Douglas Peuker for faster filtering\n",
    "    CH_generalized = CH_WGS84.copy()\n",
    "    CH_generalized[\"geometry\"] = CH_generalized[\"geometry\"].simplify(0.005)\n",
    "    # count_before = len(CH_WGS84.iloc[0][\"geometry\"].exterior.coords)  # count number of coordinate points\n",
    "    # count_after = len(CH_generalized.iloc[0][\"geometry\"].exterior.coords)  # count number of coordinate points\n",
    "    # print(round(((1 - count_after / count_before) * 100), 1), \"percent compressed\")\n",
    "\n",
    "    # select triplegs within Switzerland\n",
    "    tl = ti.preprocessing.filter.spatial_filter(tl, CH_generalized, method=\"within\")\n",
    "    return tl, len(tl)\n",
    "    \n",
    "tl, n_tl_within_CH = filter_area(tl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca84cbd9",
   "metadata": {},
   "source": [
    "## Step 3: Select tranport modes\n",
    "Check the mode list and specify which mode to exclude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5c6faab0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_mode(tl):\n",
    "    # remove triplegs with trip modes: plane, coach and ski\n",
    "    exclude = (tl[\"mode\"] == \"Mode::Airplane\") | (tl[\"mode\"] == \"Mode::Coach\") | (tl[\"mode\"] == \"Mode::Ski\")\n",
    "    tl = tl.drop(tl[exclude].index)\n",
    "\n",
    "    return tl, len(tl)\n",
    "\n",
    "tl, n_tl_mode_selection = filter_mode(tl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "167c9194",
   "metadata": {},
   "source": [
    "## Step 4: Select stages satisfying minimal requirements of duration and length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1a9fcc4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_legs(tl):\n",
    "    # remove triplegs with less than 50 metres or with less than 60 seconds\n",
    "    exclude = (tl[\"length\"] < 50) | (tl[\"dur_s\"] < 60)\n",
    "    tl = tl.drop(tl[exclude].index)\n",
    "    return tl, len(tl)\n",
    "    \n",
    "tl, n_tl_length_duration_selection = filter_legs(tl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90dd74df",
   "metadata": {},
   "source": [
    "## Step 5: Select stages with good tracking quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "36e13022",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_quality(tl):\n",
    "    # calculate average recording interval\n",
    "    tl[\"recording_interval\"] = tl[\"dur_s\"] / [(len(row.coords) - 1) for row in tl[\"geom\"]]\n",
    "    # select triplegs with more than 4 track points and with average recording interval less than 60 seconds\n",
    "    exclude = [(len(s.coords)) < 4 for s in tl[\"geom\"]] | (tl[\"recording_interval\"] > 60)\n",
    "    tl = tl.drop(tl[exclude].index)\n",
    "    return tl, len(tl)\n",
    "\n",
    "tl, n_tl_tracking_quality_selection = filter_quality(tl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77b58413",
   "metadata": {},
   "source": [
    "## Step 6: Select stages with realistic speed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "66ffcf1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_speed(tl):\n",
    "    # calculate speed\n",
    "    tl[\"speed_av\"] = tl[\"length\"] / tl[\"dur_s\"] * 3.6\n",
    "    exclude = (\n",
    "        ((tl[\"speed_av\"] > 20) & (tl[\"mode\"] == \"Mode::Walk\"))\n",
    "        | ((tl[\"speed_av\"] > 60) & (tl[\"mode\"] == \"Mode::Bicycle\"))\n",
    "        | ((tl[\"speed_av\"] > 60) & (tl[\"mode\"] == \"Mode::Ebicycle\"))\n",
    "        | ((tl[\"speed_av\"] > 250) & (tl[\"mode\"] == \"Mode::Train\"))\n",
    "        | ((tl[\"speed_av\"] > 80) & (tl[\"mode\"] == \"Mode::Tram\"))\n",
    "        | ((tl[\"speed_av\"] > 150) & (tl[\"mode\"] == \"Mode::Bus\"))\n",
    "        | ((tl[\"speed_av\"] > 50) & (tl[\"mode\"] == \"Mode::Boat\"))\n",
    "        | ((tl[\"speed_av\"] > 150) & (tl[\"mode\"] == \"Mode::Car\"))\n",
    "        | ((tl[\"speed_av\"] > 150) & (tl[\"mode\"] == \"Mode::Ecar\"))\n",
    "    )\n",
    "    tl = tl.drop(tl[exclude].index)\n",
    "    return tl, len(tl)\n",
    "\n",
    "tl, n_tl_speed_selection = filter_speed(tl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a010e77b",
   "metadata": {},
   "source": [
    "## Inspect and save preprocessed stages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa3f3bd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#triplegs original data:                       457754 \n",
      "#triplegs after step 1:                        447695 \n",
      "#triplegs after step 2:                        397593 \n",
      "#triplegs after step 3:                        395932 \n",
      "#triplegs after step 4:                        376866 \n",
      "#triplegs after step 5:                        371218 \n",
      "#triplegs after step 6 (pre-processed data):   365307\n"
     ]
    }
   ],
   "source": [
    "# show number of trip legs after each step\n",
    "print(\n",
    "    \"#triplegs original data:                      \",\n",
    "    n_tl_raw,\n",
    "    \"\\n\" \"#triplegs after step 1:                       \",\n",
    "    n_tl_user_selection,\n",
    "    \"\\n\" \"#triplegs after step 2:                       \",\n",
    "    n_tl_within_CH,\n",
    "    \"\\n\" \"#triplegs after step 3:                       \",\n",
    "    n_tl_mode_selection,\n",
    "    \"\\n\" \"#triplegs after step 4:                       \",\n",
    "    n_tl_length_duration_selection,\n",
    "    \"\\n\" \"#triplegs after step 5:                       \",\n",
    "    n_tl_tracking_quality_selection,\n",
    "    \"\\n\" \"#triplegs after step 6 (pre-processed data):  \",\n",
    "    n_tl_speed_selection,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "904041a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mode::Walk        155177\n",
      "Mode::Ecar         78758\n",
      "Mode::Car          51920\n",
      "Mode::Train        51470\n",
      "Mode::Bicycle      11575\n",
      "Mode::Bus           9436\n",
      "Mode::Tram          6231\n",
      "Mode::Ebicycle       373\n",
      "Mode::Boat           367\n",
      "Name: mode, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(tl[\"mode\"].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fe499e2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "tl.to_csv(os.path.join(\"data\", \"triplegs_preprocessed.csv\"), index_label=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11c82944",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 ('mode')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "e3307148b578e6aa85616c61b46cb8e0917f8e20a9b0d5c7552aa4399d77cb8c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}