2_bars.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sampling to create Tick, Volume and Dollar-traded bars\n",
    "\n",
    "+ 2.1 Tick Bar\n",
    "+ 2.2 Volume Bar\n",
    "+ 2.3 Volume Bar\n",
    "\n",
    "Here we are taking arbitrary values for sampling the 3 different types of bars.\n",
    "* For tick bar we are sampling 100 ticks. \n",
    "* For volume bar we are sampling 1000 units. \n",
    "* For tick bar we are sampling 1000000 dollar traded\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "import dask.dataframe as dd\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import statsmodels.api as sm \n",
    "import statsmodels.graphics.tsaplots as tsp\n",
    "from scipy.stats import jarque_bera"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = dd.read_parquet('continous_tick_data.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-1acca98f-c3a2-11ee-a464-3887d5db37eb</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "\n",
       "        <tr>\n",
       "        \n",
       "            <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
       "            <td style=\"text-align: left;\"><strong>Cluster type:</strong> distributed.LocalCluster</td>\n",
       "        \n",
       "        </tr>\n",
       "\n",
       "        \n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\"></td>\n",
       "            </tr>\n",
       "        \n",
       "\n",
       "        </table>\n",
       "\n",
       "        \n",
       "            <details>\n",
       "            <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
       "            <div class=\"jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output\">\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\">\n",
       "    </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px; margin-top: 0px;\">LocalCluster</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">e73acf42</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Workers:</strong> 4\n",
       "                </td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total threads:</strong> 8\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total memory:</strong> 15.75 GiB\n",
       "                </td>\n",
       "            </tr>\n",
       "            \n",
       "            <tr>\n",
       "    <td style=\"text-align: left;\"><strong>Status:</strong> running</td>\n",
       "    <td style=\"text-align: left;\"><strong>Using processes:</strong> True</td>\n",
       "</tr>\n",
       "\n",
       "            \n",
       "        </table>\n",
       "\n",
       "        <details>\n",
       "            <summary style=\"margin-bottom: 20px;\">\n",
       "                <h3 style=\"display: inline;\">Scheduler Info</h3>\n",
       "            </summary>\n",
       "\n",
       "            <div style=\"\">\n",
       "    <div>\n",
       "        <div style=\"width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;\"> </div>\n",
       "        <div style=\"margin-left: 48px;\">\n",
       "            <h3 style=\"margin-bottom: 0px;\">Scheduler</h3>\n",
       "            <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Scheduler-ca149975-0e3b-464c-aaee-ea63b2262087</p>\n",
       "            <table style=\"width: 100%; text-align: left;\">\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Comm:</strong> tcp://127.0.0.1:61516\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Workers:</strong> 4\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total threads:</strong> 8\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Started:</strong> Just now\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total memory:</strong> 15.75 GiB\n",
       "                    </td>\n",
       "                </tr>\n",
       "            </table>\n",
       "        </div>\n",
       "    </div>\n",
       "\n",
       "    <details style=\"margin-left: 48px;\">\n",
       "        <summary style=\"margin-bottom: 20px;\">\n",
       "            <h3 style=\"display: inline;\">Workers</h3>\n",
       "        </summary>\n",
       "\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 0</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:61551\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 2\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:61553/status\" target=\"_blank\">http://127.0.0.1:61553/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 3.94 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:61519\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> C:\\Users\\saeed\\AppData\\Local\\Temp\\dask-worker-space\\worker-n___w2x3\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 1</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:61557\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 2\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:61558/status\" target=\"_blank\">http://127.0.0.1:61558/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 3.94 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:61522\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> C:\\Users\\saeed\\AppData\\Local\\Temp\\dask-worker-space\\worker-fzbh4_17\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 2</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:61548\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 2\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:61549/status\" target=\"_blank\">http://127.0.0.1:61549/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 3.94 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:61520\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> C:\\Users\\saeed\\AppData\\Local\\Temp\\dask-worker-space\\worker-nj30a493\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 3</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:61552\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 2\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:61555/status\" target=\"_blank\">http://127.0.0.1:61555/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 3.94 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:61521\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> C:\\Users\\saeed\\AppData\\Local\\Temp\\dask-worker-space\\worker-82utkic2\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "\n",
       "    </details>\n",
       "</div>\n",
       "\n",
       "        </details>\n",
       "    </div>\n",
       "</div>\n",
       "            </details>\n",
       "        \n",
       "\n",
       "    </div>\n",
       "</div>"
      ],
      "text/plain": [
       "<Client: 'tcp://127.0.0.1:61516' processes=4 threads=8, memory=15.75 GiB>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# incase if you want to observe the job on the dask monitor dashboard\n",
    "from dask.distributed import Client\n",
    "client = Client() \n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.1 Tick Bars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 20.9 s\n",
      "Wall time: 6min 9s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "transactions = 100\n",
    "df['group'] =  (df['index']/transactions).astype(np.int64)\n",
    "df_tick_bars = df.groupby('group').agg({'rolled_price':['first','last','max','min'],\n",
    "                                     'volume': 'sum','timestamp':'min','instrument':'count'}).reset_index(drop=True)\n",
    "df_tick_bars.columns = ['open','close','high','low','volume','timestamp','count']\n",
    "df_tick_bars_dd = df_tick_bars.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8561831"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_tick_bars_dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open</th>\n",
       "      <th>close</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>volume</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>959.75</td>\n",
       "      <td>961.25</td>\n",
       "      <td>961.25</td>\n",
       "      <td>959.50</td>\n",
       "      <td>324</td>\n",
       "      <td>2003-06-30 23:00:01</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>961.25</td>\n",
       "      <td>962.50</td>\n",
       "      <td>962.50</td>\n",
       "      <td>961.25</td>\n",
       "      <td>420</td>\n",
       "      <td>2003-07-01 01:03:04</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>962.50</td>\n",
       "      <td>961.50</td>\n",
       "      <td>963.00</td>\n",
       "      <td>961.50</td>\n",
       "      <td>441</td>\n",
       "      <td>2003-07-01 02:05:06</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>961.50</td>\n",
       "      <td>961.75</td>\n",
       "      <td>962.75</td>\n",
       "      <td>961.50</td>\n",
       "      <td>578</td>\n",
       "      <td>2003-07-01 02:23:29</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>961.75</td>\n",
       "      <td>961.25</td>\n",
       "      <td>962.00</td>\n",
       "      <td>961.25</td>\n",
       "      <td>472</td>\n",
       "      <td>2003-07-01 02:40:38</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     open   close    high     low  volume           timestamp  count\n",
       "0  959.75  961.25  961.25  959.50     324 2003-06-30 23:00:01    100\n",
       "1  961.25  962.50  962.50  961.25     420 2003-07-01 01:03:04    100\n",
       "2  962.50  961.50  963.00  961.50     441 2003-07-01 02:05:06    100\n",
       "3  961.50  961.75  962.75  961.50     578 2003-07-01 02:23:29    100\n",
       "4  961.75  961.25  962.00  961.25     472 2003-07-01 02:40:38    100"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tick_bars_dd.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Volume Bars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1min 9s\n",
      "Wall time: 13min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "traded_volume = 1000\n",
    "df['volume'] = df['volume'].astype(np.uint64)\n",
    "df['group'] =  (df['volume'].cumsum()/traded_volume).astype(np.uint64)\n",
    "df_volume_bars = df.groupby('group').agg({'rolled_price':['first','last','max','min'],\n",
    "                                     'volume': 'sum','timestamp':'min','instrument':'count'})#.reset_index(drop=True)\n",
    "df_volume_bars.columns = ['open','close','high','low','volume','timestamp','count']\n",
    "df_volume_bars_dd = df_volume_bars.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4803869"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_volume_bars_dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open</th>\n",
       "      <th>close</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>volume</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>959.75</td>\n",
       "      <td>962.25</td>\n",
       "      <td>963.00</td>\n",
       "      <td>959.50</td>\n",
       "      <td>992</td>\n",
       "      <td>2003-06-30 23:00:01</td>\n",
       "      <td>270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>962.25</td>\n",
       "      <td>962.00</td>\n",
       "      <td>962.75</td>\n",
       "      <td>961.25</td>\n",
       "      <td>1006</td>\n",
       "      <td>2003-07-01 02:16:17</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>962.00</td>\n",
       "      <td>961.00</td>\n",
       "      <td>962.00</td>\n",
       "      <td>959.25</td>\n",
       "      <td>997</td>\n",
       "      <td>2003-07-01 02:59:16</td>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>961.00</td>\n",
       "      <td>959.75</td>\n",
       "      <td>961.00</td>\n",
       "      <td>959.25</td>\n",
       "      <td>1004</td>\n",
       "      <td>2003-07-01 03:35:59</td>\n",
       "      <td>269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>959.75</td>\n",
       "      <td>960.00</td>\n",
       "      <td>960.50</td>\n",
       "      <td>959.50</td>\n",
       "      <td>997</td>\n",
       "      <td>2003-07-01 04:27:02</td>\n",
       "      <td>302</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         open   close    high     low  volume           timestamp  count\n",
       "group                                                                   \n",
       "0      959.75  962.25  963.00  959.50     992 2003-06-30 23:00:01    270\n",
       "1      962.25  962.00  962.75  961.25    1006 2003-07-01 02:16:17    177\n",
       "2      962.00  961.00  962.00  959.25     997 2003-07-01 02:59:16    223\n",
       "3      961.00  959.75  961.00  959.25    1004 2003-07-01 03:35:59    269\n",
       "4      959.75  960.00  960.50  959.50     997 2003-07-01 04:27:02    302"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_volume_bars_dd.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dollar Volume Bars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 3min 47s\n",
      "Wall time: 8min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "traded_dollar_volume = 1000000\n",
    "df['group'] =  (df['dollar_volume'].cumsum()/traded_dollar_volume).astype(np.int64)\n",
    "df_dol_vol_bars = df.groupby('group').agg({'rolled_price':['first','last','max','min'],\n",
    "                                     'volume': 'sum','timestamp':'min','instrument':'count','dollar_volume':'sum'})#.reset_index(drop=True)\n",
    "df_dol_vol_bars.columns = ['open','close','high','low','volume','timestamp','count','dv_sum']\n",
    "df_dol_vol_bars_dd = df_dol_vol_bars.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6161892"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_dol_vol_bars_dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open</th>\n",
       "      <th>close</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>volume</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>count</th>\n",
       "      <th>dv_sum</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>959.75</td>\n",
       "      <td>962.00</td>\n",
       "      <td>963.00</td>\n",
       "      <td>959.50</td>\n",
       "      <td>1030</td>\n",
       "      <td>2003-06-30 23:00:01</td>\n",
       "      <td>274</td>\n",
       "      <td>990456.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>962.00</td>\n",
       "      <td>962.00</td>\n",
       "      <td>962.75</td>\n",
       "      <td>961.25</td>\n",
       "      <td>1040</td>\n",
       "      <td>2003-07-01 02:18:47</td>\n",
       "      <td>189</td>\n",
       "      <td>1000363.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>962.00</td>\n",
       "      <td>961.00</td>\n",
       "      <td>962.00</td>\n",
       "      <td>959.25</td>\n",
       "      <td>1029</td>\n",
       "      <td>2003-07-01 03:02:08</td>\n",
       "      <td>231</td>\n",
       "      <td>988433.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>960.75</td>\n",
       "      <td>960.25</td>\n",
       "      <td>961.00</td>\n",
       "      <td>959.25</td>\n",
       "      <td>1063</td>\n",
       "      <td>2003-07-01 03:37:07</td>\n",
       "      <td>311</td>\n",
       "      <td>1020434.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>960.25</td>\n",
       "      <td>960.50</td>\n",
       "      <td>960.75</td>\n",
       "      <td>959.50</td>\n",
       "      <td>1038</td>\n",
       "      <td>2003-07-01 04:42:51</td>\n",
       "      <td>310</td>\n",
       "      <td>996659.75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         open   close    high     low  volume           timestamp  count  \\\n",
       "group                                                                      \n",
       "0      959.75  962.00  963.00  959.50    1030 2003-06-30 23:00:01    274   \n",
       "1      962.00  962.00  962.75  961.25    1040 2003-07-01 02:18:47    189   \n",
       "2      962.00  961.00  962.00  959.25    1029 2003-07-01 03:02:08    231   \n",
       "3      960.75  960.25  961.00  959.25    1063 2003-07-01 03:37:07    311   \n",
       "4      960.25  960.50  960.75  959.50    1038 2003-07-01 04:42:51    310   \n",
       "\n",
       "           dv_sum  \n",
       "group              \n",
       "0       990456.75  \n",
       "1      1000363.75  \n",
       "2       988433.75  \n",
       "3      1020434.25  \n",
       "4       996659.75  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dol_vol_bars_dd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the sampled bars\n",
    "df_dol_vol_bars_dd.to_parquet('dol_vol_bars.parquet',index=False)\n",
    "df_volume_bars_dd.to_parquet('vol_bars.parquet',index=False)\n",
    "df_tick_bars_dd.to_parquet('tick_bars.parquet',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}