0-preprocess-generate_csvs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm_notebook\n",
    "import pydicom\n",
    "import itertools\n",
    "import numpy as np\n",
    "from concurrent.futures import ProcessPoolExecutor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "tqdm().pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read stage_X_train and split id/label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"stage_2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_dcm_fn = f'data/unzip/{stage}_train_images/ID_6431af929.dcm'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm: cannot remove 'data/unzip/stage_2_train_images/ID_6431af929.dcm': No such file or directory\r\n"
     ]
    }
   ],
   "source": [
    "!rm {bad_dcm_fn}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(f'data/unzip/{stage}_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_12cadc6af_epidural</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_12cadc6af_intraparenchymal</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_12cadc6af_intraventricular</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_12cadc6af_subarachnoid</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_12cadc6af_subdural</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              ID  Label\n",
       "0          ID_12cadc6af_epidural      0\n",
       "1  ID_12cadc6af_intraparenchymal      0\n",
       "2  ID_12cadc6af_intraventricular      0\n",
       "3      ID_12cadc6af_subarachnoid      0\n",
       "4          ID_12cadc6af_subdural      0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['fid'] = df_train.ID.apply(lambda x: '_'.join(x.split('_')[:2]) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.columns = ['ID', 'probability', 'fid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['label'] = df_train.ID.apply(lambda x: x.split('_')[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>probability</th>\n",
       "      <th>fid</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_12cadc6af_epidural</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_12cadc6af</td>\n",
       "      <td>epidural</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_12cadc6af_intraparenchymal</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_12cadc6af</td>\n",
       "      <td>intraparenchymal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_12cadc6af_intraventricular</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_12cadc6af</td>\n",
       "      <td>intraventricular</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_12cadc6af_subarachnoid</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_12cadc6af</td>\n",
       "      <td>subarachnoid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_12cadc6af_subdural</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_12cadc6af</td>\n",
       "      <td>subdural</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              ID  probability           fid             label\n",
       "0          ID_12cadc6af_epidural            0  ID_12cadc6af          epidural\n",
       "1  ID_12cadc6af_intraparenchymal            0  ID_12cadc6af  intraparenchymal\n",
       "2  ID_12cadc6af_intraventricular            0  ID_12cadc6af  intraventricular\n",
       "3      ID_12cadc6af_subarachnoid            0  ID_12cadc6af      subarachnoid\n",
       "4          ID_12cadc6af_subdural            0  ID_12cadc6af          subdural"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove dupes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4516842, 4)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.drop_duplicates('ID', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4516818, 4)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove corrupted image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train[df_train.fid != 'ID_6431af929'] # ID_6431af929"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4516812, 4)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create pivot table with diagnostic labels as columns\n",
    "Generates:\n",
    "* `train_diags.csv` (previously named `train_pivot.csv`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diags = df_train.pivot(index='fid', columns='label', values='probability')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>label</th>\n",
       "      <th>any</th>\n",
       "      <th>epidural</th>\n",
       "      <th>intraparenchymal</th>\n",
       "      <th>intraventricular</th>\n",
       "      <th>subarachnoid</th>\n",
       "      <th>subdural</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ID_000012eaf</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_000039fa0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_00005679d</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_00008ce3c</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_0000950d7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "label         any  epidural  intraparenchymal  intraventricular  subarachnoid  \\\n",
       "fid                                                                             \n",
       "ID_000012eaf    0         0                 0                 0             0   \n",
       "ID_000039fa0    0         0                 0                 0             0   \n",
       "ID_00005679d    0         0                 0                 0             0   \n",
       "ID_00008ce3c    0         0                 0                 0             0   \n",
       "ID_0000950d7    0         0                 0                 0             0   \n",
       "\n",
       "label         subdural  \n",
       "fid                     \n",
       "ID_000012eaf         0  \n",
       "ID_000039fa0         0  \n",
       "ID_00005679d         0  \n",
       "ID_00008ce3c         0  \n",
       "ID_0000950d7         0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(752802, 6)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diags.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diags.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>label</th>\n",
       "      <th>fid</th>\n",
       "      <th>any</th>\n",
       "      <th>epidural</th>\n",
       "      <th>intraparenchymal</th>\n",
       "      <th>intraventricular</th>\n",
       "      <th>subarachnoid</th>\n",
       "      <th>subdural</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000012eaf</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_000039fa0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_00005679d</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_00008ce3c</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_0000950d7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "label           fid  any  epidural  intraparenchymal  intraventricular  \\\n",
       "0      ID_000012eaf    0         0                 0                 0   \n",
       "1      ID_000039fa0    0         0                 0                 0   \n",
       "2      ID_00005679d    0         0                 0                 0   \n",
       "3      ID_00008ce3c    0         0                 0                 0   \n",
       "4      ID_0000950d7    0         0                 0                 0   \n",
       "\n",
       "label  subarachnoid  subdural  \n",
       "0                 0         0  \n",
       "1                 0         0  \n",
       "2                 0         0  \n",
       "3                 0         0  \n",
       "4                 0         0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(752802, 7)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diags.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diags.to_csv(f'data/{stage}_train_diags.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate fastai-ready csv image file (.png) -> labels\n",
    "This is needed for early experiments that worked with the .png dataset.\n",
    "\n",
    "Generates:\n",
    "* `train_labels_as_strings.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "d = defaultdict(list)\n",
    "for fid in df_train.fid.unique(): d[fid]\n",
    "\n",
    "for tup in df_train.itertuples():\n",
    "    if tup.probability: d[tup.fid].append(tup.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "ks, vs = [], []\n",
    "\n",
    "for k, v in d.items():\n",
    "    ks.append(k), vs.append(' '.join(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "fastai_df = pd.DataFrame(data={'fn': ks, 'labels': vs})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(752802, 2)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fastai_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "fastai_df['fn'] += '.png'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fn</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_12cadc6af.png</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_38fd7baa0.png</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_6c5d82413.png</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_aec8e68b3.png</td>\n",
       "      <td>subarachnoid any</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_4d9209c7c.png</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 fn            labels\n",
       "0  ID_12cadc6af.png                  \n",
       "1  ID_38fd7baa0.png                  \n",
       "2  ID_6c5d82413.png                  \n",
       "3  ID_aec8e68b3.png  subarachnoid any\n",
       "4  ID_4d9209c7c.png                  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fastai_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "fastai_df.to_csv(f'data/{stage}_train_labels_as_strings.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tabulate dicom data\n",
    "Generates:\n",
    "* `train_dicom.csv`\n",
    "* `test_dicom.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_i_want = ['BitsAllocated', 'BitsStored', 'Columns', 'HighBit', 'ImageOrientationPatient', \n",
    "               'ImagePositionPatient', 'Modality', 'PatientID', 'PhotometricInterpretation', \n",
    "               'PixelRepresentation', 'PixelSpacing', 'RescaleIntercept', 'RescaleSlope', \n",
    "               'Rows', 'SOPInstanceUID', 'SamplesPerPixel', 'SeriesInstanceUID', 'StudyID', \n",
    "               'StudyInstanceUID', 'WindowCenter', 'WindowWidth']\n",
    "useless_cols = [ 'PixelData' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dcm_list = list(Path(f'data/unzip/{stage}_train_images').glob('*.dcm'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = train_dcm_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0008, 0018) SOP Instance UID                    UI: ID_000012eaf\n",
       "(0008, 0060) Modality                            CS: 'CT'\n",
       "(0010, 0020) Patient ID                          LO: 'ID_f15c0eee'\n",
       "(0020, 000d) Study Instance UID                  UI: ID_30ea2b02d4\n",
       "(0020, 000e) Series Instance UID                 UI: ID_0ab5820b2a\n",
       "(0020, 0010) Study ID                            SH: ''\n",
       "(0020, 0032) Image Position (Patient)            DS: ['-125.000000', '-115.897980', '77.970825']\n",
       "(0020, 0037) Image Orientation (Patient)         DS: ['1.000000', '0.000000', '0.000000', '0.000000', '0.927184', '-0.374607']\n",
       "(0028, 0002) Samples per Pixel                   US: 1\n",
       "(0028, 0004) Photometric Interpretation          CS: 'MONOCHROME2'\n",
       "(0028, 0010) Rows                                US: 512\n",
       "(0028, 0011) Columns                             US: 512\n",
       "(0028, 0030) Pixel Spacing                       DS: ['0.488281', '0.488281']\n",
       "(0028, 0100) Bits Allocated                      US: 16\n",
       "(0028, 0101) Bits Stored                         US: 16\n",
       "(0028, 0102) High Bit                            US: 15\n",
       "(0028, 0103) Pixel Representation                US: 1\n",
       "(0028, 1050) Window Center                       DS: \"30\"\n",
       "(0028, 1051) Window Width                        DS: \"80\"\n",
       "(0028, 1052) Rescale Intercept                   DS: \"-1024\"\n",
       "(0028, 1053) Rescale Slope                       DS: \"1\"\n",
       "(7fe0, 0010) Pixel Data                          OW: Array of 524288 elements"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dicom = pydicom.dcmread(str(f))\n",
    "dicom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ID_f15c0eee'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dicom.data_element('PatientID').value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['-125.000000', '-115.897980', '77.970825']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ipp = dicom.data_element('ImagePositionPatient').value\n",
    "ipp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"-125.000000\""
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ipp[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pydicom.multival.MultiValue"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(ipp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tabulate_dicom_data(file_list):\n",
    "    l = []\n",
    "    for f in file_list:\n",
    "        dicom = pydicom.dcmread(str(f))\n",
    "        d = {}\n",
    "        for s in cols_i_want:\n",
    "            v = dicom.data_element(s).value\n",
    "            if isinstance(v, pydicom.multival.MultiValue):\n",
    "                for i in range(len(v)):\n",
    "                    d[s + '_' + str(i)] = v[i]\n",
    "            else:\n",
    "                d[s] = v\n",
    "        l.append(d)\n",
    "        \n",
    "    return l\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "with ProcessPoolExecutor(max_workers=32) as e:\n",
    "     l = list(itertools.chain.from_iterable(e.map(tabulate_dicom_data, np.array_split(train_dcm_list, 32))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_dicom = pd.DataFrame(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BitsAllocated</th>\n",
       "      <th>BitsStored</th>\n",
       "      <th>Columns</th>\n",
       "      <th>HighBit</th>\n",
       "      <th>ImageOrientationPatient_0</th>\n",
       "      <th>ImageOrientationPatient_1</th>\n",
       "      <th>ImageOrientationPatient_2</th>\n",
       "      <th>ImageOrientationPatient_3</th>\n",
       "      <th>ImageOrientationPatient_4</th>\n",
       "      <th>ImageOrientationPatient_5</th>\n",
       "      <th>...</th>\n",
       "      <th>SamplesPerPixel</th>\n",
       "      <th>SeriesInstanceUID</th>\n",
       "      <th>StudyID</th>\n",
       "      <th>StudyInstanceUID</th>\n",
       "      <th>WindowCenter</th>\n",
       "      <th>WindowCenter_0</th>\n",
       "      <th>WindowCenter_1</th>\n",
       "      <th>WindowWidth</th>\n",
       "      <th>WindowWidth_0</th>\n",
       "      <th>WindowWidth_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>512</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.927184</td>\n",
       "      <td>-0.374607</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_0ab5820b2a</td>\n",
       "      <td></td>\n",
       "      <td>ID_30ea2b02d4</td>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>512</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.968148</td>\n",
       "      <td>-0.250380</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_5f8484c3e0</td>\n",
       "      <td></td>\n",
       "      <td>ID_134d398b61</td>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>512</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_203cd6ec46</td>\n",
       "      <td></td>\n",
       "      <td>ID_b5c26cda09</td>\n",
       "      <td>50.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>12</td>\n",
       "      <td>512</td>\n",
       "      <td>11</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.994522</td>\n",
       "      <td>0.104528</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_3780d48b28</td>\n",
       "      <td></td>\n",
       "      <td>ID_974735bf79</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>512</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_84296c3845</td>\n",
       "      <td></td>\n",
       "      <td>ID_8881b1c4b1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>135.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BitsAllocated  BitsStored  Columns  HighBit  ImageOrientationPatient_0  \\\n",
       "0             16          16      512       15                        1.0   \n",
       "1             16          16      512       15                        1.0   \n",
       "2             16          16      512       15                        1.0   \n",
       "3             16          12      512       11                        1.0   \n",
       "4             16          16      512       15                        1.0   \n",
       "\n",
       "   ImageOrientationPatient_1  ImageOrientationPatient_2  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        0.0                        0.0   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   ImageOrientationPatient_3  ImageOrientationPatient_4  \\\n",
       "0                        0.0                   0.927184   \n",
       "1                        0.0                   0.968148   \n",
       "2                        0.0                   1.000000   \n",
       "3                        0.0                   0.994522   \n",
       "4                        0.0                   1.000000   \n",
       "\n",
       "   ImageOrientationPatient_5  ...  SamplesPerPixel  SeriesInstanceUID  \\\n",
       "0                  -0.374607  ...                1      ID_0ab5820b2a   \n",
       "1                  -0.250380  ...                1      ID_5f8484c3e0   \n",
       "2                   0.000000  ...                1      ID_203cd6ec46   \n",
       "3                   0.104528  ...                1      ID_3780d48b28   \n",
       "4                   0.000000  ...                1      ID_84296c3845   \n",
       "\n",
       "   StudyID StudyInstanceUID WindowCenter WindowCenter_0  WindowCenter_1  \\\n",
       "0             ID_30ea2b02d4         30.0            NaN             NaN   \n",
       "1             ID_134d398b61         30.0            NaN             NaN   \n",
       "2             ID_b5c26cda09         50.0            NaN             NaN   \n",
       "3             ID_974735bf79          NaN           40.0            40.0   \n",
       "4             ID_8881b1c4b1         35.0            NaN             NaN   \n",
       "\n",
       "   WindowWidth  WindowWidth_0  WindowWidth_1  \n",
       "0         80.0            NaN            NaN  \n",
       "1         80.0            NaN            NaN  \n",
       "2        100.0            NaN            NaN  \n",
       "3          NaN           80.0           80.0  \n",
       "4        135.0            NaN            NaN  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_dicom.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_dicom.to_csv(f'data/{stage}_train_dicom.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dcm_list = list(Path(f'data/unzip/{stage}_test_images').glob('*.dcm'))\n",
    "with ProcessPoolExecutor(max_workers=32) as e:\n",
    "     l = list(itertools.chain.from_iterable(e.map(tabulate_dicom_data, np.array_split(test_dcm_list, 32))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BitsAllocated</th>\n",
       "      <th>BitsStored</th>\n",
       "      <th>Columns</th>\n",
       "      <th>HighBit</th>\n",
       "      <th>ImageOrientationPatient_0</th>\n",
       "      <th>ImageOrientationPatient_1</th>\n",
       "      <th>ImageOrientationPatient_2</th>\n",
       "      <th>ImageOrientationPatient_3</th>\n",
       "      <th>ImageOrientationPatient_4</th>\n",
       "      <th>ImageOrientationPatient_5</th>\n",
       "      <th>...</th>\n",
       "      <th>SamplesPerPixel</th>\n",
       "      <th>SeriesInstanceUID</th>\n",
       "      <th>StudyID</th>\n",
       "      <th>StudyInstanceUID</th>\n",
       "      <th>WindowCenter</th>\n",
       "      <th>WindowCenter_0</th>\n",
       "      <th>WindowCenter_1</th>\n",
       "      <th>WindowWidth</th>\n",
       "      <th>WindowWidth_0</th>\n",
       "      <th>WindowWidth_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>12</td>\n",
       "      <td>512</td>\n",
       "      <td>11</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.981627</td>\n",
       "      <td>-0.190809</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_4d28912ba6</td>\n",
       "      <td></td>\n",
       "      <td>ID_1f6d1e8aeb</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>512</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.987688</td>\n",
       "      <td>-0.156434</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_acabdeee86</td>\n",
       "      <td></td>\n",
       "      <td>ID_4a8d7ec19f</td>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>512</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.927184</td>\n",
       "      <td>-0.374607</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_d00cee7f0c</td>\n",
       "      <td></td>\n",
       "      <td>ID_a6ca244172</td>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>512</td>\n",
       "      <td>15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.986286</td>\n",
       "      <td>-0.165048</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_a52a0112d5</td>\n",
       "      <td></td>\n",
       "      <td>ID_fa950a03af</td>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16</td>\n",
       "      <td>12</td>\n",
       "      <td>512</td>\n",
       "      <td>11</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_f552d3b922</td>\n",
       "      <td></td>\n",
       "      <td>ID_965d8b3d8e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BitsAllocated  BitsStored  Columns  HighBit  ImageOrientationPatient_0  \\\n",
       "0             16          12      512       11                        1.0   \n",
       "1             16          16      512       15                        1.0   \n",
       "2             16          16      512       15                        1.0   \n",
       "3             16          16      512       15                        1.0   \n",
       "4             16          12      512       11                        1.0   \n",
       "\n",
       "   ImageOrientationPatient_1  ImageOrientationPatient_2  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        0.0                        0.0   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   ImageOrientationPatient_3  ImageOrientationPatient_4  \\\n",
       "0                        0.0                   0.981627   \n",
       "1                        0.0                   0.987688   \n",
       "2                        0.0                   0.927184   \n",
       "3                        0.0                   0.986286   \n",
       "4                        0.0                   1.000000   \n",
       "\n",
       "   ImageOrientationPatient_5  ...  SamplesPerPixel  SeriesInstanceUID  \\\n",
       "0                  -0.190809  ...                1      ID_4d28912ba6   \n",
       "1                  -0.156434  ...                1      ID_acabdeee86   \n",
       "2                  -0.374607  ...                1      ID_d00cee7f0c   \n",
       "3                  -0.165048  ...                1      ID_a52a0112d5   \n",
       "4                   0.000000  ...                1      ID_f552d3b922   \n",
       "\n",
       "   StudyID StudyInstanceUID WindowCenter WindowCenter_0  WindowCenter_1  \\\n",
       "0             ID_1f6d1e8aeb          NaN           40.0            40.0   \n",
       "1             ID_4a8d7ec19f         30.0            NaN             NaN   \n",
       "2             ID_a6ca244172         30.0            NaN             NaN   \n",
       "3             ID_fa950a03af         30.0            NaN             NaN   \n",
       "4             ID_965d8b3d8e          NaN           36.0            36.0   \n",
       "\n",
       "   WindowWidth  WindowWidth_0  WindowWidth_1  \n",
       "0          NaN           80.0           80.0  \n",
       "1         80.0            NaN            NaN  \n",
       "2         80.0            NaN            NaN  \n",
       "3         80.0            NaN            NaN  \n",
       "4          NaN           80.0           80.0  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_dicom = pd.DataFrame(l)\n",
    "df_test_dicom.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_dicom.to_csv(f'data/{stage}_test_dicom.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add labels to the train dicom csv\n",
    "Generates:\n",
    "* `train_dicom_diags.csv` (previously named `train_dicom_pivot.csv`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_dicom_diags = pd.merge(df_train_dicom, df_diags,  how='left', left_on=['SOPInstanceUID'], right_on = ['fid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(df_train_dicom) == len(df_diags) == len(df_train_dicom_diags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_dicom_diags.to_csv(f'data/{stage}_train_dicom_diags.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}