UCSF_Demographics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4213f5c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import regex as re\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ad0621a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "372716"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "UCSF_reports = pd.read_csv('data/raw/secure_UCSF_radreports__CT_MR_MRI_US_PET__01-01-2021__10-22-2022.csv')\n",
    "UCSF_reports = UCSF_reports[UCSF_reports['Modality'].isin(['CT', 'MRI', 'US'])].reset_index(drop=True)\n",
    "len(UCSF_reports)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72f64986",
   "metadata": {},
   "source": [
    "## Exclude Studies "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7c179a73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 6\n"
     ]
    }
   ],
   "source": [
    "# Reports with same accessions\n",
    "UCSF_duplicates = UCSF_reports[UCSF_reports.duplicated(subset='Accession Number', keep=False)]\n",
    "UCSF_reports = UCSF_reports.drop_duplicates(subset='Accession Number', keep=False).reset_index(drop=True)\n",
    "print('n =', len(UCSF_duplicates))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0cf75e90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 15803\n"
     ]
    }
   ],
   "source": [
    "# Reports that are non-reportable studies \n",
    "UCSF_nonreportable_reports = UCSF_reports[UCSF_reports['Report Text'].str.contains('non-reportable study')].reset_index(drop=True)\n",
    "UCSF_reports = UCSF_reports[~UCSF_reports['Report Text'].str.contains('non-reportable study')].reset_index(drop=True)\n",
    "print('n =', len(UCSF_nonreportable_reports))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "394eda5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 715\n"
     ]
    }
   ],
   "source": [
    "# Reports located in progress notes\n",
    "OTHER_SOURCES_PATTERNS = [\n",
    "    'Refer to procedure/progress notes',\n",
    "    'See clinic note for results'\n",
    "]\n",
    "\n",
    "pattern = '|'.join([re.escape(p) for p in OTHER_SOURCES_PATTERNS])\n",
    "\n",
    "# Filtering based on the pattern\n",
    "UCSF_relocated_reports = UCSF_reports[UCSF_reports['Report Text'].str.contains(pattern, regex=True)].reset_index(drop=True)\n",
    "UCSF_reports = UCSF_reports[~UCSF_reports['Report Text'].str.contains(pattern, regex=True)].reset_index(drop=True)\n",
    "print('n =', len(UCSF_relocated_reports))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6a90ccf3",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 2912\n"
     ]
    }
   ],
   "source": [
    "# Reports with no separate findings and impressions\n",
    "def extract_impression(report_text):\n",
    "    pattern = r'SUMMARY(.*)'\n",
    "    match = re.search(pattern, report_text, re.IGNORECASE | re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(1).strip()\n",
    "    \n",
    "    pattern = r'SUMMARY:(.*?)END OF IMPRESSION'\n",
    "    match = re.search(pattern, report_text, re.IGNORECASE | re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(1).strip()\n",
    "    \n",
    "    pattern = r'Impression:(.*?)END OF IMPRESSION'\n",
    "    match = re.search(pattern, report_text, re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(1).strip()\n",
    "    \n",
    "    pattern = r'IMPRESSION(.*?)Report dictated by'\n",
    "    match = re.search(pattern, report_text, re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(1).strip()\n",
    "    \n",
    "    sections = report_text.split('\\n\\n')\n",
    "    if len(sections) == 2 or len(sections) == 3:\n",
    "        return sections[1]\n",
    "    \n",
    "    sections = report_text.split('\\n\"\\n')\n",
    "    if len(sections) == 2:\n",
    "        return sections[1]\n",
    "    \n",
    "    sections = report_text.split('\\n')\n",
    "    if len(sections) == 2:\n",
    "        return sections[1]   \n",
    "    \n",
    "    pattern = r'IMPRESSION:(.*?)Report dictated by'\n",
    "    match = re.search(pattern, report_text, re.IGNORECASE | re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(1).strip()\n",
    "    \n",
    "    pattern = r'IMPRESSION:(.*?)-+'\n",
    "    match = re.search(pattern, report_text, re.IGNORECASE | re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(0).strip()\n",
    "    \n",
    "    return ''\n",
    "\n",
    "def clean_text(text):\n",
    "    if '//ALERT//' in text and len(text.split('\\n')) > 1:\n",
    "        text = text.split('\\n')[1]\n",
    "    text = re.sub(r'Report dictated by:.*$', '', text, flags=re.DOTALL).strip()\n",
    "    text = re.sub(r'\\nThis document is.*$', '', text, flags=re.DOTALL).strip()\n",
    "    text = re.sub(r'//.*$', '', text, flags=re.DOTALL).strip()\n",
    "    text = (text.strip()\n",
    "        .replace('FINDINGS/IMPRESSION', '')\n",
    "        .replace('Findings/impression', '')\n",
    "        .replace('IMPRESSION: \\n', '')\n",
    "        .replace(':\\n', '')\n",
    "        .replace(':', '')\n",
    "        .replace('IMPRESSION:', '')\n",
    "        .replace('IMPRESSION', '')\n",
    "        .replace('Impression', ''))\n",
    "    return text\n",
    "\n",
    "def extract_source(report_text, impression):\n",
    "    return report_text.replace(impression, '').strip()\n",
    "\n",
    "UCSF_reports['Impression'] = UCSF_reports['Report Text'].apply(extract_impression)\n",
    "UCSF_reports['Processed Impression'] = UCSF_reports['Impression'].apply(clean_text)\n",
    "UCSF_reports['Processed Source'] = UCSF_reports.apply(\n",
    "    lambda x: extract_source(x['Report Text'], x['Impression']), axis=1\n",
    ")\n",
    "\n",
    "UCSF_no_separate_impression = UCSF_reports[UCSF_reports['Processed Impression'] == '']\n",
    "UCSF_reports = UCSF_reports[UCSF_reports['Processed Impression'] != ''].reset_index(drop=True)\n",
    "print('n =', len(UCSF_no_separate_impression))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69b8a364",
   "metadata": {},
   "source": [
    "## Splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4ddcfcd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "353280"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(UCSF_reports)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6f3fb30c",
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_mrns = list(set(UCSF_reports['Patient MRN']))\n",
    "train_mrns, val_test_mrns = train_test_split(patient_mrns, test_size=0.2, random_state=123)\n",
    "val_mrns, test_mrns = train_test_split(val_test_mrns, test_size=0.5, random_state=123)\n",
    "\n",
    "UCSF_reports_train = UCSF_reports[UCSF_reports['Patient MRN'].isin(train_mrns)]\n",
    "UCSF_reports_val = UCSF_reports[UCSF_reports['Patient MRN'].isin(val_mrns)]\n",
    "UCSF_reports_test = UCSF_reports[UCSF_reports['Patient MRN'].isin(test_mrns)]\n",
    "\n",
    "UCSF_reports_dict = {\n",
    "    'train': UCSF_reports_train,\n",
    "    'val': UCSF_reports_val,\n",
    "    'test': UCSF_reports_test\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b0c5fc6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "127716"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(patient_mrns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d9f47e42",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================\n",
      "train\n",
      "--------------------\n",
      "n = 282525\n",
      "102172 patients\n",
      "====================\n",
      "val\n",
      "--------------------\n",
      "n = 35631\n",
      "12772 patients\n",
      "====================\n",
      "test\n",
      "--------------------\n",
      "n = 35124\n",
      "12772 patients\n"
     ]
    }
   ],
   "source": [
    "for dataset in UCSF_reports_dict:\n",
    "    print('='*20)\n",
    "    print(dataset)\n",
    "    print('-'*20)\n",
    "    print('n =', len(UCSF_reports_dict[dataset]))\n",
    "    print(len(set(UCSF_reports_dict[dataset]['Patient MRN'])), 'patients')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3a39358b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Age (y)\n",
      "====================\n",
      "train\n",
      "--------------------\n",
      "51.188826581718445 +/- 22.835700204281263\n",
      "====================\n",
      "val\n",
      "--------------------\n",
      "50.65542168336562 +/- 22.745595264243146\n",
      "====================\n",
      "test\n",
      "--------------------\n",
      "51.21868266712219 +/- 22.89266057646915\n"
     ]
    }
   ],
   "source": [
    "print('Age (y)')\n",
    "for dataset in UCSF_reports_dict:\n",
    "    print('='*20)\n",
    "    print(dataset)\n",
    "    print('-'*20)\n",
    "    print(UCSF_reports_dict[dataset]['Patient Age'].mean(), '+/-', UCSF_reports_dict[dataset]['Patient Age'].std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e94c0bb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Patient Sex\n",
      "====================\n",
      "train\n",
      "--------------------\n",
      "Patient Sex\n",
      "Female    54.491461\n",
      "Male      45.388904\n",
      "U          0.119635\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "val\n",
      "--------------------\n",
      "Patient Sex\n",
      "Female    53.745334\n",
      "Male      46.150824\n",
      "U          0.103842\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "test\n",
      "--------------------\n",
      "Patient Sex\n",
      "Female    54.313290\n",
      "Male      45.578522\n",
      "U          0.108188\n",
      "Name: proportion, dtype: float64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3747228/125900428.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Patient Sex'] = UCSF_reports_dict[dataset]['Patient Sex'].fillna('U')\n",
      "/tmp/ipykernel_3747228/125900428.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Patient Sex'] = UCSF_reports_dict[dataset]['Patient Sex'].fillna('U')\n",
      "/tmp/ipykernel_3747228/125900428.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Patient Sex'] = UCSF_reports_dict[dataset]['Patient Sex'].fillna('U')\n"
     ]
    }
   ],
   "source": [
    "print('Patient Sex')\n",
    "for dataset in UCSF_reports_dict:\n",
    "    print('='*20)\n",
    "    print(dataset)\n",
    "    print('-'*20)\n",
    "    UCSF_reports_dict[dataset]['Patient Sex'] = UCSF_reports_dict[dataset]['Patient Sex'].fillna('U')\n",
    "    print(UCSF_reports_dict[dataset]['Patient Sex'].value_counts(normalize=True)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "efd452b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Modality\n",
      "====================\n",
      "train\n",
      "--------------------\n",
      "Modality\n",
      "CT     42.332537\n",
      "MRI    30.064242\n",
      "US     27.603221\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "val\n",
      "--------------------\n",
      "Modality\n",
      "CT     42.266566\n",
      "MRI    30.128259\n",
      "US     27.605175\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "test\n",
      "--------------------\n",
      "Modality\n",
      "CT     42.164901\n",
      "MRI    30.437877\n",
      "US     27.397221\n",
      "Name: proportion, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('Modality')\n",
    "for dataset in UCSF_reports_dict:\n",
    "    print('='*20)\n",
    "    print(dataset)\n",
    "    print('-'*20)\n",
    "    print(UCSF_reports_dict[dataset]['Modality'].value_counts(normalize=True)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cddd15e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Patient Status\n",
      "====================\n",
      "train\n",
      "--------------------\n",
      "Patient Status\n",
      "Outpatient    0.647125\n",
      "Inpatient     0.209288\n",
      "Emergency     0.120035\n",
      "Other         0.023552\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "val\n",
      "--------------------\n",
      "Patient Status\n",
      "Outpatient    0.639920\n",
      "Inpatient     0.213690\n",
      "Emergency     0.122169\n",
      "Other         0.024220\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "test\n",
      "--------------------\n",
      "Patient Status\n",
      "Outpatient    0.649328\n",
      "Inpatient     0.210711\n",
      "Emergency     0.118267\n",
      "Other         0.021695\n",
      "Name: proportion, dtype: float64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3747228/2375280094.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Patient Status'] = UCSF_reports_dict[dataset]['Patient Status'].fillna('Other')\n",
      "/tmp/ipykernel_3747228/2375280094.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Patient Status'] = UCSF_reports_dict[dataset]['Patient Status'].fillna('Other')\n",
      "/tmp/ipykernel_3747228/2375280094.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Patient Status'] = UCSF_reports_dict[dataset]['Patient Status'].fillna('Other')\n"
     ]
    }
   ],
   "source": [
    "print('Patient Status')\n",
    "for dataset in UCSF_reports_dict:\n",
    "    print('='*20)\n",
    "    print(dataset)\n",
    "    print('-'*20)\n",
    "    UCSF_reports_dict[dataset]['Patient Status'] = UCSF_reports_dict[dataset]['Patient Status'].fillna('Other')\n",
    "    print(UCSF_reports_dict[dataset]['Patient Status'].value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "46da4fd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Stat\n",
      "====================\n",
      "train\n",
      "--------------------\n",
      "Is Stat\n",
      "False    99.957172\n",
      "True      0.042828\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "val\n",
      "--------------------\n",
      "Is Stat\n",
      "False    99.952289\n",
      "True      0.047711\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "test\n",
      "--------------------\n",
      "Is Stat\n",
      "False    99.945906\n",
      "True      0.054094\n",
      "Name: proportion, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('Is Stat')\n",
    "for dataset in UCSF_reports_dict:\n",
    "    print('='*20)\n",
    "    print(dataset)\n",
    "    print('-'*20)\n",
    "    print(UCSF_reports_dict[dataset]['Is Stat'].value_counts(normalize=True)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "95eec056",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Body Part Imaged\n",
      "====================\n",
      "train\n",
      "--------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3747228/2226351875.py:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Body Part Imaged'] = UCSF_reports_dict[dataset]['Exam Description'].apply(label_body_part)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Body Part Imaged\n",
      "ABDOMEN/PELVIS    25.584992\n",
      "BRAIN             18.058225\n",
      "OTHER             15.274754\n",
      "CHEST             13.626051\n",
      "SPINE              8.445624\n",
      "NECK               4.721706\n",
      "RENAL/KIDNEY       4.224759\n",
      "EXTREMITY          4.069020\n",
      "PROSTATE           1.331918\n",
      "BREAST             1.159897\n",
      "KNEE               1.153880\n",
      "LIVER              0.897974\n",
      "HIP                0.796744\n",
      "HEART              0.510397\n",
      "HEAD               0.144058\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "val\n",
      "--------------------\n",
      "Body Part Imaged\n",
      "ABDOMEN/PELVIS    25.660240\n",
      "BRAIN             18.175185\n",
      "OTHER             14.961691\n",
      "CHEST             13.583677\n",
      "SPINE              8.607673\n",
      "NECK               4.745867\n",
      "RENAL/KIDNEY       4.164913\n",
      "EXTREMITY          4.027392\n",
      "PROSTATE           1.389240\n",
      "KNEE               1.097359\n",
      "BREAST             1.094552\n",
      "LIVER              0.898094\n",
      "HIP                0.839157\n",
      "HEART              0.541663\n",
      "HEAD               0.213297\n",
      "Name: proportion, dtype: float64\n",
      "====================\n",
      "test\n",
      "--------------------\n",
      "Body Part Imaged\n",
      "ABDOMEN/PELVIS    25.819952\n",
      "BRAIN             17.816877\n",
      "OTHER             15.029609\n",
      "CHEST             13.631705\n",
      "SPINE              8.731921\n",
      "NECK               4.657784\n",
      "EXTREMITY          4.162396\n",
      "RENAL/KIDNEY       4.116843\n",
      "PROSTATE           1.321034\n",
      "BREAST             1.224234\n",
      "KNEE               1.135975\n",
      "LIVER              0.942376\n",
      "HIP                0.771552\n",
      "HEART              0.444141\n",
      "HEAD               0.193600\n",
      "Name: proportion, dtype: float64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3747228/2226351875.py:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Body Part Imaged'] = UCSF_reports_dict[dataset]['Exam Description'].apply(label_body_part)\n",
      "/tmp/ipykernel_3747228/2226351875.py:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  UCSF_reports_dict[dataset]['Body Part Imaged'] = UCSF_reports_dict[dataset]['Exam Description'].apply(label_body_part)\n"
     ]
    }
   ],
   "source": [
    "def label_body_part(exam_description):\n",
    "    if 'ABDOMEN' in exam_description or 'PELVIS' in exam_description:\n",
    "        return 'ABDOMEN/PELVIS'\n",
    "    elif 'CHEST' in exam_description:\n",
    "        return 'CHEST'\n",
    "    elif 'BRAIN' in exam_description:\n",
    "        return 'BRAIN'\n",
    "    elif 'NECK' in exam_description:\n",
    "        return 'NECK'\n",
    "    elif 'SPINE' in exam_description or 'SPINAL' in exam_description:\n",
    "        return 'SPINE'\n",
    "    elif 'EXTREMITY' in exam_description:\n",
    "        return 'EXTREMITY'\n",
    "    elif 'KIDNEY' in exam_description or 'RENAL' in exam_description:\n",
    "        return 'RENAL/KIDNEY'\n",
    "    elif 'BREAST' in exam_description:\n",
    "        return 'BREAST'\n",
    "    elif 'LIVER' in exam_description:\n",
    "        return 'LIVER'\n",
    "    elif 'PROSTATE' in exam_description:\n",
    "        return 'PROSTATE'  \n",
    "    elif 'KNEE' in exam_description:\n",
    "        return 'KNEE'  \n",
    "    elif 'HEAD' in exam_description:\n",
    "        return 'HEAD'\n",
    "    elif 'HIP' in exam_description:\n",
    "        return 'HIP'\n",
    "    elif 'HEART' in exam_description:\n",
    "        return 'HEART'\n",
    "    \n",
    "    return 'OTHER'\n",
    "    \n",
    "UCSF_reports['Body Part Imaged'] = UCSF_reports['Exam Description'].apply(label_body_part)\n",
    "\n",
    "print('Body Part Imaged')\n",
    "for dataset in UCSF_reports_dict:\n",
    "    print('='*20)\n",
    "    print(dataset)\n",
    "    print('-'*20)\n",
    "    UCSF_reports_dict[dataset]['Body Part Imaged'] = UCSF_reports_dict[dataset]['Exam Description'].apply(label_body_part)\n",
    "    print(UCSF_reports_dict[dataset]['Body Part Imaged'].value_counts(normalize=True)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e594be7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in UCSF_reports_dict:\n",
    "    UCSF_reports_dict[dataset].to_csv(f'data/processed/UCSF_{dataset}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7bd3605",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "report_gen",
   "language": "python",
   "name": "report_gen"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}