data-collection.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "import csv\n",
    "import synapseclient\n",
    "from synapseclient import Project, File, Folder, Entity, Link\n",
    "from synapseclient import Schema, Column, Table, Row, RowSet, as_table_columns\n",
    "from synapseclient import Activity, Wiki\n",
    "from synapseclient import Evaluation, Submission, SubmissionStatus"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "connecting to Synapse.org"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome, tomer zamir!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "email = '***************'\n",
    "password ='sMeuN8fNJfMC4Kw'\n",
    "syn = synapseclient.Synapse()\n",
    "syn.login(email,password)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "project = syn.get('syn4993293')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# downloading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "folder=r'C:\\Machine Learning\\vp-ws\\mPower\\voice-data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading 0 files, 1000 cached locally\n",
      "Downloading 0 files, 1000 cached locally\n",
      "Downloading 0 files, 1000 cached locally\n",
      "Downloading 200 files, 800 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n",
      "Downloading 1000 files, 0 cached locally\n"
     ]
    }
   ],
   "source": [
    "for offset in range(0,65000,1000):\n",
    "    results = syn.tableQuery('SELECT * FROM syn5511444 LIMIT 1000 OFFSET '+str(offset))\n",
    "    file_map = syn.downloadTableColumns(results,['audio_audio.m4a'],folder, verbose=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "downloading the table as csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Query the table and download the data as a dataframe\n",
    "table_id = 'syn5511444'\n",
    "query = 'SELECT * FROM %s' % table_id\n",
    "df = syn.tableQuery(query).asDataFrame()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recordId</th>\n",
       "      <th>healthCode</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>appVersion</th>\n",
       "      <th>phoneInfo</th>\n",
       "      <th>audio_audio.m4a</th>\n",
       "      <th>audio_countdown.m4a</th>\n",
       "      <th>medTimepoint</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>72137_9</th>\n",
       "      <td>177d6c17-6635-4884-8481-a671fca06266</td>\n",
       "      <td>639e8a78-3631-4231-bda1-c911c1b169e5</td>\n",
       "      <td>1425904701000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6</td>\n",
       "      <td>5404521</td>\n",
       "      <td>5404533</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72138_9</th>\n",
       "      <td>d59f922b-3ab9-4105-be30-4c2c481e99ed</td>\n",
       "      <td>52fe366a-2a9f-4260-9fb1-0fbc637a6cf4</td>\n",
       "      <td>1425926292000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 5s (GSM)</td>\n",
       "      <td>5410381</td>\n",
       "      <td>5410398</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72139_9</th>\n",
       "      <td>0e85415c-75ee-45d2-8d38-ac7b2c8c5c50</td>\n",
       "      <td>67bdd316-26fc-4fc7-8431-bf9f41a649dd</td>\n",
       "      <td>1425926706000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6 Plus</td>\n",
       "      <td>5390859</td>\n",
       "      <td>5390881</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72140_9</th>\n",
       "      <td>40a9e3a9-1518-4f3f-a9a3-0715233c42a7</td>\n",
       "      <td>45b4e2ca-8d15-4736-828c-829e3d4177f4</td>\n",
       "      <td>1425926937000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6</td>\n",
       "      <td>5701183</td>\n",
       "      <td>5701251</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72141_9</th>\n",
       "      <td>e3b8f800-2c0d-4b42-ba44-f940464dd7b5</td>\n",
       "      <td>45b4e2ca-8d15-4736-828c-829e3d4177f4</td>\n",
       "      <td>1425926960000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6</td>\n",
       "      <td>5750379</td>\n",
       "      <td>5750420</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     recordId  \\\n",
       "72137_9  177d6c17-6635-4884-8481-a671fca06266   \n",
       "72138_9  d59f922b-3ab9-4105-be30-4c2c481e99ed   \n",
       "72139_9  0e85415c-75ee-45d2-8d38-ac7b2c8c5c50   \n",
       "72140_9  40a9e3a9-1518-4f3f-a9a3-0715233c42a7   \n",
       "72141_9  e3b8f800-2c0d-4b42-ba44-f940464dd7b5   \n",
       "\n",
       "                                   healthCode      createdOn  \\\n",
       "72137_9  639e8a78-3631-4231-bda1-c911c1b169e5  1425904701000   \n",
       "72138_9  52fe366a-2a9f-4260-9fb1-0fbc637a6cf4  1425926292000   \n",
       "72139_9  67bdd316-26fc-4fc7-8431-bf9f41a649dd  1425926706000   \n",
       "72140_9  45b4e2ca-8d15-4736-828c-829e3d4177f4  1425926937000   \n",
       "72141_9  45b4e2ca-8d15-4736-828c-829e3d4177f4  1425926960000   \n",
       "\n",
       "                   appVersion        phoneInfo  audio_audio.m4a  \\\n",
       "72137_9  version 1.0, build 7         iPhone 6          5404521   \n",
       "72138_9  version 1.0, build 7  iPhone 5s (GSM)          5410381   \n",
       "72139_9  version 1.0, build 7    iPhone 6 Plus          5390859   \n",
       "72140_9  version 1.0, build 7         iPhone 6          5701183   \n",
       "72141_9  version 1.0, build 7         iPhone 6          5750379   \n",
       "\n",
       "         audio_countdown.m4a                        medTimepoint  \n",
       "72137_9              5404533  I don't take Parkinson medications  \n",
       "72138_9              5410398  I don't take Parkinson medications  \n",
       "72139_9              5390881  I don't take Parkinson medications  \n",
       "72140_9              5701251  I don't take Parkinson medications  \n",
       "72141_9              5750420  I don't take Parkinson medications  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download the entire table as a file\n",
    "table_file = syn.downloadTable(table_id)\n",
    "\n",
    "# Load the data from the file into a Pandas dataframe\n",
    "df = pd.read_csv(table_file)\n",
    "\n",
    "# Save the dataframe to a CSV file\n",
    "df.to_csv('table.csv', index=False)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# mapping the data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "downloading a map to sync between file handles in the csv and file names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                     recordId  \\\n",
      "72137_9  177d6c17-6635-4884-8481-a671fca06266   \n",
      "72138_9  d59f922b-3ab9-4105-be30-4c2c481e99ed   \n",
      "\n",
      "                                   healthCode      createdOn  \\\n",
      "72137_9  639e8a78-3631-4231-bda1-c911c1b169e5  1425904701000   \n",
      "72138_9  52fe366a-2a9f-4260-9fb1-0fbc637a6cf4  1425926292000   \n",
      "\n",
      "                   appVersion        phoneInfo  audio_audio.m4a  \\\n",
      "72137_9  version 1.0, build 7         iPhone 6          5404521   \n",
      "72138_9  version 1.0, build 7  iPhone 5s (GSM)          5410381   \n",
      "\n",
      "         audio_countdown.m4a                        medTimepoint  \n",
      "72137_9              5404533  I don't take Parkinson medications  \n",
      "72138_9              5410398  I don't take Parkinson medications  \n",
      "Downloading 22 files, 65000 cached locally\n"
     ]
    }
   ],
   "source": [
    "# QUERY THE mPower PROJECT (syn4993293) FOR ALL OF THE TABLES\n",
    "tables = syn.getChildren('syn4993293', ['table'])\n",
    "tables = [t for t in tables if t['name'].startswith('Voice')]\n",
    "\n",
    "allData = {table['name']: syn.tableQuery('SELECT * FROM %s' % table['id']) for table in tables}\n",
    "#print(allData)\n",
    "df = allData['Voice Activity'].asDataFrame()\n",
    "print(df.head(2))\n",
    "\n",
    "# return a python dictionary where the keys are the filehandleIds stored in the downloaded column\n",
    "voiceMap = syn.downloadTableColumns(allData['Voice Activity'], \"audio_audio.m4a\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voiceMap type:  <class 'collections.OrderedDict'>\n",
      "voiceMap type:  65022\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print('voiceMap type: ',type(voiceMap))\n",
    "print(print('voiceMap type: ',len(voiceMap)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_handle</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5404521</td>\n",
       "      <td>c:/users/tomer/.synapsecache/521/5404521/audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5410381</td>\n",
       "      <td>c:/users/tomer/.synapsecache/381/5410381/audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5390859</td>\n",
       "      <td>c:/users/tomer/.synapsecache/859/5390859/audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5701183</td>\n",
       "      <td>c:/users/tomer/.synapsecache/183/5701183/audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5750379</td>\n",
       "      <td>c:/users/tomer/.synapsecache/379/5750379/audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65017</th>\n",
       "      <td>6092288</td>\n",
       "      <td>C:\\Users\\tomer\\.synapseCache\\288\\6092288\\audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65018</th>\n",
       "      <td>6092590</td>\n",
       "      <td>C:\\Users\\tomer\\.synapseCache\\590\\6092590\\audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65019</th>\n",
       "      <td>6091758</td>\n",
       "      <td>C:\\Users\\tomer\\.synapseCache\\758\\6091758\\audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65020</th>\n",
       "      <td>6092458</td>\n",
       "      <td>C:\\Users\\tomer\\.synapseCache\\458\\6092458\\audio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65021</th>\n",
       "      <td>6091340</td>\n",
       "      <td>C:\\Users\\tomer\\.synapseCache\\340\\6091340\\audio...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65022 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      file_handle                                          file_name\n",
       "0         5404521  c:/users/tomer/.synapsecache/521/5404521/audio...\n",
       "1         5410381  c:/users/tomer/.synapsecache/381/5410381/audio...\n",
       "2         5390859  c:/users/tomer/.synapsecache/859/5390859/audio...\n",
       "3         5701183  c:/users/tomer/.synapsecache/183/5701183/audio...\n",
       "4         5750379  c:/users/tomer/.synapsecache/379/5750379/audio...\n",
       "...           ...                                                ...\n",
       "65017     6092288  C:\\Users\\tomer\\.synapseCache\\288\\6092288\\audio...\n",
       "65018     6092590  C:\\Users\\tomer\\.synapseCache\\590\\6092590\\audio...\n",
       "65019     6091758  C:\\Users\\tomer\\.synapseCache\\758\\6091758\\audio...\n",
       "65020     6092458  C:\\Users\\tomer\\.synapseCache\\458\\6092458\\audio...\n",
       "65021     6091340  C:\\Users\\tomer\\.synapseCache\\340\\6091340\\audio...\n",
       "\n",
       "[65022 rows x 2 columns]"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "\n",
    "# Extract the keys and values from voiceMap\n",
    "keys = list(voiceMap.keys())\n",
    "values = list(voiceMap.values())\n",
    "\n",
    "# Add the keys and values to the DataFrame as separate columns\n",
    "df['file_handle'] = keys\n",
    "df['file_name'] = values\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64400"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# Define the pattern to match the prefix\n",
    "pattern = r'c:/machine learning/vp-ws/mpower/voice-data/'\n",
    "\n",
    "# Use the `str.contains` method to check if the values in the column matches the pattern\n",
    "mask = df['file_name'].str.contains(pattern)\n",
    "\n",
    "# Use the `sum` function to count the number of rows that match the pattern\n",
    "count = mask.sum()\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[mask] #deleting rows that contain lost files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the prefix to remove\n",
    "prefix = 'c:/machine learning/vp-ws/mpower/voice-data/'\n",
    "df = df.apply(lambda s: s.str.replace(prefix, '', regex=True))\n",
    "suffix = '.tmp'\n",
    "df = df.apply(lambda s: s.str.replace(suffix, '', regex=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_handle</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>600</th>\n",
       "      <td>5392732</td>\n",
       "      <td>audio_audio.m4a-5371dd55-0099-4a63-b6a8-2fd81d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>601</th>\n",
       "      <td>5391081</td>\n",
       "      <td>audio_audio.m4a-8b104d88-57a3-4757-abe0-ab00ea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>602</th>\n",
       "      <td>5388193</td>\n",
       "      <td>audio_audio.m4a-2750d6be-f15f-4615-9f3f-9eab31...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>603</th>\n",
       "      <td>5402960</td>\n",
       "      <td>audio_audio.m4a-136c4ee0-3654-4689-b5fe-e986e6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>5405269</td>\n",
       "      <td>audio_audio.m4a-73f1c8af-a0cc-4f1a-a951-bf623c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    file_handle                                          file_name\n",
       "600     5392732  audio_audio.m4a-5371dd55-0099-4a63-b6a8-2fd81d...\n",
       "601     5391081  audio_audio.m4a-8b104d88-57a3-4757-abe0-ab00ea...\n",
       "602     5388193  audio_audio.m4a-2750d6be-f15f-4615-9f3f-9eab31...\n",
       "603     5402960  audio_audio.m4a-136c4ee0-3654-4689-b5fe-e986e6...\n",
       "604     5405269  audio_audio.m4a-73f1c8af-a0cc-4f1a-a951-bf623c..."
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the dataframe to a CSV file\n",
    "df.to_csv('voiceMap.csv', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# converting tmp to wav"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydub import AudioSegment\n",
    "import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder=r'C:\\Machine Learning\\vp-ws\\mPower\\voice-data'\n",
    "if not os.path.exists('wav_files'):\n",
    "    os.makedirs('wav_files')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in os.listdir(folder):\n",
    "    try:\n",
    "\n",
    "        audio = AudioSegment.from_file(folder + '/' + file)\n",
    "\n",
    "        wav_file = file[:-3] + 'wav'  # Replace the '.tmp' extension with '.wav'\n",
    "        audio.export('wav_files/' + wav_file, format='wav')\n",
    "    except Exception as e:\n",
    "        print(f'Error converting {file}: {e}')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# merging the tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_d = pd.read_csv('diagnosis.csv')\n",
    "df_e = pd.read_csv('else.csv')\n",
    "df_map = pd.read_csv('voiceMap.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged = pd.merge(df_d, df_e, on='healthCode')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROW_ID_x</th>\n",
       "      <th>ROW_VERSION_x</th>\n",
       "      <th>healthCode</th>\n",
       "      <th>professional-diagnosis</th>\n",
       "      <th>ROW_ID_y</th>\n",
       "      <th>ROW_VERSION_y</th>\n",
       "      <th>recordId</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>appVersion</th>\n",
       "      <th>phoneInfo</th>\n",
       "      <th>audio_audio.m4a</th>\n",
       "      <th>audio_countdown.m4a</th>\n",
       "      <th>medTimepoint</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20580</td>\n",
       "      <td>21</td>\n",
       "      <td>639e8a78-3631-4231-bda1-c911c1b169e5</td>\n",
       "      <td>False</td>\n",
       "      <td>72137</td>\n",
       "      <td>9</td>\n",
       "      <td>177d6c17-6635-4884-8481-a671fca06266</td>\n",
       "      <td>1425904701000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6</td>\n",
       "      <td>5404521</td>\n",
       "      <td>5404533</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ROW_ID_x  ROW_VERSION_x                            healthCode  \\\n",
       "0     20580             21  639e8a78-3631-4231-bda1-c911c1b169e5   \n",
       "\n",
       "  professional-diagnosis  ROW_ID_y  ROW_VERSION_y  \\\n",
       "0                  False     72137              9   \n",
       "\n",
       "                               recordId      createdOn            appVersion  \\\n",
       "0  177d6c17-6635-4884-8481-a671fca06266  1425904701000  version 1.0, build 7   \n",
       "\n",
       "  phoneInfo  audio_audio.m4a  audio_countdown.m4a  \\\n",
       "0  iPhone 6          5404521              5404533   \n",
       "\n",
       "                         medTimepoint  \n",
       "0  I don't take Parkinson medications  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged.rename(columns={'audio_audio.m4a': 'file_handle'},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROW_ID_x</th>\n",
       "      <th>ROW_VERSION_x</th>\n",
       "      <th>healthCode</th>\n",
       "      <th>professional-diagnosis</th>\n",
       "      <th>ROW_ID_y</th>\n",
       "      <th>ROW_VERSION_y</th>\n",
       "      <th>recordId</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>appVersion</th>\n",
       "      <th>phoneInfo</th>\n",
       "      <th>file_handle</th>\n",
       "      <th>audio_countdown.m4a</th>\n",
       "      <th>medTimepoint</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20580</td>\n",
       "      <td>21</td>\n",
       "      <td>639e8a78-3631-4231-bda1-c911c1b169e5</td>\n",
       "      <td>False</td>\n",
       "      <td>72137</td>\n",
       "      <td>9</td>\n",
       "      <td>177d6c17-6635-4884-8481-a671fca06266</td>\n",
       "      <td>1425904701000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6</td>\n",
       "      <td>5404521</td>\n",
       "      <td>5404533</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ROW_ID_x  ROW_VERSION_x                            healthCode  \\\n",
       "0     20580             21  639e8a78-3631-4231-bda1-c911c1b169e5   \n",
       "\n",
       "  professional-diagnosis  ROW_ID_y  ROW_VERSION_y  \\\n",
       "0                  False     72137              9   \n",
       "\n",
       "                               recordId      createdOn            appVersion  \\\n",
       "0  177d6c17-6635-4884-8481-a671fca06266  1425904701000  version 1.0, build 7   \n",
       "\n",
       "  phoneInfo  file_handle  audio_countdown.m4a  \\\n",
       "0  iPhone 6      5404521              5404533   \n",
       "\n",
       "                         medTimepoint  \n",
       "0  I don't take Parkinson medications  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged = pd.merge(df_merged, df_map, on='file_handle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROW_ID_x</th>\n",
       "      <th>ROW_VERSION_x</th>\n",
       "      <th>healthCode</th>\n",
       "      <th>professional-diagnosis</th>\n",
       "      <th>ROW_ID_y</th>\n",
       "      <th>ROW_VERSION_y</th>\n",
       "      <th>recordId</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>appVersion</th>\n",
       "      <th>phoneInfo</th>\n",
       "      <th>file_handle</th>\n",
       "      <th>audio_countdown.m4a</th>\n",
       "      <th>medTimepoint</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20582</td>\n",
       "      <td>21</td>\n",
       "      <td>52fe366a-2a9f-4260-9fb1-0fbc637a6cf4</td>\n",
       "      <td>False</td>\n",
       "      <td>83210</td>\n",
       "      <td>10</td>\n",
       "      <td>5c3b56ea-2a8a-4a9b-9d04-271928ff7044</td>\n",
       "      <td>1426297608000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 5s (GSM)</td>\n",
       "      <td>5773552</td>\n",
       "      <td>5773599</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "      <td>audio_audio.m4a-b68fe4c0-db31-4b39-8f8f-7fb874...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ROW_ID_x  ROW_VERSION_x                            healthCode  \\\n",
       "0     20582             21  52fe366a-2a9f-4260-9fb1-0fbc637a6cf4   \n",
       "\n",
       "  professional-diagnosis  ROW_ID_y  ROW_VERSION_y  \\\n",
       "0                  False     83210             10   \n",
       "\n",
       "                               recordId      createdOn            appVersion  \\\n",
       "0  5c3b56ea-2a8a-4a9b-9d04-271928ff7044  1426297608000  version 1.0, build 7   \n",
       "\n",
       "         phoneInfo  file_handle  audio_countdown.m4a  \\\n",
       "0  iPhone 5s (GSM)      5773552              5773599   \n",
       "\n",
       "                         medTimepoint  \\\n",
       "0  I don't take Parkinson medications   \n",
       "\n",
       "                                           file_name  \n",
       "0  audio_audio.m4a-b68fe4c0-db31-4b39-8f8f-7fb874...  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.head(1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "clean the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(62717, 14)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 62717 entries, 0 to 62716\n",
      "Data columns (total 14 columns):\n",
      " #   Column                  Non-Null Count  Dtype \n",
      "---  ------                  --------------  ----- \n",
      " 0   ROW_ID_x                62717 non-null  int64 \n",
      " 1   ROW_VERSION_x           62717 non-null  int64 \n",
      " 2   healthCode              62717 non-null  object\n",
      " 3   professional-diagnosis  62315 non-null  object\n",
      " 4   ROW_ID_y                62717 non-null  int64 \n",
      " 5   ROW_VERSION_y           62717 non-null  int64 \n",
      " 6   recordId                62717 non-null  object\n",
      " 7   createdOn               62717 non-null  int64 \n",
      " 8   appVersion              62717 non-null  object\n",
      " 9   phoneInfo               62717 non-null  object\n",
      " 10  file_handle             62717 non-null  int64 \n",
      " 11  audio_countdown.m4a     62717 non-null  int64 \n",
      " 12  medTimepoint            62495 non-null  object\n",
      " 13  file_name               62717 non-null  object\n",
      "dtypes: int64(7), object(7)\n",
      "memory usage: 7.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df_merged.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, True, nan], dtype=object)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged['professional-diagnosis'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ROW_ID_x                    0\n",
       "ROW_VERSION_x               0\n",
       "healthCode                  0\n",
       "professional-diagnosis    402\n",
       "ROW_ID_y                    0\n",
       "ROW_VERSION_y               0\n",
       "recordId                    0\n",
       "createdOn                   0\n",
       "appVersion                  0\n",
       "phoneInfo                   0\n",
       "file_handle                 0\n",
       "audio_countdown.m4a         0\n",
       "medTimepoint              222\n",
       "file_name                   0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged = df_merged.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(62096, 14)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4763"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of subjects\n",
    "df_merged['healthCode'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged['professional-diagnosis']=df_merged['professional-diagnosis'].astype(bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39283"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of subjects with PD\n",
    "df_merged['professional-diagnosis'].eq(True).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22813"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of control subjects\n",
    "df_merged['professional-diagnosis'].eq(False).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROW_ID_x</th>\n",
       "      <th>ROW_VERSION_x</th>\n",
       "      <th>healthCode</th>\n",
       "      <th>professional-diagnosis</th>\n",
       "      <th>ROW_ID_y</th>\n",
       "      <th>ROW_VERSION_y</th>\n",
       "      <th>recordId</th>\n",
       "      <th>createdOn</th>\n",
       "      <th>appVersion</th>\n",
       "      <th>phoneInfo</th>\n",
       "      <th>file_handle</th>\n",
       "      <th>audio_countdown.m4a</th>\n",
       "      <th>medTimepoint</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20582</td>\n",
       "      <td>21</td>\n",
       "      <td>52fe366a-2a9f-4260-9fb1-0fbc637a6cf4</td>\n",
       "      <td>False</td>\n",
       "      <td>83210</td>\n",
       "      <td>10</td>\n",
       "      <td>5c3b56ea-2a8a-4a9b-9d04-271928ff7044</td>\n",
       "      <td>1426297608000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 5s (GSM)</td>\n",
       "      <td>5773552</td>\n",
       "      <td>5773599</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "      <td>audio_audio.m4a-b68fe4c0-db31-4b39-8f8f-7fb874...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20582</td>\n",
       "      <td>21</td>\n",
       "      <td>52fe366a-2a9f-4260-9fb1-0fbc637a6cf4</td>\n",
       "      <td>False</td>\n",
       "      <td>83211</td>\n",
       "      <td>10</td>\n",
       "      <td>9a3d4bd0-a6a6-4218-a442-4bdf4785e953</td>\n",
       "      <td>1426297632000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 5s (GSM)</td>\n",
       "      <td>5805736</td>\n",
       "      <td>5805781</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "      <td>audio_audio.m4a-3e719e45-187b-4cbb-8ce1-8ccf37...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20594</td>\n",
       "      <td>21</td>\n",
       "      <td>26ce0a95-9946-49f5-b3db-2f095ac9fcd4</td>\n",
       "      <td>False</td>\n",
       "      <td>72815</td>\n",
       "      <td>9</td>\n",
       "      <td>e9fb4ed1-0b62-4ac4-a271-b8b140b696e7</td>\n",
       "      <td>1425939825000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6</td>\n",
       "      <td>5400756</td>\n",
       "      <td>5400772</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "      <td>audio_audio.m4a-579f1a37-3edd-4df7-b80b-bffc14...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20595</td>\n",
       "      <td>21</td>\n",
       "      <td>a9b28b95-b9ee-4442-98c9-1cf9b8cf4a36</td>\n",
       "      <td>False</td>\n",
       "      <td>76066</td>\n",
       "      <td>9</td>\n",
       "      <td>093c7c85-6476-49e1-8047-4823d47a5356</td>\n",
       "      <td>1426015917000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6 Plus</td>\n",
       "      <td>5565223</td>\n",
       "      <td>5565307</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "      <td>audio_audio.m4a-eeee8102-596c-4030-8f77-b40460...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20595</td>\n",
       "      <td>21</td>\n",
       "      <td>a9b28b95-b9ee-4442-98c9-1cf9b8cf4a36</td>\n",
       "      <td>False</td>\n",
       "      <td>76068</td>\n",
       "      <td>9</td>\n",
       "      <td>43ca6e05-be58-4e13-84f3-1536ef491a7e</td>\n",
       "      <td>1426015942000</td>\n",
       "      <td>version 1.0, build 7</td>\n",
       "      <td>iPhone 6 Plus</td>\n",
       "      <td>5458885</td>\n",
       "      <td>5458962</td>\n",
       "      <td>I don't take Parkinson medications</td>\n",
       "      <td>audio_audio.m4a-4374d36b-4020-4e7a-860b-44aadf...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ROW_ID_x  ROW_VERSION_x                            healthCode  \\\n",
       "0     20582             21  52fe366a-2a9f-4260-9fb1-0fbc637a6cf4   \n",
       "1     20582             21  52fe366a-2a9f-4260-9fb1-0fbc637a6cf4   \n",
       "2     20594             21  26ce0a95-9946-49f5-b3db-2f095ac9fcd4   \n",
       "3     20595             21  a9b28b95-b9ee-4442-98c9-1cf9b8cf4a36   \n",
       "4     20595             21  a9b28b95-b9ee-4442-98c9-1cf9b8cf4a36   \n",
       "\n",
       "   professional-diagnosis  ROW_ID_y  ROW_VERSION_y  \\\n",
       "0                   False     83210             10   \n",
       "1                   False     83211             10   \n",
       "2                   False     72815              9   \n",
       "3                   False     76066              9   \n",
       "4                   False     76068              9   \n",
       "\n",
       "                               recordId      createdOn            appVersion  \\\n",
       "0  5c3b56ea-2a8a-4a9b-9d04-271928ff7044  1426297608000  version 1.0, build 7   \n",
       "1  9a3d4bd0-a6a6-4218-a442-4bdf4785e953  1426297632000  version 1.0, build 7   \n",
       "2  e9fb4ed1-0b62-4ac4-a271-b8b140b696e7  1425939825000  version 1.0, build 7   \n",
       "3  093c7c85-6476-49e1-8047-4823d47a5356  1426015917000  version 1.0, build 7   \n",
       "4  43ca6e05-be58-4e13-84f3-1536ef491a7e  1426015942000  version 1.0, build 7   \n",
       "\n",
       "         phoneInfo  file_handle  audio_countdown.m4a  \\\n",
       "0  iPhone 5s (GSM)      5773552              5773599   \n",
       "1  iPhone 5s (GSM)      5805736              5805781   \n",
       "2         iPhone 6      5400756              5400772   \n",
       "3    iPhone 6 Plus      5565223              5565307   \n",
       "4    iPhone 6 Plus      5458885              5458962   \n",
       "\n",
       "                         medTimepoint  \\\n",
       "0  I don't take Parkinson medications   \n",
       "1  I don't take Parkinson medications   \n",
       "2  I don't take Parkinson medications   \n",
       "3  I don't take Parkinson medications   \n",
       "4  I don't take Parkinson medications   \n",
       "\n",
       "                                           file_name  \n",
       "0  audio_audio.m4a-b68fe4c0-db31-4b39-8f8f-7fb874...  \n",
       "1  audio_audio.m4a-3e719e45-187b-4cbb-8ce1-8ccf37...  \n",
       "2  audio_audio.m4a-579f1a37-3edd-4df7-b80b-bffc14...  \n",
       "3  audio_audio.m4a-eeee8102-596c-4030-8f77-b40460...  \n",
       "4  audio_audio.m4a-4374d36b-4020-4e7a-860b-44aadf...  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged.to_csv('merged.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing unnecessary columns - leaving a table for the diagnosis of each recording\n",
    "df = df_merged[['professional-diagnosis', 'file_name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('mPower_data.csv', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# deleting audio-files with no diagnosis"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "adding '.wav' as a suffix to each file-name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'recordings-diagnosis.csv'\n",
    "df = pd.read_csv(file)\n",
    "df['file_name'] = df['file_name'] + '.wav'\n",
    "df.to_csv(file, index=False)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "checking and deleting files that do not exist in the recordings-diagnosis csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " # Setting the path to the directory containing the files we want to delete\n",
    "directory = 'C:\\\\Machine Learning\\\\vp-ws\\\\mPower\\\\wav_files'\n",
    "\n",
    "# Setting the path to the CSV file\n",
    "csv_file = 'C:\\\\Machine Learning\\\\vp-ws\\\\mPower\\\\recordings-diagnosis.csv'\n",
    "\n",
    "# Read the CSV file and store the names in a list\n",
    "with open(csv_file, 'r') as file:\n",
    "    reader = csv.reader(file)\n",
    "    names = [row[1] for row in reader]\n",
    "# Iterate over the files in the directory\n",
    "for filename in os.listdir(directory):\n",
    "    # Check if the file name is not in the list of names\n",
    "    if filename not in names:\n",
    "        # Construct the full file path\n",
    "        file_path = os.path.join(directory, filename)\n",
    "        # Delete the file\n",
    "        os.remove(file_path)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "removing rows of files that do not exist in the folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open the input CSV file and create a CSV reader\n",
    "with open(csv_file, 'r') as input_csv:\n",
    "    reader = csv.reader(input_csv)\n",
    "    \n",
    "    # Get the column names from the first row of the input CSV file\n",
    "    column_names = next(reader)\n",
    "    \n",
    "    # Open the output CSV file and create a CSV writer\n",
    "    with open('output.csv', 'w', newline='') as output_csv:\n",
    "        writer = csv.writer(output_csv)\n",
    "        \n",
    "        # Write the column names to the output CSV file\n",
    "        writer.writerow(column_names)\n",
    "        \n",
    "        # Iterate over the rows in the input CSV file\n",
    "        for row in reader:\n",
    "            # Check if the file corresponding to the row exists in the directory\n",
    "            file_path = os.path.join(directory, row[1])\n",
    "            if os.path.exists(file_path):\n",
    "                # Write the row to the output CSV file if the file exists\n",
    "                writer.writerow(row)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4506fe570a6538beb6869ac47d4d2d17f3a19d2d7f0538e6472cca21b669a493"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}