Convert to Photon-HDF5 48-spot smFRET.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dir_ = 'data/'\n",
    "fname = dir_ + 'dsDNA_-8TA+10NTD_700pM_1Dto20Adilut_600mW_conf1_reall_5.dat'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_name = 'dsDNA TA-8_NTD+10'\n",
    "description = '48-spot smFRET measurement of dsDNA.'\n",
    "dye_names = 'ATTO550, ATTO647N'\n",
    "buffer_name = 'TE50'\n",
    "\n",
    "author = 'Antonino Ingargiola'\n",
    "author_affiliation = 'UCLA'\n",
    "creator = 'Antonino Ingargiola'\n",
    "creator_affiliation = 'UCLA'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.7.3+15.ged8ec9f'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import tables\n",
    "from tqdm import tqdm_notebook, tnrange  # progress bar\n",
    "\n",
    "import phconvert as phc\n",
    "phc.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import niconverter as nic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess spot data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t_start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('data/dsDNA_-8TA+10NTD_700pM_1Dto20Adilut_600mW_conf1_reall_5.dat',\n",
       " {'acquisition_duration': 46.045105437499998,\n",
       "  'clock_frequency': 80000000.0,\n",
       "  'format': 'x',\n",
       "  'nchannels': 96,\n",
       "  'num_timestamps': 75626617,\n",
       "  'offset': 65,\n",
       "  'words_per_photon': 1})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h5file, meta = nic.ni96ch_process_spots(fname, close=True)\n",
    "ts_unit = 1 / meta['clock_frequency']\n",
    "fname, meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39.522247076034546"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.time() - t_start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('data/dsDNA_-8TA+10NTD_700pM_1Dto20Adilut_600mW_conf1_reall_5.dat',\n",
       " {'acquisition_duration': 46.045105437499998,\n",
       "  'clock_frequency': 80000000.0,\n",
       "  'format': 'x',\n",
       "  'nchannels': 96,\n",
       "  'num_timestamps': 75626617,\n",
       "  'offset': 65,\n",
       "  'words_per_photon': 1})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h5file = tables.open_file(fname + '_raw_temp_spots.hdf5', mode='a')\n",
    "dt, endianess, meta = nic._detectformat(fname)\n",
    "ts_unit = 1 / meta['clock_frequency']\n",
    "t2, aem2 = nic.get_photon_data_arr(h5file, spots=np.arange(48))\n",
    "meta['acquisition_duration'] = nic._duration(t2, ts_unit)\n",
    "fname, meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40.350224018096924"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.time() - t_start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Measurement duration: 46.0 s\n",
      "Transfer rate: 6.27 MB/s\n"
     ]
    }
   ],
   "source": [
    "print('Measurement duration: %.1f s' %  meta['acquisition_duration'])\n",
    "print('Transfer rate: %.2f MB/s' % (meta['num_timestamps'] / meta['acquisition_duration'] * 4 / (1024**2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "neg_jumps = []\n",
    "for tx in tqdm_notebook(t2):\n",
    "    d = np.diff(tx[:])\n",
    "    assert (d >= 0).all()\n",
    "    neg_jumps.append(d[d<0])\n",
    "    \n",
    "missed_ov_jumps = [len(j) for j in neg_jumps]\n",
    "for ch, jumps in enumerate(missed_ov_jumps):\n",
    "    if jumps > 0:\n",
    "        print('CH%d: %s' % (ch, jumps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44.00565195083618"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.time() - t_start"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Create Photon-HDF5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acquisition_duration': 46.045105437499998,\n",
       " 'description': '48-spot smFRET measurement of dsDNA.',\n",
       " 'identity': {'author': 'Antonino Ingargiola',\n",
       "  'author_affiliation': 'UCLA',\n",
       "  'creator': 'Antonino Ingargiola',\n",
       "  'creator_affiliation': 'UCLA'},\n",
       " 'sample': {'buffer_name': 'TE50',\n",
       "  'dye_names': 'ATTO550, ATTO647N',\n",
       "  'num_dyes': 2,\n",
       "  'sample_name': 'dsDNA TA-8_NTD+10'}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = dict(\n",
    "    description = description,\n",
    "    acquisition_duration = meta['acquisition_duration'],\n",
    "\n",
    "    sample = dict(\n",
    "        sample_name=sample_name,\n",
    "        dye_names=dye_names,\n",
    "        buffer_name=buffer_name,\n",
    "        num_dyes = len(dye_names.split(','))),\n",
    "\n",
    "    identity = dict(\n",
    "        author=author,\n",
    "        author_affiliation=author_affiliation,\n",
    "        creator=creator,\n",
    "        creator_affiliation=creator_affiliation),\n",
    "    )\n",
    "metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/dsDNA_-8TA+10NTD_700pM_1Dto20Adilut_600mW_conf1_reall_5.dat'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/dsDNA_-8TA+10NTD_700pM_1Dto20Adilut_600mW_conf1_reall_5.dat_raw_temp_spots.hdf5'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h5file.filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = nic.create_ph5data_smFRET_48spots(fname, h5file, ts_unit, metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['_filename',\n",
       " 'acquisition_duration',\n",
       " 'description',\n",
       " 'sample',\n",
       " 'identity',\n",
       " 'setup',\n",
       " 'provenance']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[f for f in data.keys() if not f.startswith('photon')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving: data/dsDNA_-8TA+10NTD_700pM_1Dto20Adilut_600mW_conf1_reall_5.dat_raw_temp_spots.hdf5\n",
      "1 loop, best of 1: 1.25 s per loop\n"
     ]
    }
   ],
   "source": [
    "%%timeit -n1 -r1\n",
    "phc.hdf5.save_photon_hdf5(data, h5file=h5file, overwrite=True, close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45.34969210624695"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.time() - t_start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'stop' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-1975202a44b3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'stop' is not defined"
     ]
    }
   ],
   "source": [
    "stop "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if data['_data_file'].isopen:\n",
    "    data['_data_file'].close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "h5file = tables.open_file(h5file.filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ts_list, aem = nic.get_photon_data_arr(h5file, range(48))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "neg_jumps = []\n",
    "for tx in tqdm_notebook(ts_list):\n",
    "    d = np.diff(tx[:])\n",
    "    assert (d > 0).all()\n",
    "    neg_jumps.append(d[d<0])\n",
    "assert sum(len(j) for j in neg_jumps) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jump_stats = [(len(j), min(j), max(j), np.mean(j)) if len(j) > 0 else (0, 0, 0, 0) for j in neg_jumps]\n",
    "\n",
    "for ch, jumps in enumerate(jump_stats):\n",
    "    if jumps[0] > 0:\n",
    "        print('CH%02d: %4d [%4d %4d] <%d>' % (ch, *jumps))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "widgets": {
   "state": {
    "1d0291a189b94f3b840ca8a5a816dcee": {
     "views": [
      {
       "cell_index": 6
      }
     ]
    },
    "f514ea08694347788a6f5350448e2ac1": {
     "views": [
      {
       "cell_index": 11
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}