DPSF_Single_Cell.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Shane Herbert Single-cell RNA-seq on Hydra"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This tutorial is a step by step guide to do the pre-processing of Single-cell RNA-seq data in Hydra. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import subprocess, os, sys, signal, pip\n",
    "import rpy2\n",
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "import threading\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We set up the input and the output foloder. The input folder contains all the fastq files and the output folder is the place where all the results would be saved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# inputFolder : Folder name with all the fastq files\n",
    "intputFolder = '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs'\n",
    "\n",
    "# Setting the output folder where all the results would be stored\n",
    "outputFolder = '/mnt/mr01-home01/mqbsxsm2/scratch/Shane_Herbert/output/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we define all the custom functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use `lambda` to write a one off function. Functions that we are going to use once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "remove_extension = lambda x: x.split('.')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_args(read1, read2, ref_genome, genome_dir, output_folders):\n",
    "    '''Set the input and output path for a given pair of reads'''\n",
    "    r1_shortname = remove_extension(os.path.basename(read1))\n",
    "\n",
    "    args = {  \n",
    "        'r1_input': read1,\n",
    "        'r2_input': read2,\n",
    "        'ref_genome': ref_genome,\n",
    "        'genome_dir': genome_dir,\n",
    "    }\n",
    "    \n",
    "    output_paths = {folder: os.path.join(outputFolder, folder, r1_shortname) for folder in output_folders}\n",
    "    \n",
    "    return dict(args, **output_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting up the Genome folder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are using Mouse cells, so we set mouse genome"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ref_genome = '/mnt/data-sets/bcf/genomeIndexes/danRer10/annotation/Danio_rerio.GRCz10.84.gtf.Processed'\n",
    "genome_dir = '/mnt/data-sets/bcf/genomeIndexes/danRer10/STAR'\n",
    "#ref_genome = '/mnt/data-sets/bcf/genomeIndexes/hg38_analysisSet/annotation/gencode.v24.annotation.gtf'\n",
    "#genome_dir = '/mnt/data-sets/bcf/genomeIndexes/hg38_analysisSet/STAR_gencode24'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting up the Software paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we set the path for all the tools we will be using. For this session we will only use trimmomatic and STAR."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trimmomatic = 'module load apps/trimmomatic/0.36/noarch; trimmomatic'\n",
    "STAR = 'module load apps/star/2.4.2a/gcc-4.8.5; STAR'\n",
    "htseq = 'module load apps/htseq/0.6.1p1/gcc-4.8.5+python-2.7.8+numpy-1.9.2; module load libs/pysam/0.8.3/gcc-4.8.5+python-2.7.8; python -m HTSeq.scripts.count'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "softwares = {    \n",
    "    'trimmomatic':trimmomatic,\n",
    "    'STAR':STAR,\n",
    "    'htseq':htseq}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating output folder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we create the output folder. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output_folders = [ 'Trimmomatic_Output', 'STAR_output'         # Trimmomatic, Star\n",
    "                 , 'htseq_output'\n",
    "                 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for folder in output_folders:\n",
    "    os.makedirs(os.path.join(outputFolder, folder))\n",
    "    #print(os.path.join(outputFolder, folder))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading all the fastq files for read1 and read2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now read all the fasta files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "root, folders, files = os.walk(intputFolder).__next__()\n",
    "\n",
    "files = [f for f in files if not f.startswith('.')] #remove hidden files if there exist\n",
    "reads1 = sorted([os.path.join(root, f) for f in files if 'R1' in f])\n",
    "reads2 = sorted([os.path.join(root, f) for f in files if 'R2' in f])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C01_S17_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C02_S9_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C03_S1_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C04_S65_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C05_S57_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C06_S49_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C07_S18_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C08_S10_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C09_S2_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C10_S66_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C11_S58_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C12_S50_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C13_S19_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C14_S11_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C15_S3_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C16_S67_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C17_S59_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C18_S51_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C19_S20_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C20_S12_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C21_S4_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C22_S68_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C23_S60_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C24_S52_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C25_S5_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C26_S13_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C27_S21_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C28_S53_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C29_S61_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C30_S69_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C31_S6_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C32_S14_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C33_S22_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C34_S54_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C35_S62_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C36_S70_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C37_S7_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C38_S15_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C39_S23_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C40_S55_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C41_S63_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C42_S71_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C43_S8_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C44_S16_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C45_S24_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C46_S56_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C47_S64_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C48_S72_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C49_S25_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C50_S33_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C51_S41_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C52_S73_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C53_S81_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C54_S89_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C55_S26_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C56_S34_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C57_S42_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C58_S74_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C59_S82_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C60_S90_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C61_S27_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C62_S35_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C63_S43_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C64_S75_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C65_S83_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C66_S91_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C67_S28_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C68_S36_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C69_S44_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C70_S76_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C71_S84_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C72_S92_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C73_S45_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C74_S37_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C75_S29_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C76_S93_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C77_S85_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C78_S77_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C79_S46_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C80_S38_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C81_S30_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C82_S94_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C83_S86_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C84_S78_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C85_S47_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C86_S39_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C87_S31_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C88_S95_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C89_S87_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C90_S79_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C91_S48_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C92_S40_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C93_S32_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C94_S96_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C95_S88_R1_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C96_S80_R1_001.fastq.gz']\n",
      "['/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C01_S17_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C02_S9_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C03_S1_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C04_S65_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C05_S57_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C06_S49_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C07_S18_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C08_S10_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C09_S2_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C10_S66_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C11_S58_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C12_S50_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C13_S19_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C14_S11_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C15_S3_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C16_S67_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C17_S59_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C18_S51_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C19_S20_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C20_S12_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C21_S4_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C22_S68_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C23_S60_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C24_S52_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C25_S5_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C26_S13_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C27_S21_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C28_S53_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C29_S61_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C30_S69_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C31_S6_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C32_S14_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C33_S22_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C34_S54_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C35_S62_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C36_S70_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C37_S7_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C38_S15_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C39_S23_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C40_S55_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C41_S63_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C42_S71_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C43_S8_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C44_S16_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C45_S24_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C46_S56_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C47_S64_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C48_S72_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C49_S25_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C50_S33_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C51_S41_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C52_S73_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C53_S81_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C54_S89_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C55_S26_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C56_S34_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C57_S42_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C58_S74_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C59_S82_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C60_S90_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C61_S27_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C62_S35_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C63_S43_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C64_S75_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C65_S83_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C66_S91_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C67_S28_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C68_S36_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C69_S44_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C70_S76_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C71_S84_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C72_S92_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C73_S45_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C74_S37_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C75_S29_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C76_S93_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C77_S85_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C78_S77_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C79_S46_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C80_S38_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C81_S30_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C82_S94_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C83_S86_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C84_S78_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C85_S47_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C86_S39_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C87_S31_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C88_S95_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C89_S87_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C90_S79_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C91_S48_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C92_S40_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C93_S32_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C94_S96_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C95_S88_R2_001.fastq.gz', '/mnt/fls01-bcf01/ngsdata/Analysis/2016/nextseq/161025_NB500968_0056_AHGCMVAFXX_analysis/Raphael_Thuret/fastqs/C96_S80_R2_001.fastq.gz']\n"
     ]
    }
   ],
   "source": [
    "print(reads1)\n",
    "print(reads2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting up the command list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we set up  the commands that we will use for the pre-processing. For this example we will only use `Trimmomatic` and `STAR` aligner."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cmds = [    \n",
    "    '{trimmomatic} PE -phred33 {r1_input} {r2_input} {Trimmomatic_Output}_r1_paired.fq {Trimmomatic_Output}_r1_unpaired.fq {Trimmomatic_Output}_r2_paired.fq {Trimmomatic_Output}_r2_upaired.fq ILLUMINACLIP:/opt/gridware/depots/4baff5c5/el7/pkg/apps/trimmomatic/0.36/noarch/share/adapters/TruSeq3-PE-2.fa:2:30:10 SLIDINGWINDOW:4:20 MINLEN:36', \n",
    "    '{STAR} --runMode alignReads \\\n",
    "    --runThreadN 1 \\\n",
    "    --genomeLoad NoSharedMemory \\\n",
    "    --genomeDir {genome_dir} \\\n",
    "    --sjdbGTFfile {ref_genome} \\\n",
    "    --readFilesIn {Trimmomatic_Output}_r1_paired.fq {Trimmomatic_Output}_r2_paired.fq \\\n",
    "    --outSAMtype BAM Unsorted \\\n",
    "    --outFileNamePrefix {STAR_output}',\n",
    "    '{htseq} --format=bam --stranded=reverse --type=exon --idattr=gene_id {STAR_output}Aligned.out.bam {ref_genome} > {htseq_output}.txt'\n",
    "    ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__We define the function in case we want to parallelize the run__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_cmd(cmds, args):\n",
    "    for cmd in cmds: \n",
    "        print(threading.currentThread().getName(), 'Starting')\n",
    "        #print(cmd.format(**args))\n",
    "        subprocess.call(cmd.format(**args), shell=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at the Number of cores in the node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Cores: 32\n"
     ]
    }
   ],
   "source": [
    "print(\"Total Cores:\",multiprocessing.cpu_count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Multithread starts __"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#for read1, read2 in zip(reads1, reads2):\n",
    "#    args = get_args(read1, read2, ref_genome, genome_dir, output_folders)\n",
    "#    args = dict(args, **softwares)\n",
    "#    try:\n",
    "#        t = threading.Thread(target=run_cmd, args = (cmds,args))\n",
    "#        t.daemon = True       \n",
    "#        t.start()        \n",
    "#        #t.join(timeout=None)\n",
    "#    except:\n",
    "#        print(\"Cannot Start Thread\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequential Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n",
      "MainThread Starting\n"
     ]
    }
   ],
   "source": [
    "for read1, read2 in zip(reads1, reads2):\n",
    "    args = get_args(read1, read2, ref_genome, genome_dir, output_folders)\n",
    "    args = dict(args, **softwares)\n",
    "    run_cmd(cmds,args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Architecture:          x86_64\r\n",
      "CPU op-mode(s):        32-bit, 64-bit\r\n",
      "Byte Order:            Little Endian\r\n",
      "CPU(s):                16\r\n",
      "On-line CPU(s) list:   0-15\r\n",
      "Thread(s) per core:    1\r\n",
      "Core(s) per socket:    8\r\n",
      "Socket(s):             2\r\n",
      "NUMA node(s):          2\r\n",
      "Vendor ID:             GenuineIntel\r\n",
      "CPU family:            6\r\n",
      "Model:                 62\r\n",
      "Stepping:              4\r\n",
      "CPU MHz:               2599.968\r\n",
      "BogoMIPS:              5199.24\r\n",
      "Virtualization:        VT-x\r\n",
      "L1d cache:             32K\r\n",
      "L1i cache:             32K\r\n",
      "L2 cache:              256K\r\n",
      "L3 cache:              20480K\r\n",
      "NUMA node0 CPU(s):     0-7\r\n",
      "NUMA node1 CPU(s):     8-15\r\n"
     ]
    }
   ],
   "source": [
    "! lscpu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating genome-index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apps/star/2.4.2a/gcc-4.4.7\n",
      " | -- libs/gcc/system\n",
      " |    * --> OK\n",
      " |\n",
      " OK\n",
      "Jun 08 12:38:50 ..... Started STAR run\n",
      "Jun 08 12:38:50 ... Starting to generate Genome files\n",
      "Jun 08 12:40:05 ... starting to sort  Suffix Array. This may take a long time...\n",
      "Jun 08 12:40:23 ... sorting Suffix Array chunks and saving them to disk...\n",
      "Jun 08 13:07:05 ... loading chunks from disk, packing SA...\n",
      "Jun 08 13:14:14 ... Finished generating suffix array\n",
      "Jun 08 13:14:14 ... starting to generate Suffix Array index...\n",
      "Jun 08 13:36:07 ..... Processing annotations GTF\n",
      "Jun 08 13:36:23 ..... Inserting junctions into the genome indices\n",
      "Jun 08 13:40:04 ... writing Genome to disk ...\n",
      "Jun 08 13:40:31 ... writing Suffix Array to disk ...\n",
      "Jun 08 13:43:53 ... writing SAindex to disk\n",
      "Jun 08 13:44:10 ..... Finished successfully\n"
     ]
    }
   ],
   "source": [
    "! module load apps/star/2.4.2a/gcc-4.4.7; STAR  --runMode genomeGenerate   --runThreadN 4   --genomeDir /mnt/single-cell/users/mqbsxsm2/STAR_Gencode_vM5  --genomeFastaFiles /mnt/data-sets/bcf/genomeIndexes/mm10_random_chrM_chrUn/fasta/mm10_random_chrM_chrUn.fa     --sjdbGTFfile /mnt/data-sets/bcf/genomeIndexes/mm10_random_chrM_chrUn/annotation/Gencode-M5/gencode.vM5.annotation.gtf   --sjdbOverhang 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing htseq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apps/htseq/0.6.1p1/gcc-4.4.7+python-2.7.5+numpy-1.7.1\n",
      " | -- libs/gcc/system\n",
      " |    * --> OK\n",
      " | -- apps/python/2.7.5/gcc-4.4.7\n",
      " |    | -- libs/gcc/system ... SKIPPED (already loaded)\n",
      " |    * --> OK\n",
      " | -- libs/numpy/1.7.1/gcc-4.4.7+python-2.7.5+atlas-3.10.1\n",
      " |    | -- libs/gcc/system ... SKIPPED (already loaded)\n",
      " |    | -- apps/python/2.7.5/gcc-4.4.7 ... SKIPPED (already loaded)\n",
      " |    * --> OK\n",
      " |\n",
      " OK\n",
      "libs/pysam/0.8.3/gcc-4.4.7+python-2.7.5\n",
      " | -- libs/gcc/system ... SKIPPED (already loaded)\n",
      " | -- apps/python/2.7.5/gcc-4.4.7 ... SKIPPED (already loaded)\n",
      " |\n",
      " OK\n",
      "100000 GFF lines processed.\n",
      "200000 GFF lines processed.\n",
      "300000 GFF lines processed.\n",
      "400000 GFF lines processed.\n",
      "500000 GFF lines processed.\n",
      "600000 GFF lines processed.\n",
      "700000 GFF lines processed.\n",
      "800000 GFF lines processed.\n",
      "900000 GFF lines processed.\n",
      "1000000 GFF lines processed.\n",
      "1100000 GFF lines processed.\n",
      "1200000 GFF lines processed.\n",
      "1300000 GFF lines processed.\n",
      "1400000 GFF lines processed.\n",
      "1500000 GFF lines processed.\n",
      "1523143 GFF lines processed.\n",
      "100000 SAM alignment record pairs processed.\n",
      "200000 SAM alignment record pairs processed.\n",
      "300000 SAM alignment record pairs processed.\n",
      "400000 SAM alignment record pairs processed.\n",
      "500000 SAM alignment record pairs processed.\n",
      "600000 SAM alignment record pairs processed.\n",
      "700000 SAM alignment record pairs processed.\n",
      "800000 SAM alignment record pairs processed.\n",
      "900000 SAM alignment record pairs processed.\n",
      "1000000 SAM alignment record pairs processed.\n",
      "1100000 SAM alignment record pairs processed.\n",
      "1200000 SAM alignment record pairs processed.\n",
      "1300000 SAM alignment record pairs processed.\n",
      "1400000 SAM alignment record pairs processed.\n",
      "1500000 SAM alignment record pairs processed.\n",
      "1600000 SAM alignment record pairs processed.\n",
      "1700000 SAM alignment record pairs processed.\n",
      "1800000 SAM alignment record pairs processed.\n",
      "1900000 SAM alignment record pairs processed.\n",
      "2000000 SAM alignment record pairs processed.\n",
      "2100000 SAM alignment record pairs processed.\n",
      "2200000 SAM alignment record pairs processed.\n",
      "2300000 SAM alignment record pairs processed.\n",
      "2400000 SAM alignment record pairs processed.\n",
      "2500000 SAM alignment record pairs processed.\n",
      "2600000 SAM alignment record pairs processed.\n",
      "2700000 SAM alignment record pairs processed.\n",
      "2800000 SAM alignment record pairs processed.\n",
      "2886283 SAM alignment pairs processed.\n"
     ]
    }
   ],
   "source": [
    "! module load apps/htseq/0.6.1p1/gcc-4.4.7+python-2.7.5+numpy-1.7.1; module load libs/pysam/0.8.3/gcc-4.4.7+python-2.7.5; python -m HTSeq.scripts.count --format=bam --stranded=reverse --type=exon --idattr=gene_id /mnt/mr01-home01/mqbsxsm2/scratch/Single_cell_Read_Effect/AllLanes/output/STAR_output/C01_R1Aligned.out.bam /mnt/data-sets/bcf/genomeIndexes/mm10_random_chrM_chrUn/annotation/Gencode-M5/gencode.vM5.annotation.gtf > /mnt/mr01-home01/mqbsxsm2/scratch/Single_cell_Read_Effect/AllLanes/output/htseq_output/C01_R1.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apps/cufflinks/2.2.1/gcc-4.4.7+boost-1.49.0+samtools-0.1.19+eigen-3.0.5\n",
      " | -- libs/gcc/system\n",
      " |    * --> OK\n",
      " | -- libs/boost/1.49.0/gcc-4.4.7+openmpi-1.6.5+python-2.7.5\n",
      " |    | -- libs/gcc/system ... SKIPPED (already loaded)\n",
      " |    | -- mpi/openmpi/1.6.5/gcc-4.4.7\n",
      " |    |    | -- libs/gcc/system ... SKIPPED (already loaded)\n",
      " |    |    * --> OK\n",
      " |    * --> OK\n",
      " | -- apps/samtools/0.1.19/gcc-4.4.7\n",
      " |    | -- libs/gcc/system ... SKIPPED (already loaded)\n",
      " |    * --> OK\n",
      " |\n",
      " OK\n"
     ]
    }
   ],
   "source": [
    "! module load apps/cufflinks/2.2.1/gcc-4.4.7+boost-1.49.0+samtools-0.1.19+eigen-3.0.5; gffread -E gtf/All.gff -T -o gtf/AllConverted.gtf \n",
    "#! module load apps/cufflinks/2.2.1/gcc-4.4.7+boost-1.49.0+samtools-0.1.19+eigen-3.0.5; gffread -h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "! pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}