testing done, pending review

.gitignore

@@ -11,3 +11,8 @@ testing*
 venv/
 params*.json
 sample_manifest*
+*.log
+tests/sample-data/*
+tests/ref/*
+!**/.gitkeep
+.nf-test/

modules.json

@@ -7,14 +7,14 @@
                 "nf-core": {
                     "fastqc": {
                         "branch": "master",
-                        "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d",
+                        "git_sha": "dc94b6ee04a05ddb9f7ae050712ff30a13149164",
                         "installed_by": [
                             "modules"
                         ]
                     },
                     "multiqc": {
                         "branch": "master",
-                        "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d",
+                        "git_sha": "cf17ca47590cc578dfb47db1c2a44ef86f89976d",
                         "installed_by": [
                             "modules"
                         ]
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}

modules/local/get_software_versions.nf

@@ -9,11 +9,7 @@ process GET_SOFTWARE_VERSIONS {
         saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) }
 
     conda (params.enable_conda ? "conda-forge::python=3.8.3" : null)
-    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
-        container "https://depot.galaxyproject.org/singularity/python:3.8.3"
-    } else {
-        container "quay.io/biocontainers/python:3.8.3"
-    }
+    container "quay.io/biocontainers/python:3.8.3"
 
     cache false
 

modules/local/read_modification/main.nf

@@ -20,17 +20,27 @@ process APPEND_STRINGS_TO_FQ {
 
     script:
     def software = getSoftwareName(task.process)
-    def prefix   = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}"
+    def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}"
     def input = reads
     def append_start = params.append_start ? params.append_start : ""
     def append_end = params.append_end ? params.append_end : ""
     def append_quality_start = params.append_start ? params.append_quality*append_start.length() : ""
     def append_quality_end = params.append_end ? params.append_quality*append_end.length() : ""
     def output = "${prefix}.modified.fq.gz"
+    """
+    zcat ${input} | awk -v append_start="${append_start}" -v append_end="${append_end}" -v append_quality_start="${append_quality_start}" -v append_quality_end="${append_quality_end}" '
+        NR % 4 == 2 { print append_start \$0 append_end; next }
+        NR % 4 == 0 { print append_quality_start \$0 append_quality_end; next }
+        { print \$0 }
+        ' | gzip > ${output}
+    """
 
-    $/
-        zcat ${input} | \
-        sed -e '2~4s/^\(.*\)$/${append_start}\1${append_end}/' -e '4~4s/^\(.*\)$/${append_quality_start}\1${append_quality_end}/' | \
-        gzip > ${output}
-    /$
+
+    // "zcat ${input} | sed -e '2~4s/^\(.*\)$/${append_start}\1${append_end}/' -e '4~4s/^\(.*\)$/${append_quality_start}\1${append_quality_end}/' | gzip > ${output}'
+
+    // $/
+    //     zcat ${input} | \
+    //     sed -e '2~4s/^\(.*\)$/${append_start}\1${append_end}/' -e '4~4s/^\(.*\)$/${append_quality_start}\1${append_quality_end}/' | \
+    //     gzip > ${output}
+    // /$
 }

modules/local/samplesheet_check.nf

@@ -10,11 +10,7 @@ process SAMPLESHEET_CHECK_FASTQ {
         saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) }
 
     conda (params.enable_conda ? "conda-forge::python=3.8.3" : null)
-    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
-        container "https://depot.galaxyproject.org/singularity/python:3.8.3"
-    } else {
-        container "quay.io/biocontainers/python:3.8.3"
-    }
+    container "quay.io/biocontainers/python:3.8.3"
 
     input:
     path samplesheet
@@ -37,11 +33,7 @@ process SAMPLESHEET_CHECK_CRAM {
         saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) }
 
     conda (params.enable_conda ? "conda-forge::python=3.8.3" : null)
-    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
-        container "https://depot.galaxyproject.org/singularity/python:3.8.3"
-    } else {
-        container "quay.io/biocontainers/python:3.8.3"
-    }
+    container "quay.io/biocontainers/python:3.8.3"
 
     input:
     path samplesheet

modules/nf-core/fastqc/main.nf

@@ -11,13 +11,8 @@ process FASTQC {
         mode: params.publish_dir_mode,
         saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) }
 
-    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)
-    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
-        container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0"
-    } else {
-        container "quay.io/biocontainers/fastqc:0.11.9--0"
-    }
-
+    // conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)
+    container "quay.io/biocontainers/fastqc:0.11.9--0"  // conda safe
     input:
     tuple val(meta), path(reads)
 

modules/nf-core/multiqc/main.nf

@@ -10,12 +10,8 @@ process MULTIQC {
         mode: params.publish_dir_mode,
         saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:[:], publish_by_meta:[]) }
 
-    conda (params.enable_conda ? "bioconda::multiqc=1.14" : null)
-    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
-        container "https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0"
-    } else {
-        container "quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0"
-    }
+    // conda (params.enable_conda ? "bioconda::multiqc=1.14" : null)
+    container "quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0"  // conda safe
 
     input:
     path multiqc_files

nf-test.config

@@ -0,0 +1,8 @@
+config {
+
+    testsDir "tests"
+    workDir System.getenv("NFT_WORKDIR") ?: ".nf-test"
+    configFile "tests/nextflow.config"
+    profile "singularity"
+
+}

tests/TESTS.md

@@ -0,0 +1,24 @@
+# Test datasets and files
+
+end-to-end tests for the quants pipeline using the nf-test framework.
+
+Having retrieved data and placed it in the ref and sample-data folders, tests can be run with `nf-test test tests/test[#].main.nf.test`
+
+To confirm that you've retrieved the correct data, and named it appropriately if necessary, run `diff -q <(md5sum tests/sample-data/*) tests/sample-data-checksums` and `diff -q <(md5sum tests/ref/*) tests/ref-checksums`. If the commands return nothing (exit code 0), the data is as expected. Note that if you do not run this command from the top level directory of the repo, it will fail, as md5sum will output relative paths along with checksums.
+
+## Test parameters
+
+| Test | Source | FASTQ or CRAM | SE or PE| RevComp | Read merging | Adapter trimming | Primer Trimming | Read filtering | Read modification | QC | Quantification |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | 
+| `test1.main.nf.test` | Trimmed | FASTQ | SE | Y | N | N | N | N | Y | N | Y |
+| `test2.main.nf.test` | Trimmed | FASTQ | SE | Y | N | N | N | N | Y | Y | Y |
+| `test3.main.nf.test` | Raw | FASTQ | SE | Y | N | Y | Y | N | Y | Y | Y |
+| `test4.main.nf.test` | Raw | CRAM | SE | Y | N | Y | Y | N | Y | Y | Y |
+| `test5.main.nf.test` | Raw | FASTQ | PE | N | Y (SeqPrep) | Y | Y | N | N | Y | Y |
+| `test6.main.nf.test` | Raw | FASTQ | PE | N | Y (Flash2) | Y | Y | Y | N | Y | Y |
+| `test7.main.nf.test` | Raw | FASTQ | PE | N | Y (SeqPrep) | Y | Y | Y | N | Y | Y |
+| `test8.main.nf.test` | Raw | FASTQ | PE | N | Y (Flash2) | Y | Y | Y | N | Y | Y |
+| `test9.main.nf.test` | Raw | FASTQ | PE | N | Y (Flash2) | Y | Y | Y | N | Y | Y |
+| `test10.main.nf.test` | Raw | FASTQ | SE | N | N | Y | Y | Y | N | Y | Y |
+| `test11.main.nf.test` | Raw | FASTQ | SE | N | N | Y | Y | N | Y | Y | Y |
+| `test12.main.nf.test` | Raw | CRAM | SE | N | N | Y | Y | N | Y | Y | Y |

tests/manifests/raw-cram-se-noRC.csv

@@ -0,0 +1,2 @@
+sample,cram_file
+test_sample,../../../tests/sample-data/1.cram

tests/manifests/raw-cram-se.csv

@@ -0,0 +1,2 @@
+sample,cram_file
+test_sample,../../../tests/sample-data/2.cram

tests/manifests/raw-fastq-pe-multiple-samples.csv

@@ -0,0 +1,5 @@
+sample,fastq_1,fastq_2
+lib,../../../tests/sample-data/libR1.fq.gz,../../../tests/sample-data/libR2.fq.gz
+u,../../../tests/sample-data/uR1.fq.gz,../../../tests/sample-data/uR2.fq.gz
+v,../../../tests/sample-data/vR1.fq.gz,../../../tests/sample-data/vR2.fq.gz
+w,../../../tests/sample-data/wR1.fq.gz,../../../tests/sample-data/wR2.fq.gz

tests/manifests/raw-fastq-pe.csv

@@ -0,0 +1,2 @@
+sample,fastq_1,fastq_2
+test_sample,../../../tests/sample-data/xR1.fq.gz,../../../tests/sample-data/xR2.fq.gz

tests/manifests/raw-fastq-se-multiple-samples.csv

@@ -0,0 +1,4 @@
+sample,fastq_1,fastq_2
+HG_CH_1_4_A,/lustre/scratch125/casm/team78/quants_test_data/DATA/2.fq.gz,
+HG_CH_1_4_B,/lustre/scratch125/casm/team78/quants_test_data/DATA/3.fq.gz,
+HG_CH_1_4_C,/lustre/scratch125/casm/team78/quants_test_data/DATA/4.fq.gz,

tests/manifests/raw-fastq-se-noRC.csv

@@ -0,0 +1,2 @@
+sample,fastq_1,fastq_2
+test_sample,../../../tests/sample-data/1.fq.gz,

tests/manifests/raw-fastq-se.csv

@@ -0,0 +1,2 @@
+sample,fastq_1,fastq_2
+test_sample,../../../tests/sample-data/2.fq.gz,

tests/manifests/trimmed-fastq-se.csv

@@ -0,0 +1,2 @@
+sample,fastq_1,fastq_2
+test_sample,../../../tests/sample-data/sge.fq.gz,

tests/nextflow.config

@@ -0,0 +1,5 @@
+/*
+========================================================================================
+    Nextflow config file for running tests
+========================================================================================
+*/

tests/ref-checksums

@@ -0,0 +1,5 @@
+485b9de403b0278ed6684ad1348d37a8  tests/ref/1.csv
+3e403419a08708609c9341a07ef7ac98  tests/ref/2.csv
+e3c702dee536c45e8eeae36996fcd207  tests/ref/3.tsv
+ac0899f173efe28a5b4b9b161f232d93  tests/ref/4.tsv
+075cc1feb687db2c899a48fb263076a4  tests/ref/5.tsv

tests/sample-data-checksums

@@ -0,0 +1,17 @@
+77d92378b88ed28dd3f7d5d580971a53  tests/sample-data/1.cram
+26d427131d4720aa7ed07e8018c526b1  tests/sample-data/1.fq.gz
+b822820791a9fb4fde3af96afa9308a5  tests/sample-data/2.cram
+023bc001077093ad6586a4c5f601ad37  tests/sample-data/2.fq.gz
+0d131b633a2af323b025a756ae933d7b  tests/sample-data/3.fq.gz
+47201d8a28b69328823c8df973041ec2  tests/sample-data/4.fq.gz
+2ab872bb3f7de763c27d8811e3e2c622  tests/sample-data/libR1.fq.gz
+65888a8f410e81e0633a814dc77743e5  tests/sample-data/libR2.fq.gz
+5bf45c57e729d8629e99df3979d1c97c  tests/sample-data/sge.fq.gz
+4da5792d1d6a5d998f0ace0870c55f29  tests/sample-data/uR1.fq.gz
+2ef7c9ad9eaf02f903fafd340095fbd5  tests/sample-data/uR2.fq.gz
+232f948fbbd7bd49f2cf9735c527a744  tests/sample-data/vR1.fq.gz
+5dce830523e9927aba8433ff79b56db4  tests/sample-data/vR2.fq.gz
+77ffaf4d0dcf665bd9ba31e8da885c16  tests/sample-data/wR1.fq.gz
+2777816cca4302e9c042dfaeb17d6dea  tests/sample-data/wR2.fq.gz
+0aee12c4562d18eba0300ba0a144371a  tests/sample-data/xR1.fq.gz
+6a789cedb3b9f64479ec6c875eceae1a  tests/sample-data/xR2.fq.gz

tests/test1.main.nf.test

@@ -0,0 +1,31 @@
+nextflow_pipeline {
+
+    name "Test Workflow main.nf"
+    script "main.nf"
+
+    test("Should run without failures") {
+
+        when {
+            params {
+                input = "${projectDir}/tests/manifests/trimmed-fastq-se.csv"
+                single_end = true
+                input_type = "fastq"
+                read_modification = true
+                append_start = "ATTCACGTTATGCTGTCCAATCTCT"
+                append_end = "GAAATCTAAAATTACCTTCCCAGGG"
+                append_quality = "?"
+                read_transform = "reverse_complement"
+                transform_library = true
+                pyquest_library_converter_options = "-N 1 -S 24"
+                quantification = "pyquest"
+                oligo_library = "${projectDir}/tests/ref/2.csv"
+            }
+        }
+
+        then {
+            assert workflow.success
+        }
+
+    }
+
+}

tests/test10.main.nf.test

@@ -0,0 +1,30 @@
+// TEST BROKEN
+nextflow_pipeline {
+
+    name "Test Workflow main.nf"
+    script "main.nf"
+
+    test("Should run without failures") {
+
+        when {
+            params {
+                input = "${projectDir}/tests/manifests/raw-fastq-se-multiple-samples.csv"
+                single_end = true
+                input_type = "fastq"
+                raw_sequencing_qc = true
+                adapter_trimming = "cutadapt"
+                adapter_cutadapt_options = "-a 'AGATCGGAAGAGCGGTTCAGCAGGAATGCCG' -A 'AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT' -m 1"
+                adapter_trimming_qc = true
+                primer_trimming = "cutadapt"
+                primer_cutadapt_options = "-g 'TCTACATTTTATTTGCACTACTTGAAGG...ATACTTTTAAGTTCTGGGGTCATGTG' -G 'CACATGACCCCAGAACTTAAA...AGTGCAAATAAAATGTAGA' -m 1"
+                primer_trimming_qc = true
+            }
+        }
+
+        then {
+            assert workflow.success
+        }
+
+    }
+
+}

tests/test11.main.nf.test

@@ -0,0 +1,37 @@
+nextflow_pipeline {
+
+    name "Test Workflow main.nf"
+    script "main.nf"
+
+    test("Should run without failures") {
+
+        when {
+            params {
+                input = "${projectDir}/tests/manifests/raw-fastq-se-noRC.csv"
+                single_end = true
+                input_type = "fastq"
+                raw_sequencing_qc = true
+                adapter_trimming = "cutadapt"
+                adapter_cutadapt_options = "-a AGATCGGAAGAGCACACGTCTGAACTCCAGTCA -m 1"
+                adapter_trimming_qc = true
+                primer_trimming = "cutadapt"
+                primer_cutadapt_options = "-g 'AGCATGTCTAAGGCATATCAGAGCA...TGAGCTGGCAGAGGGGCCATTGTGG' -m 1"
+                primer_trimming_qc = true
+                read_modification = true
+                append_start = "AGCATGTCTAAGGCATATCAGAGCA"
+                append_end = "TGAGCTGGCAGAGGGGCCATTGTGG"
+                append_quality = "?"
+                transform_library = true
+                pyquest_library_converter_options = "-N 1 -S 24"
+                quantification = "pyquest"
+                oligo_library = "${projectDir}/tests/ref/1.csv"
+            }
+        }
+
+        then {
+            assert workflow.success
+        }
+
+    }
+
+}

tests/test12.main.nf.test

@@ -0,0 +1,37 @@
+nextflow_pipeline {
+
+    name "Test Workflow main.nf"
+    script "main.nf"
+
+    test("Should run without failures") {
+
+        when {
+            params {
+                input = "${projectDir}/tests/manifests/raw-cram-se-noRC.csv"
+                single_end = true
+                input_type = "cram"
+                raw_sequencing_qc = true
+                adapter_trimming = "cutadapt"
+                adapter_cutadapt_options = "-a AGATCGGAAGAGCACACGTCTGAACTCCAGTCA -m 1"
+                adapter_trimming_qc = true
+                primer_trimming = "cutadapt"
+                primer_cutadapt_options = "-g 'AGCATGTCTAAGGCATATCAGAGCA...TGAGCTGGCAGAGGGGCCATTGTGG' -m 1"
+                primer_trimming_qc = true
+                read_modification = true
+                append_start = "AGCATGTCTAAGGCATATCAGAGCA"
+                append_end = "TGAGCTGGCAGAGGGGCCATTGTGG"
+                append_quality = "?"
+                transform_library = true
+                pyquest_library_converter_options = "-N 1 -S 24"
+                quantification = "pyquest"
+                oligo_library = "${projectDir}/tests/ref/1.csv"
+            }
+        }
+
+        then {
+            assert workflow.success
+        }
+
+    }
+
+}

tests/test2.main.nf.test

@@ -0,0 +1,30 @@
+nextflow_pipeline {
+
+    name "Test Workflow main.nf"
+    script "main.nf"
+
+    test("Should run without failures") {
+
+        when {
+            params {
+                input = "${projectDir}/tests/manifests/trimmed-fastq-se.csv"
+                single_end = true
+                input_type = "fastq"
+                raw_sequencing_qc = true
+                read_modification = true
+                append_start = "ATTCACGTTATGCTGTCCAATCTCT"
+                append_end = "GAAATCTAAAATTACCTTCCCAGGG"
+                append_quality = "?"
+                read_transform = "reverse_complement"
+                quantification = "pyquest"
+                oligo_library = "${projectDir}/tests/ref/3.tsv"
+            }
+        }
+
+        then {
+            assert workflow.success
+        }
+
+    }
+
+}