diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000000..1bfb75a86c --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,25 @@ +# CAW Contributing Guidelines + +Hi there! Many thanks for taking an interest in improving CAW. + +We try to manage the required tasks for CAW using GitHub issues, you probably came to this page when creating one. Please use the prefilled template to save time. + +However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) + +> If you need help using CAW then the best place to go is the Gitter chatroom where you can ask us questions directly: https://gitter.im/SciLifeLab/CAW + +## Contribution workflow +If you'd like to write some code for CAW, the standard workflow +is as follows: + +1. Check that there isn't already an issue about your idea in the + [CAW issues](https://github.com/SciLifeLab/CAW/issues) to avoid + duplicating work. + * Feel free to add a new issue here for the same reason. +2. Fork the CAW repository to your GitHub account +3. Make the necessary changes / additions within your forked repository +4. Submit a Pull Request against the master branch and wait for the code to be reviewed and merged. + +If you're not used to this workflow with git, you can start with some [basic docs from GitHub](https://help.github.com/articles/fork-a-repo/) or even their [excellent interactive tutorial](https://try.github.io/). + +For further information/help, please consult the [CAW documentation](https://github.com/SciLifeLab/CAW#documentation) and don't hesitate to get in touch on [Gitter](https://gitter.im/SciLifeLab/CAW) or contact us: maxime.garcia@scilifelab.se, szilveszter.juhos@scilifelab.se diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..9bf1ceb308 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,43 @@ + + +## Expected Behavior + + +## Current Behavior + + +## Possible Solution + + +## Context (for bugs) + +- **CAW `nextflow.log`:** + + - [please drag and drop the `.nextflow.log` file that will help understand the error here] +- **CAW `input.TSV`:** + + - [please drag and drop the `TSV` file that will help reproduce or understand the error here] diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md new file mode 100644 index 0000000000..21378aebba --- /dev/null +++ b/.github/RELEASE_CHECKLIST.md @@ -0,0 +1,21 @@ +# Release checklist +This checklist is for our own reference + +1. Check that everything is up to date and ready to go +2. Increase version numbers. +3. Update version numbers in code: `main.nf`, `buildContainers.nf`, `buildReferences.nf` +4. If any changes on any containers, match the tag to current version `docker.config`, `singularity.config`, `singularity-path.config`. +5. Build, and get the containers. + - `./scripts/do_all.sh --push` + - `./scripts/do_all.sh --pull` +6. Test against sample data. + - Check for any command line errors + - Check version numbers are printed correctly + - `./scripts/test.sh -p docker` + - `./scripts/test.sh -p singularity` + - `./scripts/test.sh -p singularityPath` +7. Commit and push version updates +8. Make a [release](https://github.com/SciLifeLab/CAW/releases) on GitHub - list PRs as changelog. +9. Tweet that new version is released +10. Commit and push. Continue making more awesome :metal: +11. Have fika :cake: diff --git a/.gitignore b/.gitignore index 2c63d79039..44cb9cd70d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,6 @@ work/ .nextflow* *.img *.tar.gz +report.html* timeline.html* trace.txt* diff --git a/.travis.yml b/.travis.yml index 77be1bb5c7..e12269ff8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,19 +8,19 @@ services: - docker env: - - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=ANNOTATESNPEFF TOOL_INSTALL=all - - NXF_VER=0.25.6 PROFILE=docker TEST=ANNOTATESNPEFF TOOL_INSTALL=nextflow - - NXF_VER=0.25.6 PROFILE=docker TEST=ANNOTATEVEP TOOL_INSTALL=nextflow - - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=RECALIBRATE TOOL_INSTALL=all - - NXF_VER=0.25.6 PROFILE=docker TEST=RECALIBRATE TOOL_INSTALL=nextflow - - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=REALIGN TOOL_INSTALL=all - - NXF_VER=0.25.6 PROFILE=docker TEST=REALIGN TOOL_INSTALL=nextflow - - NXF_VER=0.25.6 PROFILE=docker TEST=BUILDCONTAINERS TOOL_INSTALL=nextflow - - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=MAPPING TOOL_INSTALL=all - - NXF_VER=0.25.6 PROFILE=docker TEST=MAPPING TOOL_INSTALL=nextflow + - NXF_VER=0.26.0 SGT_VER=2.3.1 PROFILE=singularity TEST=ANNOTATESNPEFF TOOL_INSTALL=all + - NXF_VER=0.26.0 PROFILE=docker TEST=ANNOTATESNPEFF TOOL_INSTALL=nextflow + - NXF_VER=0.26.0 PROFILE=docker TEST=ANNOTATEVEP TOOL_INSTALL=nextflow + - NXF_VER=0.26.0 SGT_VER=2.3.1 PROFILE=singularity TEST=RECALIBRATE TOOL_INSTALL=all + - NXF_VER=0.26.0 PROFILE=docker TEST=RECALIBRATE TOOL_INSTALL=nextflow + - NXF_VER=0.26.0 SGT_VER=2.3.1 PROFILE=singularity TEST=REALIGN TOOL_INSTALL=all + - NXF_VER=0.26.0 PROFILE=docker TEST=REALIGN TOOL_INSTALL=nextflow + - NXF_VER=0.26.0 PROFILE=docker TEST=BUILDCONTAINERS TOOL_INSTALL=nextflow + - NXF_VER=0.26.0 SGT_VER=2.3.1 PROFILE=singularity TEST=MAPPING TOOL_INSTALL=all + - NXF_VER=0.26.0 PROFILE=docker TEST=MAPPING TOOL_INSTALL=nextflow install: # Install Nextflow (and Singularity if needed) - "./scripts/install.sh --tool $TOOL_INSTALL" script: - - "./scripts/test.sh --profile $PROFILE --test $TEST" + - "travis_wait 50 ./scripts/test.sh --profile $PROFILE --test $TEST" diff --git a/README.md b/README.md index 22d449cfdf..201b95b478 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,19 @@ CAW is a complete open source pipeline to detect somatic variants from WGS data The pipeline uses [Nextflow][nextflow-link], a bioinformatics domain specific language for workflow building and [Singularity](http://singularity.lbl.gov/), a container technology specific for high-performance computing. -This pipeline is primarily used with cluster on the Swedish [UPPMAX systems](https://www.uppmax.uu.se/). However, the pipeline should be able to run on any system that supports Nextflow. The pipeline comes with some configuration for different systems. See the [documentation](#documentation) for more information. +This pipeline is primarily used with cluster on the Swedish [UPPMAX systems](https://www.uppmax.uu.se/). +However, the pipeline should be able to run on any system that supports Nextflow. +The pipeline comes with some configuration for different systems. +See the [documentation](#documentation) for more information. -We utilize [GATK best practices](https://software.broadinstitute.org/gatk/best-practices/) to align, realign and recalibrate short-read data in parallel for both normal and tumor sample. After these preprocessing steps, several somatic variant callers scan the resulting BAM files: [MuTect1][mutect1-link], [MuTect2][gatk-link] and [Strelka][strelka-link] are used to find somatic SNVs and small indels, also [GATK HaplotyeCaller][gatk-link] for both the normal and the tumor sample. For structural variants we use [Manta][manta-link]. Furthermore, we are applying [ASCAT][ascat-link] to estimate sample heterogeneity, ploidy and CNVs. +Caw is based on [GATK best practices](https://software.broadinstitute.org/gatk/best-practices/) to align, realign and recalibrate short-read data in parallel for both normal and tumor sample. +After these preprocessing steps, several somatic variant callers scan the resulting BAM files: [MuTect1][mutect1-link], [MuTect2][gatk-link], [Freebayes][freebayes-link] and [Strelka][strelka-link] are used to find somatic SNVs and small indels, also [GATK HaplotyeCaller][gatk-link] for both the normal and the tumor sample. +For structural variants we use [Manta][manta-link]. +Furthermore, we are applying [ASCAT][ascat-link] to estimate sample heterogeneity, ploidy and CNVs. -The pipeline can begin the analysis either from raw FASTQ files, only from the realignment step, or directly with any subset of variant callers using recalibrated BAM files. At the end of the analysis the resulting VCF files are merged to facilitate further downstream processing, though results from each caller are also retained. The flow is capable of accommodating additional variant calling software or CNV callers. It is also prepared to process normal, tumor and several relapse samples. +The pipeline can begin the analysis either from raw FASTQ files, only from the realignment step, or directly with any subset of variant callers using recalibrated BAM files. +At the end of the analysis the resulting VCF files are merged to facilitate further downstream processing, though results from each caller are also retained. +The flow is capable of accommodating additional variant calling software or CNV callers. It is also prepared to process normal, tumor and several relapse samples. Besides variant calls, the workflow provides quality controls presented by [MultiQC][multiqc-link]. @@ -40,7 +48,10 @@ The CAW pipeline comes with documentation about the pipeline, found in the `doc/ 14. [More information about ASCAT](doc/ASCAT.md) 15. [Folder structure](doc/FOLDER.md) -For further information/help contact: maxime.garcia@scilifelab.se, szilveszter.juhos@scilifelab.se or join the gitter chat: [gitter.im/SciLifeLab/CAW][gitter-link]. +## Contributions & Support + +- [Contributions guidelines](.github/CONTRIBUTING.md) +For further information/help, don't hesitate to get in touch on [Gitter][gitter-link] or contact us: maxime.garcia@scilifelab.se, szilveszter.juhos@scilifelab.se ## Authors @@ -63,6 +74,7 @@ For further information/help contact: maxime.garcia@scilifelab.se, szilveszter.j [ascat-link]: https://github.com/Crick-CancerGenomics/ascat [caw-site-link]: http://opensource.scilifelab.se/projects/caw/ +[freebayes-link]: https://github.com/ekg/freebayes [gatk-link]: https://github.com/broadgsa/gatk-protected [gitter-badge]: https://badges.gitter.im/SciLifeLab/CAW.svg [gitter-link]: https://gitter.im/SciLifeLab/CAW @@ -76,7 +88,7 @@ For further information/help contact: maxime.garcia@scilifelab.se, szilveszter.j [nextflow-link]: https://www.nextflow.io/ [ngi-link]: https://ngisweden.scilifelab.se/ [scilifelab-link]: https://www.scilifelab.se/ -[scilifelab-stockholm-link]: https://www.scilifelab.se/platforms/ngi/ +[scilifelab-stockholm-link]: https://www.scilifelab.se/facilities/ngi-stockholm/ [strelka-link]: https://github.com/Illumina/strelka [travis-badge]: https://api.travis-ci.org/SciLifeLab/CAW.svg [travis-link]: https://travis-ci.org/SciLifeLab/CAW diff --git a/buildContainers.nf b/buildContainers.nf index 7ed7ca4a4e..4e55c761c5 100644 --- a/buildContainers.nf +++ b/buildContainers.nf @@ -36,18 +36,18 @@ kate: syntax groovy; space-indent on; indent-width 2; ================================================================================ */ -version = '1.2.4' +version = '1.2.5' // Check that Nextflow version is up to date enough // try / throw / catch works for NF versions < 0.25 when this was implemented nf_required_version = '0.25.0' try { - if( ! nextflow.version.matches(">= $nf_required_version") ){ + if( ! nextflow.version.matches(">= ${nf_required_version}") ){ throw GroovyException('Nextflow version too old') } } catch (all) { log.error "====================================================\n" + - " Nextflow version $nf_required_version required! You are running v$workflow.nextflow.version.\n" + + " Nextflow version ${nf_required_version} required! You are running v${workflow.nextflow.version}.\n" + " Pipeline execution will continue, but things may break.\n" + " Please update Nextflow.\n" + "============================================================" @@ -55,14 +55,14 @@ try { if (params.help) exit 0, helpMessage() if (params.version) exit 0, versionMessage() -if (!isAllowedParams(params)) exit 1, "params is unknown, see --help for more information" +if (!isAllowedParams(params)) exit 1, "params unknown, see --help for more information" if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project " // Default params: // Such params are overridden by command line or configuration definitions -// containerPath is empty -params.containerPath = '' +// containerPath is current Directory +params.containerPath = "${baseDir}" // all containers to be build params.containers = 'all' // Docker will not be used @@ -70,22 +70,26 @@ params.docker = false // Containers will not be pushed on DockerHub params.push = false // DockerHub repository is maxulysse +// TODO Change to a SciLifeLab repository params.repository = 'maxulysse' // Singularity will not be used params.singularity = false -verbose = params.verbose +// Define containers to handle (build/push or pull) containersList = defineContainersList() containers = params.containers.split(',').collect {it.trim()} containers = containers == ['all'] ? containersList : containers -docker = params.docker ? true : false + +// push only to DockerHub, so only when using Docker push = params.docker && params.push ? true : false -repository = params.repository + +// by default the tag will be the current version tag = params.tag ? params.tag : version -singularity = params.singularity ? true : false -containerPath = params.singularity && params.containerPath ? params.containerPath : "." -if (!docker && !singularity) exit 1, 'No builder choose, specify --docker or --singularity, see --help for more information' +// to simplify verbose mode +verbose = params.verbose + +if (!params.docker && !params.singularity) exit 1, 'No container technology choosed, specify --docker or --singularity, see --help for more information' if (!checkContainers(containers,containersList)) exit 1, 'Unknown container(s), see --help for more information' @@ -101,68 +105,68 @@ dockerContainers = containers singularityContainers = containers process BuildDockerContainers { - tag {repository + "/" + container + ":" + tag} + tag {"${params.repository}/${container}:${tag}"} input: val container from dockerContainers output: - val container into dockerContainersBuilt + val container into containersBuilt - when: docker + when: params.docker script: """ - docker build -t $repository/$container:$tag $baseDir/containers/$container/. + docker build -t ${params.repository}/${container}:${tag} ${baseDir}/containers/${container}/. """ } -if (verbose) dockerContainersBuilt = dockerContainersBuilt.view { - "Docker container: $repository/$it:$tag built." +if (verbose) containersBuilt = containersBuilt.view { + "Docker container: ${params.repository}/${it}:${tag} built." } process PullSingularityContainers { - tag {repository + "/" + container + ":" + tag} + tag {"${params.repository}/${container}:${tag}"} - publishDir containerPath, mode: 'move' + publishDir "${params.containerPath}", mode: 'move' input: val container from singularityContainers output: - file("*.img") into singularityContainersPulled + file("${container}-${tag}.img") into imagePulled - when: singularity + when: params.singularity script: """ - singularity pull --name $container-${tag}.img docker://$repository/$container:$tag + singularity pull --name ${container}-${tag}.img docker://${params.repository}/${container}:${tag} """ } -if (verbose) singularityContainersPulled = singularityContainersPulled.view { - "Singularity container: $it pulled." +if (verbose) imagePulled = imagePulled.view { + "Singularity image: ${it.fileName} pulled." } process PushDockerContainers { - tag {repository + "/" + container + ":" + tag} + tag {params.repository + "/" + container + ":" + tag} input: - val container from dockerContainersBuilt + val container from containersBuilt output: - val container into dockerContainersPushed + val container into containersPushed - when: docker && push + when: params.docker && push script: """ - docker push $repository/$container:$tag + docker push ${params.repository}/${container}:${tag} """ } -if (verbose) dockerContainersPushed = dockerContainersPushed.view { - "Docker container: $repository/$it:$tag pushed." +if (verbose) containersPushed = containersPushed.view { + "Docker container: ${params.repository}/${it}:${tag} pushed." } /* @@ -173,13 +177,13 @@ if (verbose) dockerContainersPushed = dockerContainersPushed.view { def cawMessage() { // Display CAW message - log.info "CANCER ANALYSIS WORKFLOW ~ $version - " + this.grabRevision() + (workflow.commitId ? " [$workflow.commitId]" : "") + log.info "CANCER ANALYSIS WORKFLOW ~ ${version} - " + this.grabRevision() + (workflow.commitId ? " [${workflow.commitId}]" : "") } def checkContainerExistence(container, list) { try {assert list.contains(container)} catch (AssertionError ae) { - println("Unknown container: $container") + println("Unknown container: ${container}") return false } return true @@ -210,6 +214,7 @@ def checkParams(it) { 'containerPath', 'containers', 'docker', + 'genome_base', 'genome', 'genomes', 'help', @@ -217,6 +222,9 @@ def checkParams(it) { 'no-reports', 'noGVCF', 'noReports', + 'out-dir', + 'outDir', + 'params', 'project', 'push', 'repository', @@ -273,12 +281,12 @@ def helpMessage() { // Display help message this.cawMessage() log.info " Usage:" - log.info " nextflow run SciLifeLab/buildContainers.nf [--docker] [--push]" + log.info " nextflow run SciLifeLab/CAW/buildContainers.nf [--docker] [--push]" log.info " [--containers ] [--singularity]" log.info " [--containerPath ]" log.info " [--tag ] [--repository ]" log.info " Example:" - log.info " nextflow run . --docker --containers multiqc,fastqc" + log.info " nextflow run SciLifeLab/CAW/buildContainers.nf --docker --containers caw" log.info " --containers: Choose which containers to build" log.info " Default: all" log.info " Possible values:" @@ -291,10 +299,10 @@ def helpMessage() { log.info " --push: Push containers to DockerHub" log.info " --repository: Build containers under given repository" log.info " Default: maxulysse" - log.info " --singularity: Build containers using Singularity" - log.info " --containerPath: Select where to download containers" - log.info " Default: $PWD" - log.info " --tag`: Build containers using given tag" + log.info " --singularity: Download Singularity images" + log.info " --containerPath: Select where to download images" + log.info " Default: \$PWD" + log.info " --tag`: Choose the tag for the containers" log.info " Default (version number): " + version log.info " --version" log.info " displays version number and more informations" @@ -314,16 +322,17 @@ def isAllowedParams(params) { def minimalInformationMessage() { // Minimal information message - log.info "Command Line: $workflow.commandLine" - log.info "Project Dir : $workflow.projectDir" - log.info "Launch Dir : $workflow.launchDir" - log.info "Work Dir : $workflow.workDir" + log.info "Command Line: " + workflow.commandLine + log.info "Project Dir : " + workflow.projectDir + log.info "Launch Dir : " + workflow.launchDir + log.info "Work Dir : " + workflow.workDir + log.info "Cont. Path : " + params.containerPath log.info "Containers : " + containers.join(', ') } def nextflowMessage() { // Nextflow message (version + build) - log.info "N E X T F L O W ~ version $workflow.nextflow.version $workflow.nextflow.build" + log.info "N E X T F L O W ~ version ${workflow.nextflow.version} ${workflow.nextflow.build}" } def startMessage() { @@ -335,8 +344,8 @@ def startMessage() { def versionMessage() { // Display version message log.info "CANCER ANALYSIS WORKFLOW" - log.info " version : $version" - log.info workflow.commitId ? "Git info : $workflow.repository - $workflow.revision [$workflow.commitId]" : " revision : " + this.grabRevision() + log.info " version : " + version + log.info workflow.commitId ? "Git info : ${workflow.repository} - ${workflow.revision} [${workflow.commitId}]" : " revision : " + this.grabRevision() } workflow.onComplete { @@ -344,10 +353,10 @@ workflow.onComplete { this.nextflowMessage() this.cawMessage() this.minimalInformationMessage() - log.info "Completed at: $workflow.complete" - log.info "Duration : $workflow.duration" - log.info "Success : $workflow.success" - log.info "Exit status : $workflow.exitStatus" + log.info "Completed at: " + workflow.complete + log.info "Duration : " + workflow.duration + log.info "Success : " + workflow.success + log.info "Exit status : " + workflow.exitStatus log.info "Error report: " + (workflow.errorReport ?: '-') } @@ -355,5 +364,6 @@ workflow.onError { // Display error message this.nextflowMessage() this.cawMessage() - log.info "Workflow execution stopped with the following message: " + workflow.errorMessage + log.info "Workflow execution stopped with the following message:" + log.info " " + workflow.errorMessage } diff --git a/buildReferences.nf b/buildReferences.nf index c3e52718eb..14a3e46be8 100644 --- a/buildReferences.nf +++ b/buildReferences.nf @@ -39,18 +39,18 @@ kate: syntax groovy; space-indent on; indent-width 2; ================================================================================ */ -version = '1.2.4' +version = '1.2.5' // Check that Nextflow version is up to date enough // try / throw / catch works for NF versions < 0.25 when this was implemented nf_required_version = '0.25.0' try { - if( ! nextflow.version.matches(">= $nf_required_version") ){ + if( ! nextflow.version.matches(">= ${nf_required_version}") ){ throw GroovyException('Nextflow version too old') } } catch (all) { log.error "====================================================\n" + - " Nextflow version $nf_required_version required! You are running v$workflow.nextflow.version.\n" + + " Nextflow version ${nf_required_version} required! You are running v${workflow.nextflow.version}.\n" + " Pipeline execution will continue, but things may break.\n" + " Please update Nextflow.\n" + "============================================================" @@ -58,14 +58,16 @@ try { if (params.help) exit 0, helpMessage() if (params.version) exit 0, versionMessage() -if (!isAllowedParams(params)) exit 1, "params is unknown, see --help for more information" +if (!isAllowedParams(params)) exit 1, "params unknown, see --help for more information" if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project " // Default params: // Such params are overridden by command line or configuration definitions -// No download +// No download of reference source files params.download = false +// outDir is References/${params.genome} +params.outDir = "${baseDir}/References/${params.genome}" // refDir is empty params.refDir = '' @@ -91,7 +93,7 @@ if (params.genome == "smallGRCh37") { [ '1000G_phase1.indels.b37.vcf.gz', '1000G_phase3_20130502_SNP_maf0.3.loci.tar.bz2', - 'b37_cosmic_v74.noCHR.sort.4.1.vcf.tar.bz2', + 'GRCh37_Cosmic_v83.vcf.tar.bz2', 'dbsnp_138.b37.vcf.gz', 'human_g1k_v37_decoy.fasta.gz', 'Mills_and_1000G_gold_standard.indels.b37.vcf.gz', @@ -99,7 +101,7 @@ if (params.genome == "smallGRCh37") { ] } else exit 1, "Can't build this reference genome" -if (download && params.genome != "smallGRCh37") exit 1, "Not possible to download $params.genome references files" +if (download && params.genome != "smallGRCh37") exit 1, "Not possible to download ${params.genome} references files" if (!download) referencesFiles.each{checkFile(params.refDir + "/" + it)} @@ -124,18 +126,18 @@ process ProcessReference { if (download) """ - wget https://github.com/szilvajuhos/smallRef/raw/master/$reference + wget https://github.com/szilvajuhos/smallRef/raw/master/${reference} """ else """ - ln -s $params.refDir/$reference . + ln -s ${params.refDir}/${reference} . """ } if (verbose) processedFiles = processedFiles.view { - "Files preprocessed : $it.fileName" + "Files preprocessed : ${it.fileName}" } compressedfiles = Channel.create() @@ -154,19 +156,19 @@ process DecompressFile { file("*.{vcf,fasta,loci}") into decompressedFiles script: - realReference="readlink $reference" + realReference="readlink ${reference}" if (reference =~ ".gz") """ - gzip -d -c \$($realReference) > $reference.baseName + gzip -d -c \$(${realReference}) > ${reference.baseName} """ else if (reference =~ ".tar.bz2") """ - tar xvjf \$($realReference) + tar xvjf \$(${realReference}) """ } if (verbose) decompressedFiles = decompressedFiles.view { - "Files decomprecessed: $it.fileName" + "Files decomprecessed: ${it.fileName}" } fastaFile = Channel.create() @@ -180,7 +182,7 @@ decompressedFiles notCompressedfiles .mix(otherFiles) - .collectFile(storeDir: "References/" + params.genome) + .collectFile(storeDir: params.outDir) fastaForBWA = Channel.create() fastaForPicard = Channel.create() @@ -191,7 +193,7 @@ fastaFile.into(fastaForBWA,fastaForPicard,fastaForSAMTools) process BuildBWAindexes { tag {reference} - publishDir "References/" + params.genome, mode: 'copy' + publishDir params.outDir, mode: 'copy' input: file(reference) from fastaForBWA @@ -203,21 +205,21 @@ process BuildBWAindexes { script: """ - bwa index $reference + bwa index ${reference} """ } if (verbose) fastaFileToKeep.view { - "Fasta File : $it.fileName" + "Fasta File : ${it.fileName}" } if (verbose) bwaIndexes.flatten().view { - "BWA index : $it.fileName" + "BWA index : ${it.fileName}" } process BuildPicardIndex { tag {reference} - publishDir "References/" + params.genome, mode: 'copy' + publishDir params.outDir, mode: 'copy' input: file(reference) from fastaForPicard @@ -230,19 +232,19 @@ process BuildPicardIndex { java -Xmx${task.memory.toGiga()}g \ -jar \$PICARD_HOME/picard.jar \ CreateSequenceDictionary \ - REFERENCE=$reference \ + REFERENCE=${reference} \ OUTPUT=${reference.baseName}.dict """ } if (verbose) picardIndex.view { - "Picard index : $it.fileName" + "Picard index : ${it.fileName}" } process BuildSAMToolsIndex { tag {reference} - publishDir "References/" + params.genome, mode: 'copy' + publishDir params.outDir, mode: 'copy' input: file(reference) from fastaForSAMTools @@ -252,18 +254,18 @@ process BuildSAMToolsIndex { script: """ - samtools faidx $reference + samtools faidx ${reference} """ } if (verbose) samtoolsIndex.view { - "SAMTools index : $it.fileName" + "SAMTools index : ${it.fileName}" } process BuildVCFIndex { tag {reference} - publishDir "References/" + params.genome, mode: 'copy' + publishDir params.outDir, mode: 'copy' input: file(reference) from vcfFiles @@ -274,15 +276,15 @@ process BuildVCFIndex { script: """ - \$IGVTOOLS_HOME/igvtools index $reference + \$IGVTOOLS_HOME/igvtools index ${reference} """ } if (verbose) vcfIndexed.view { - "VCF indexed : $it.fileName" + "VCF indexed : ${it.fileName}" } if (verbose) vcfIndex.view { - "VCF index : $it.fileName" + "VCF index : ${it.fileName}" } /* @@ -293,13 +295,13 @@ if (verbose) vcfIndex.view { def cawMessage() { // Display CAW message - log.info "CANCER ANALYSIS WORKFLOW ~ $version - " + this.grabRevision() + (workflow.commitId ? " [$workflow.commitId]" : "") + log.info "CANCER ANALYSIS WORKFLOW ~ ${version} - " + this.grabRevision() + (workflow.commitId ? " [${workflow.commitId}]" : "") } def checkFile(it) { // Check file existence final f = file(it) - if (!f.exists()) exit 1, "Missing file: $it, see --help for more information" + if (!f.exists()) exit 1, "Missing file: ${it}, see --help for more information" return true } @@ -320,6 +322,7 @@ def checkParams(it) { 'containers', 'docker', 'download', + 'genome_base', 'genome', 'genomes', 'help', @@ -327,6 +330,9 @@ def checkParams(it) { 'no-reports', 'noGVCF', 'noReports', + 'out-dir', + 'outDir', + 'params', 'project', 'push', 'ref-dir', @@ -372,11 +378,14 @@ def helpMessage() { log.info " Download reference files. (only with --genome smallGRCh37)" log.info " --refDir " log.info " Specify a directory containing reference files." + log.info " --outDir " + log.info " Specify an output directory" + log.info " Default: \$PWD/References/" log.info " --genome " - log.info " Use a specific genome version." + log.info " Choose which genome to build references from" log.info " Possible values are:" log.info " GRCh37" - log.info " smallGRCh37 (Build a small reference (for tests))" + log.info " smallGRCh37" log.info " --help" log.info " you're reading it" log.info " --version" @@ -397,16 +406,17 @@ def isAllowedParams(params) { def minimalInformationMessage() { // Minimal information message - log.info "Command Line: $workflow.commandLine" - log.info "Project Dir : $workflow.projectDir" - log.info "Launch Dir : $workflow.launchDir" - log.info "Work Dir : $workflow.workDir" + log.info "Command Line: " + workflow.commandLine + log.info "Project Dir : " + workflow.projectDir + log.info "Launch Dir : " + workflow.launchDir + log.info "Work Dir : " + workflow.workDir + log.info "Out Dir : " + params.outDir log.info "Genome : " + params.genome } def nextflowMessage() { // Nextflow message (version + build) - log.info "N E X T F L O W ~ version $workflow.nextflow.version $workflow.nextflow.build" + log.info "N E X T F L O W ~ version ${workflow.nextflow.version} ${workflow.nextflow.build}" } def startMessage() { @@ -418,8 +428,8 @@ def startMessage() { def versionMessage() { // Display version message log.info "CANCER ANALYSIS WORKFLOW" - log.info " version : $version" - log.info workflow.commitId ? "Git info : $workflow.repository - $workflow.revision [$workflow.commitId]" : " revision : " + this.grabRevision() + log.info " version : " + version + log.info workflow.commitId ? "Git info : ${workflow.repository} - ${workflow.revision} [${workflow.commitId}]" : " revision : " + this.grabRevision() } workflow.onComplete { @@ -427,10 +437,10 @@ workflow.onComplete { this.nextflowMessage() this.cawMessage() this.minimalInformationMessage() - log.info "Completed at: $workflow.complete" - log.info "Duration : $workflow.duration" - log.info "Success : $workflow.success" - log.info "Exit status : $workflow.exitStatus" + log.info "Completed at: " + workflow.complete + log.info "Duration : " + workflow.duration + log.info "Success : " + workflow.success + log.info "Exit status : " + workflow.exitStatus log.info "Error report: " + (workflow.errorReport ?: '-') } @@ -438,5 +448,6 @@ workflow.onError { // Display error message this.nextflowMessage() this.cawMessage() - log.info "Workflow execution stopped with the following message: " + workflow.errorMessage + log.info "Workflow execution stopped with the following message:" + log.info " " + workflow.errorMessage } diff --git a/configuration/aws-batch.config b/configuration/aws-batch.config new file mode 100644 index 0000000000..b91e10bf93 --- /dev/null +++ b/configuration/aws-batch.config @@ -0,0 +1,30 @@ +/* +vim: syntax=groovy +-*- mode: groovy;-*- + * ------------------------------------------------- + * Nextflow config file for Amazon Web Services + * ------------------------------------------------- + * Imported under the 'aws' Nextflow profile in nextflow.config + * Defines reference genomes, using paths from s3 + * To be use with AWS Batch + */ + +params { + genome_base = params.genome == 'GRCh37' ? "s3://caw-references/grch37" : params.genome == 'GRCh38' ? "s3://caw-references/grch38" : "s3://caw-references/smallgrch37" + repository='maxulysse' + tag='1.2.5' +} + +executor.name = 'awsbatch' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' + +process { + executor = 'awsbatch' + queue = 'caw-job-queue' + + errorStrategy = {task.exitStatus == 143 ? 'retry' : 'terminate'} + maxErrors = '-1' + maxRetries = 2 + cpus = 2 + memory = 7.GB +} diff --git a/configuration/base.config b/configuration/base.config new file mode 100644 index 0000000000..eba6dcedc0 --- /dev/null +++ b/configuration/base.config @@ -0,0 +1,17 @@ +/* +vim: syntax=groovy +-*- mode: groovy;-*- + * ------------------------------------------------- + * Nextflow config file for CAW project + * ------------------------------------------------- + * Basic configuration for some process + * ------------------------------------------------- + */ + +process { + $ConcatVCF { + // For unknown reasons, ConcatVCF sometimes fails with SIGPIPE + // (exit code 141). Rerunning the process will usually work. + errorStrategy = {task.exitStatus == 141 ? 'retry' : 'terminate'} + } +} diff --git a/configuration/containers.config b/configuration/containers.config index d8015cca13..b35c6d701b 100644 --- a/configuration/containers.config +++ b/configuration/containers.config @@ -10,6 +10,10 @@ vim: syntax=groovy */ process { + // Add container for processes without containers + $CreateIntervalBeds.container = "${params.repository}/caw:${params.tag}" + $GenerateMultiQCconfig.container = "${params.repository}/caw:${params.tag}" + $BuildBWAindexes.container = "${params.repository}/caw:${params.tag}" $BuildPicardIndex.container = "${params.repository}/picard:${params.tag}" $BuildSAMToolsIndex.container = "${params.repository}/caw:${params.tag}" @@ -38,7 +42,7 @@ process { $RunSamtoolsStats.container = "${params.repository}/caw:${params.tag}" $RunSingleManta.container = "${params.repository}/caw:${params.tag}" $RunSingleStrelka.container = "${params.repository}/caw:${params.tag}" - $RunSnpeff.container = {params.genome == "GRCh38" ? "${params.repository}/snpeffgrch38:${params.tag}" : "${params.repository}/snpeffgrch37:${params.tag}"} + $RunSnpeff.container = {params.genome == 'GRCh38' ? "${params.repository}/snpeffgrch38:${params.tag}" : "${params.repository}/snpeffgrch37:${params.tag}"} $RunStrelka.container = "${params.repository}/caw:${params.tag}" - $RunVEP.container = {params.genome == "GRCh38" ? "${params.repository}/vepgrch38:${params.tag}" : "${params.repository}/vepgrch37:${params.tag}"} + $RunVEP.container = {params.genome == 'GRCh38' ? "${params.repository}/vepgrch38:${params.tag}" : "${params.repository}/vepgrch37:${params.tag}"} } diff --git a/configuration/docker.config b/configuration/docker.config index 69711042ad..876fc9c5a6 100644 --- a/configuration/docker.config +++ b/configuration/docker.config @@ -14,5 +14,7 @@ docker { fixOwnership = true } -params.repository='maxulysse' -params.tag='1.2.3' +params { + repository='maxulysse' + tag='1.2.5' +} diff --git a/configuration/genomes.config b/configuration/genomes.config index 406dec6fe0..f65ea98c30 100644 --- a/configuration/genomes.config +++ b/configuration/genomes.config @@ -16,51 +16,48 @@ vim: syntax=groovy params { genomes { 'GRCh37' { - bundleDir = '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' - acLoci = "$bundleDir/1000G_phase3_20130502_SNP_maf0.3.loci" - cosmic = "$bundleDir/GRCh37_Cosmic_v83.vcf" + acLoci = "${params.genome_base}/1000G_phase3_20130502_SNP_maf0.3.loci" + cosmic = "${params.genome_base}/GRCh37_Cosmic_v83.vcf" cosmicIndex = "${cosmic}.idx" - dbsnp = "$bundleDir/dbsnp_138.b37.vcf" + dbsnp = "${params.genome_base}/dbsnp_138.b37.vcf" dbsnpIndex = "${dbsnp}.idx" - genomeFile = "$bundleDir/human_g1k_v37_decoy.fasta" + genomeFile = "${params.genome_base}/human_g1k_v37_decoy.fasta" bwaIndex = "${genomeFile}.{amb,ann,bwt,pac,sa}" - genomeDict = "$bundleDir/human_g1k_v37_decoy.dict" + genomeDict = "${params.genome_base}/human_g1k_v37_decoy.dict" genomeIndex = "${genomeFile}.fai" - intervals = "$bundleDir/wgs_calling_regions_CAW.list" - knownIndels = "$bundleDir/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.vcf" - knownIndelsIndex = "$bundleDir/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.vcf.idx" + intervals = "${params.genome_base}/wgs_calling_regions_CAW.list" + knownIndels = "${params.genome_base}/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.vcf" + knownIndelsIndex = "${params.genome_base}/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.vcf.idx" snpeffDb = "GRCh37.75" } 'GRCh38' { - bundleDir = '/sw/data/uppnex/ToolBox/hg38bundle' - acLoci = "${bundleDir}/1000G_phase3_GRCh38_maf0.3.loci" - cosmic = "${bundleDir}/COSMICv80.vcf" + acLoci = "${params.genome_base}/1000G_phase3_GRCh38_maf0.3.loci" + cosmic = "${params.genome_base}/COSMICv80.vcf" cosmicIndex = "${cosmic}.idx" - dbsnp = "$bundleDir/dbsnp_146.hg38.vcf.gz" + dbsnp = "${params.genome_base}/dbsnp_146.hg38.vcf.gz" dbsnpIndex = "${dbsnp}.tbi" - genomeFile = "$bundleDir/Homo_sapiens_assembly38.fasta" - genomeDict = "$bundleDir/Homo_sapiens_assembly38.dict" + genomeFile = "${params.genome_base}/Homo_sapiens_assembly38.fasta" + genomeDict = "${params.genome_base}/Homo_sapiens_assembly38.dict" genomeIndex = "${genomeFile}.fai" bwaIndex = "${genomeFile}.64.{amb,ann,bwt,pac,sa,alt}" - intervals = "$bundleDir/wgs_calling_regions.hg38.bed" - knownIndels = "$bundleDir/{Mills_and_1000G_gold_standard.indels.hg38,beta/Homo_sapiens_assembly38.known_indels}.vcf.gz" - knownIndelsIndex = "$bundleDir/{Mills_and_1000G_gold_standard.indels.hg38,beta/Homo_sapiens_assembly38.known_indels}.vcf.gz.tbi" + intervals = "${params.genome_base}/wgs_calling_regions.hg38.bed" + knownIndels = "${params.genome_base}/{Mills_and_1000G_gold_standard.indels.hg38,beta/Homo_sapiens_assembly38.known_indels}.vcf.gz" + knownIndelsIndex = "${params.genome_base}/{Mills_and_1000G_gold_standard.indels.hg38,beta/Homo_sapiens_assembly38.known_indels}.vcf.gz.tbi" snpeffDb = "GRCh38.86" } 'smallGRCh37' { - bundleDir = 'References/smallGRCh37' - acLoci = "$bundleDir/1000G_phase3_20130502_SNP_maf0.3.small.loci" - cosmic = "$bundleDir/b37_cosmic_v74.noCHR.sort.4.1.small.vcf" + acLoci = "${params.genome_base}/1000G_phase3_20130502_SNP_maf0.3.small.loci" + cosmic = "${params.genome_base}/b37_cosmic_v74.noCHR.sort.4.1.small.vcf" cosmicIndex = "${cosmic}.idx" - dbsnp = "$bundleDir/dbsnp_138.b37.small.vcf" + dbsnp = "${params.genome_base}/dbsnp_138.b37.small.vcf" dbsnpIndex = "${dbsnp}.idx" - genomeFile = "$bundleDir/human_g1k_v37_decoy.small.fasta" + genomeFile = "${params.genome_base}/human_g1k_v37_decoy.small.fasta" bwaIndex = "${genomeFile}.{amb,ann,bwt,pac,sa}" - genomeDict = "$bundleDir/human_g1k_v37_decoy.small.dict" + genomeDict = "${params.genome_base}/human_g1k_v37_decoy.small.dict" genomeIndex = "${genomeFile}.fai" - intervals = "$bundleDir/small.intervals" - knownIndels = "$bundleDir/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.small.vcf" - knownIndelsIndex = "$bundleDir/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.small.vcf.idx" + intervals = "${params.genome_base}/small.intervals" + knownIndels = "${params.genome_base}/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.small.vcf" + knownIndelsIndex = "${params.genome_base}/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.small.vcf.idx" snpeffDb = "GRCh37.75" } } diff --git a/configuration/singularity-path.config b/configuration/singularity-path.config index 8ce9cb3d93..b96b78edf8 100644 --- a/configuration/singularity-path.config +++ b/configuration/singularity-path.config @@ -15,8 +15,10 @@ singularity { runOptions = "--bind /scratch" } -params.containerPath='containers' -params.tag='1.2.3' +params { + containerPath='containers' + tag='1.2.5' +} process { $BuildBWAindexes.container = "${params.containerPath}/caw-${params.tag}.img" @@ -48,7 +50,7 @@ process { $RunSamtoolsStats.container = "${params.containerPath}/caw-${params.tag}.img" $RunSingleManta.container = "${params.containerPath}/caw-${params.tag}.img" $RunSingleStrelka.container = "${params.containerPath}/caw-${params.tag}.img" - $RunSnpeff.container = {params.genome == "GRCh38" ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"} + $RunSnpeff.container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"} $RunStrelka.container = "${params.containerPath}/caw-${params.tag}.img" - $RunVEP.container = {params.genome == "GRCh38" ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"} + $RunVEP.container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"} } diff --git a/configuration/singularity.config b/configuration/singularity.config index 198b45a9e2..429cf47b0b 100644 --- a/configuration/singularity.config +++ b/configuration/singularity.config @@ -13,5 +13,7 @@ singularity { enabled = true } -params.repository='docker://maxulysse' -params.tag='1.2.3' +params { + repository='docker://maxulysse' + tag='1.2.5' +} diff --git a/configuration/travis.config b/configuration/travis.config index 4247856093..010669bc1b 100644 --- a/configuration/travis.config +++ b/configuration/travis.config @@ -11,6 +11,7 @@ vim: syntax=groovy params { genome = 'smallGRCh37' + genome_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : params.genome == 'GRCh38' ? '/sw/data/uppnex/ToolBox/hg38bundle' : 'References/smallGRCh37' } process { diff --git a/configuration/uppmax-localhost.config b/configuration/uppmax-localhost.config index ba91dc2508..0ef861d959 100644 --- a/configuration/uppmax-localhost.config +++ b/configuration/uppmax-localhost.config @@ -14,6 +14,7 @@ env { } params { + genome_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : params.genome == 'GRCh38' ? '/sw/data/uppnex/ToolBox/hg38bundle' : 'References/smallGRCh37' singleCPUMem = 8.GB totalMemory = 104.GB // change to 240 on irma } @@ -58,9 +59,6 @@ process { $ConcatVCF { cpus = 8 - // For unknown reasons, ConcatVCF sometimes fails with SIGPIPE - // (exit code 141). Rerunning the process will usually work. - errorStrategy = {task.exitStatus == 141 ? 'retry' : 'terminate'} } $CreateRecalibrationTable { cpus = 16 diff --git a/configuration/uppmax-slurm.config b/configuration/uppmax-slurm.config index 38549cc48b..ae38a5866c 100644 --- a/configuration/uppmax-slurm.config +++ b/configuration/uppmax-slurm.config @@ -9,6 +9,7 @@ vim: syntax=groovy */ params { + genome_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : params.genome == 'GRCh38' ? '/sw/data/uppnex/ToolBox/hg38bundle' : 'References/smallGRCh37' runTime = 48.h singleCPUMem = 7.GB // for processes that are using more memory but a single CPU only. Use the 'core' queue for these } diff --git a/containers/multiqc/Dockerfile b/containers/multiqc/Dockerfile index 42f7e99e15..8b5cf6541d 100644 --- a/containers/multiqc/Dockerfile +++ b/containers/multiqc/Dockerfile @@ -1,12 +1,9 @@ -FROM python:2.7-slim +FROM ewels/multiqc:v1.3 LABEL \ author="Maxime Garcia" \ description="MultiQC image used in CAW" \ maintainer="maxime.garcia@scilifelab.se" -# Build container -ADD \ - build.sh /usr/bin/ -RUN \ - build.sh +# Create UPPMAX directories +RUN mkdir /pica /proj /scratch /sw diff --git a/containers/multiqc/build.sh b/containers/multiqc/build.sh deleted file mode 100755 index 4c6bf899d6..0000000000 --- a/containers/multiqc/build.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -set -eu pipefail - -# Local ENV variables -MULTIQC_VERSION=1.1 - -# Install libraries -apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - git \ - wget - -# Install pip -wget --quiet -O /opt/get-pip.py https://bootstrap.pypa.io/get-pip.py -python /opt/get-pip.py - -# Install tools -pip install networkx==1.11 # fix issue 592 -pip install git+git://github.com/ewels/MultiQC.git@v${MULTIQC_VERSION} - -# Clean up install -cd / -apt-get remove -y \ - g++ \ - git \ - wget -rm -rf /build /var/lib/apt/lists/* /opt/get-pip.py - -# Create UPPMAX directories -mkdir /pica /proj /scratch /sw diff --git a/containers/mutect1/Dockerfile b/containers/mutect1/Dockerfile index 7c0d2a3f6e..1e2c72e1fa 100644 --- a/containers/mutect1/Dockerfile +++ b/containers/mutect1/Dockerfile @@ -23,5 +23,6 @@ RUN \ && unzip muTect-${MUTECT_VERSION}-bin.zip -d ${MUTECT_HOME} \ && rm muTect-${MUTECT_VERSION}-bin.zip \ && mv ${MUTECT_HOME}/muTect-${MUTECT_VERSION}.jar ${MUTECT_HOME}/muTect.jar + # Create UPPMAX directories RUN mkdir /pica /proj /scratch /sw diff --git a/data/tiny/dummy_normal_reads_1.fastq.gz b/data/tiny/dummy/normal/dummy_n_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/dummy_normal_reads_1.fastq.gz rename to data/tiny/dummy/normal/dummy_n_R1_xxx.fastq.gz diff --git a/data/tiny/dummy_normal_reads_2.fastq.gz b/data/tiny/dummy/normal/dummy_n_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/dummy_normal_reads_2.fastq.gz rename to data/tiny/dummy/normal/dummy_n_R2_xxx.fastq.gz diff --git a/data/tiny/dummy_tumour_reads_1.fastq.gz b/data/tiny/dummy/tumor/dummy_t_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/dummy_tumour_reads_1.fastq.gz rename to data/tiny/dummy/tumor/dummy_t_R1_xxx.fastq.gz diff --git a/data/tiny/dummy_tumour_reads_2.fastq.gz b/data/tiny/dummy/tumor/dummy_t_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/dummy_tumour_reads_2.fastq.gz rename to data/tiny/dummy/tumor/dummy_t_R2_xxx.fastq.gz diff --git a/data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_1.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_1.fastq.gz rename to data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R1_xxx.fastq.gz diff --git a/data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_2.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_2.fastq.gz rename to data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R2_xxx.fastq.gz diff --git a/data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_1.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_1.fastq.gz rename to data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R1_xxx.fastq.gz diff --git a/data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_2.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_2.fastq.gz rename to data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R2_xxx.fastq.gz diff --git a/data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_1.fastq.gz b/data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_1.fastq.gz rename to data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R1_xxx.fastq.gz diff --git a/data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_2.fastq.gz b/data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_2.fastq.gz rename to data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R2_xxx.fastq.gz diff --git a/data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_1.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_1.fastq.gz rename to data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R1_xxx.fastq.gz diff --git a/data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_2.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_2.fastq.gz rename to data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R2_xxx.fastq.gz diff --git a/data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_1.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_1.fastq.gz rename to data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R1_xxx.fastq.gz diff --git a/data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_2.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_2.fastq.gz rename to data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R2_xxx.fastq.gz diff --git a/data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_1.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_1.fastq.gz rename to data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R1_xxx.fastq.gz diff --git a/data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_2.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_2.fastq.gz rename to data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R2_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.3.ACCAACTG_1.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.3.ACCAACTG_1.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R1_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.3.ACCAACTG_2.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.3.ACCAACTG_2.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R2_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.5.ACCAACTG_1.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.5.ACCAACTG_1.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R1_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.5.ACCAACTG_2.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.5.ACCAACTG_2.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R2_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.6.ACCAACTG_1.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.6.ACCAACTG_1.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R1_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.6.ACCAACTG_2.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.6.ACCAACTG_2.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R2_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.7.ACCAACTG_1.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.7.ACCAACTG_1.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R1_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.7.ACCAACTG_2.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.7.ACCAACTG_2.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R2_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.8.ACCAACTG_1.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.8.ACCAACTG_1.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R1_xxx.fastq.gz diff --git a/data/tiny/C09D_MANTA_TUMOUR_111207.8.ACCAACTG_2.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/C09D_MANTA_TUMOUR_111207.8.ACCAACTG_2.fastq.gz rename to data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R2_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L001_R1_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L001_R1_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L001_R2_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L001_R2_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L002_R1_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L002_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L002_R1_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L002_R1_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L002_R2_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L002_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L002_R2_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L002_R2_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L004_R1_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L004_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L004_R1_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L004_R1_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L004_R2_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L004_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L004_R2_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L004_R2_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L007_R1_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L007_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L007_R1_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L007_R1_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L007_R2_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L007_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L007_R2_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L007_R2_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L008_R1_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L008_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L008_R1_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L008_R1_xxx.fastq.gz diff --git a/data/tiny/tiny/normal/tiny_normal_L008_R2_TEST.fastq.gz b/data/tiny/tiny/normal/tiny_n_L008_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny/normal/tiny_normal_L008_R2_TEST.fastq.gz rename to data/tiny/tiny/normal/tiny_n_L008_R2_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L001_R1.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L001_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L001_R1.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L001_R1_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L001_R2.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L001_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L001_R2.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L001_R2_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L002_R1.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L002_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L002_R1.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L002_R1_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L002_R2.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L002_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L002_R2.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L002_R2_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L003_R1.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L003_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L003_R1.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L003_R1_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L003_R2.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L003_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L003_R2.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L003_R2_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L005_R1.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L005_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L005_R1.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L005_R1_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L005_R2.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L005_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L005_R2.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L005_R2_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L006_R1.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L006_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L006_R1.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L006_R1_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L006_R2.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L006_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L006_R2.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L006_R2_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L007_R1.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L007_R1_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L007_R1.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L007_R1_xxx.fastq.gz diff --git a/data/tiny/tiny_tumor_L007_R2.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L007_R2_xxx.fastq.gz similarity index 100% rename from data/tiny/tiny_tumor_L007_R2.fastq.gz rename to data/tiny/tiny/tumor/tiny_t_L007_R2_xxx.fastq.gz diff --git a/data/tiny/tiny_normal_L001_R1.fastq.gz b/data/tiny/tiny_normal_L001_R1.fastq.gz deleted file mode 100644 index b25487aa91..0000000000 Binary files a/data/tiny/tiny_normal_L001_R1.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L001_R2.fastq.gz b/data/tiny/tiny_normal_L001_R2.fastq.gz deleted file mode 100644 index b917c80cef..0000000000 Binary files a/data/tiny/tiny_normal_L001_R2.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L002_R1.fastq.gz b/data/tiny/tiny_normal_L002_R1.fastq.gz deleted file mode 100644 index 509295fdf7..0000000000 Binary files a/data/tiny/tiny_normal_L002_R1.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L002_R2.fastq.gz b/data/tiny/tiny_normal_L002_R2.fastq.gz deleted file mode 100644 index ae0e1e6ee7..0000000000 Binary files a/data/tiny/tiny_normal_L002_R2.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L004_R1.fastq.gz b/data/tiny/tiny_normal_L004_R1.fastq.gz deleted file mode 100644 index 29e4a3bd15..0000000000 Binary files a/data/tiny/tiny_normal_L004_R1.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L004_R2.fastq.gz b/data/tiny/tiny_normal_L004_R2.fastq.gz deleted file mode 100644 index f645870a8f..0000000000 Binary files a/data/tiny/tiny_normal_L004_R2.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L007_R1.fastq.gz b/data/tiny/tiny_normal_L007_R1.fastq.gz deleted file mode 100644 index 8e3c3d295a..0000000000 Binary files a/data/tiny/tiny_normal_L007_R1.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L007_R2.fastq.gz b/data/tiny/tiny_normal_L007_R2.fastq.gz deleted file mode 100644 index d03e2369c0..0000000000 Binary files a/data/tiny/tiny_normal_L007_R2.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L008_R1.fastq.gz b/data/tiny/tiny_normal_L008_R1.fastq.gz deleted file mode 100644 index ec28edfb5d..0000000000 Binary files a/data/tiny/tiny_normal_L008_R1.fastq.gz and /dev/null differ diff --git a/data/tiny/tiny_normal_L008_R2.fastq.gz b/data/tiny/tiny_normal_L008_R2.fastq.gz deleted file mode 100644 index f02dc49a05..0000000000 Binary files a/data/tiny/tiny_normal_L008_R2.fastq.gz and /dev/null differ diff --git a/data/tsv/tiny-manta-s3.tsv b/data/tsv/tiny-manta-s3.tsv new file mode 100644 index 0000000000..60ded74015 --- /dev/null +++ b/data/tsv/tiny-manta-s3.tsv @@ -0,0 +1,11 @@ +1234 XX 0 1234N 1234N_M1 s3://caw-test-data/manta/normal/C097F_N_111207.1.AGTTGCTT_R1_xxx.fastq.gz s3://caw-test-data/manta/normal/C097F_N_111207.1.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M2 s3://caw-test-data/manta/normal/C097F_N_111207.2.AGTTGCTT_R1_xxx.fastq.gz s3://caw-test-data/manta/normal/C097F_N_111207.2.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M4 s3://caw-test-data/manta/normal/C09DF_N_111207.4.AGTTGCTT_R1_xxx.fastq.gz s3://caw-test-data/manta/normal/C09DF_N_111207.4.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M5 s3://caw-test-data/manta/normal/D0F23_N_111212.1.AGTTGCTT_R1_xxx.fastq.gz s3://caw-test-data/manta/normal/D0F23_N_111212.1.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M6 s3://caw-test-data/manta/normal/D0F23_N_111212.2.AGTTGCTT_R1_xxx.fastq.gz s3://caw-test-data/manta/normal/D0F23_N_111212.2.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M7 s3://caw-test-data/manta/normal/D0F23_N_111212.3.AGTTGCTT_R1_xxx.fastq.gz s3://caw-test-data/manta/normal/D0F23_N_111212.3.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M1 s3://caw-test-data/manta/tumor/C09D_T_111207.3.ACCAACTG_R1_xxx.fastq.gz s3://caw-test-data/manta/tumor/C09D_T_111207.3.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M2 s3://caw-test-data/manta/tumor/C09D_T_111207.5.ACCAACTG_R1_xxx.fastq.gz s3://caw-test-data/manta/tumor/C09D_T_111207.5.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M3 s3://caw-test-data/manta/tumor/C09D_T_111207.6.ACCAACTG_R1_xxx.fastq.gz s3://caw-test-data/manta/tumor/C09D_T_111207.6.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M4 s3://caw-test-data/manta/tumor/C09D_T_111207.7.ACCAACTG_R1_xxx.fastq.gz s3://caw-test-data/manta/tumor/C09D_T_111207.7.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M5 s3://caw-test-data/manta/tumor/C09D_T_111207.8.ACCAACTG_R1_xxx.fastq.gz s3://caw-test-data/manta/tumor/C09D_T_111207.8.ACCAACTG_R2_xxx.fastq.gz diff --git a/data/tsv/tiny-manta.tsv b/data/tsv/tiny-manta.tsv index 33a74ea431..2c354fa968 100644 --- a/data/tsv/tiny-manta.tsv +++ b/data/tsv/tiny-manta.tsv @@ -1,11 +1,11 @@ -1234 XX 0 1234N 1234N_M1 data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_1.fastq.gz data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M2 data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_1.fastq.gz data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M4 data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_1.fastq.gz data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M5 data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_1.fastq.gz data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M6 data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_1.fastq.gz data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M7 data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_1.fastq.gz data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_2.fastq.gz -1234 XX 1 9876T 9876T_M1 data/tiny/C09D_MANTA_TUMOUR_111207.3.ACCAACTG_1.fastq.gz data/tiny/C09D_MANTA_TUMOUR_111207.3.ACCAACTG_2.fastq.gz -1234 XX 1 9876T 9876T_M2 data/tiny/C09D_MANTA_TUMOUR_111207.5.ACCAACTG_1.fastq.gz data/tiny/C09D_MANTA_TUMOUR_111207.5.ACCAACTG_2.fastq.gz -1234 XX 1 9876T 9876T_M3 data/tiny/C09D_MANTA_TUMOUR_111207.6.ACCAACTG_1.fastq.gz data/tiny/C09D_MANTA_TUMOUR_111207.6.ACCAACTG_2.fastq.gz -1234 XX 1 9876T 9876T_M4 data/tiny/C09D_MANTA_TUMOUR_111207.7.ACCAACTG_1.fastq.gz data/tiny/C09D_MANTA_TUMOUR_111207.7.ACCAACTG_2.fastq.gz -1234 XX 1 9876T 9876T_M5 data/tiny/C09D_MANTA_TUMOUR_111207.8.ACCAACTG_1.fastq.gz data/tiny/C09D_MANTA_TUMOUR_111207.8.ACCAACTG_2.fastq.gz +1234 XX 0 1234N 1234N_M1 data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R1_xxx.fastq.gz data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M2 data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R1_xxx.fastq.gz data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M4 data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R1_xxx.fastq.gz data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M5 data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R1_xxx.fastq.gz data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M6 data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R1_xxx.fastq.gz data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_M7 data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R1_xxx.fastq.gz data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M1 data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R1_xxx.fastq.gz data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M2 data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R1_xxx.fastq.gz data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M3 data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R1_xxx.fastq.gz data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M4 data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R1_xxx.fastq.gz data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_M5 data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R1_xxx.fastq.gz data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R2_xxx.fastq.gz diff --git a/data/tsv/tiny-s3.tsv b/data/tsv/tiny-s3.tsv new file mode 100644 index 0000000000..e9270bb5a9 --- /dev/null +++ b/data/tsv/tiny-s3.tsv @@ -0,0 +1,13 @@ +1234 XX 0 1234N 1234N_1 s3://caw-test-data/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz s3://caw-test-data/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_2 s3://caw-test-data/tiny/normal/tiny_n_L002_R1_xxx.fastq.gz s3://caw-test-data/tiny/normal/tiny_n_L002_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_4 s3://caw-test-data/tiny/normal/tiny_n_L004_R1_xxx.fastq.gz s3://caw-test-data/tiny/normal/tiny_n_L004_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_7 s3://caw-test-data/tiny/normal/tiny_n_L007_R1_xxx.fastq.gz s3://caw-test-data/tiny/normal/tiny_n_L007_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_8 s3://caw-test-data/tiny/normal/tiny_n_L008_R1_xxx.fastq.gz s3://caw-test-data/tiny/normal/tiny_n_L008_R2_xxx.fastq.gz +1234 XX 0 1234N DUMMY_N s3://caw-test-data/dummy/normal/dummy_n_R1_xxx.fastq.gz s3://caw-test-data/dummy/normal/dummy_n_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_1 s3://caw-test-data/tiny/tumor/tiny_t_L001_R1_xxx.fastq.gz s3://caw-test-data/tiny/tumor/tiny_t_L001_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_2 s3://caw-test-data/tiny/tumor/tiny_t_L002_R1_xxx.fastq.gz s3://caw-test-data/tiny/tumor/tiny_t_L002_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_3 s3://caw-test-data/tiny/tumor/tiny_t_L003_R1_xxx.fastq.gz s3://caw-test-data/tiny/tumor/tiny_t_L003_R2_xxx.fastq.gz +1234 XX 1 9876T DUMMY_T s3://caw-test-data/dummy/tumor/dummy_t_R1_xxx.fastq.gz s3://caw-test-data/dummy/tumor/dummy_t_R2_xxx.fastq.gz +1234 XX 1 9877R 9877R_5 s3://caw-test-data/tiny/tumor/tiny_t_L005_R1_xxx.fastq.gz s3://caw-test-data/tiny/tumor/tiny_t_L005_R2_xxx.fastq.gz +1234 XX 1 9877R 9877R_6 s3://caw-test-data/tiny/tumor/tiny_t_L006_R1_xxx.fastq.gz s3://caw-test-data/tiny/tumor/tiny_t_L006_R2_xxx.fastq.gz +1234 XX 1 9877R 9877R_7 s3://caw-test-data/tiny/tumor/tiny_t_L007_R1_xxx.fastq.gz s3://caw-test-data/tiny/tumor/tiny_t_L007_R2_xxx.fastq.gz diff --git a/data/tsv/tiny-single-manta.tsv b/data/tsv/tiny-single-manta.tsv deleted file mode 100644 index 29a7b5f137..0000000000 --- a/data/tsv/tiny-single-manta.tsv +++ /dev/null @@ -1,6 +0,0 @@ -1234 XX 0 1234N 1234N_M1 data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_1.fastq.gz data/tiny/C097F_MANTA_NORMAL_111207.1.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M2 data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_1.fastq.gz data/tiny/C097F_MANTA_NORMAL_111207.2.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M4 data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_1.fastq.gz data/tiny/C09DF_MANTA_NORMAL_111207.4.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M5 data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_1.fastq.gz data/tiny/D0F23_MANTA_NORMAL_111212.1.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M6 data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_1.fastq.gz data/tiny/D0F23_MANTA_NORMAL_111212.2.AGTTGCTT_2.fastq.gz -1234 XX 0 1234N 1234N_M7 data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_1.fastq.gz data/tiny/D0F23_MANTA_NORMAL_111212.3.AGTTGCTT_2.fastq.gz diff --git a/data/tsv/tiny.tsv b/data/tsv/tiny.tsv index 09301edb08..d122b64562 100644 --- a/data/tsv/tiny.tsv +++ b/data/tsv/tiny.tsv @@ -1,19 +1,13 @@ -1234 XX 0 1234N 1234N_1 data/tiny/tiny_normal_L001_R1.fastq.gz data/tiny/tiny_normal_L001_R2.fastq.gz -1234 XX 0 1234N 1234N_2 data/tiny/tiny_normal_L002_R1.fastq.gz data/tiny/tiny_normal_L002_R2.fastq.gz -1234 XX 0 1234N 1234N_4 data/tiny/tiny_normal_L004_R1.fastq.gz data/tiny/tiny_normal_L004_R2.fastq.gz -1234 XX 0 1234N 1234N_7 data/tiny/tiny_normal_L007_R1.fastq.gz data/tiny/tiny_normal_L007_R2.fastq.gz -1234 XX 0 1234N 1234N_8 data/tiny/tiny_normal_L008_R1.fastq.gz data/tiny/tiny_normal_L008_R2.fastq.gz -1234 XX 0 1234N DUMMY_N data/tiny/dummy_normal_reads_1.fastq.gz data/tiny/dummy_normal_reads_1.fastq.gz -1234 XX 1 9876T 9876T_1 data/tiny/tiny_tumor_L001_R1.fastq.gz data/tiny/tiny_tumor_L001_R2.fastq.gz -1234 XX 1 9876T 9876T_2 data/tiny/tiny_tumor_L002_R1.fastq.gz data/tiny/tiny_tumor_L002_R2.fastq.gz -1234 XX 1 9876T 9876T_3 data/tiny/tiny_tumor_L003_R1.fastq.gz data/tiny/tiny_tumor_L003_R2.fastq.gz -1234 XX 1 9876T 9876T_5 data/tiny/tiny_tumor_L005_R1.fastq.gz data/tiny/tiny_tumor_L005_R2.fastq.gz -1234 XX 1 9876T 9876T_6 data/tiny/tiny_tumor_L006_R1.fastq.gz data/tiny/tiny_tumor_L006_R2.fastq.gz -1234 XX 1 9876T 9876T_7 data/tiny/tiny_tumor_L007_R1.fastq.gz data/tiny/tiny_tumor_L007_R2.fastq.gz -1234 XX 1 9876T DUMMY_T data/tiny/dummy_tumour_reads_1.fastq.gz data/tiny/dummy_tumour_reads_1.fastq.gz -1234 XX 1 9877R 9877R_1 data/tiny/tiny_tumor_L001_R1.fastq.gz data/tiny/tiny_tumor_L001_R2.fastq.gz -1234 XX 1 9877R 9877R_2 data/tiny/tiny_tumor_L002_R1.fastq.gz data/tiny/tiny_tumor_L002_R2.fastq.gz -1234 XX 1 9877R 9877R_3 data/tiny/tiny_tumor_L003_R1.fastq.gz data/tiny/tiny_tumor_L003_R2.fastq.gz -1234 XX 1 9877R 9877R_5 data/tiny/tiny_tumor_L005_R1.fastq.gz data/tiny/tiny_tumor_L005_R2.fastq.gz -1234 XX 1 9877R 9877R_6 data/tiny/tiny_tumor_L006_R1.fastq.gz data/tiny/tiny_tumor_L006_R2.fastq.gz -1234 XX 1 9877R 9877R_7 data/tiny/tiny_tumor_L007_R1.fastq.gz data/tiny/tiny_tumor_L007_R2.fastq.gz +1234 XX 0 1234N 1234N_1 data/tiny/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz data/tiny/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_2 data/tiny/tiny/normal/tiny_n_L002_R1_xxx.fastq.gz data/tiny/tiny/normal/tiny_n_L002_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_4 data/tiny/tiny/normal/tiny_n_L004_R1_xxx.fastq.gz data/tiny/tiny/normal/tiny_n_L004_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_7 data/tiny/tiny/normal/tiny_n_L007_R1_xxx.fastq.gz data/tiny/tiny/normal/tiny_n_L007_R2_xxx.fastq.gz +1234 XX 0 1234N 1234N_8 data/tiny/tiny/normal/tiny_n_L008_R1_xxx.fastq.gz data/tiny/tiny/normal/tiny_n_L008_R2_xxx.fastq.gz +1234 XX 0 1234N DUMMY_N data/tiny/dummy/normal/dummy_n_R1_xxx.fastq.gz data/tiny/dummy/normal/dummy_n_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_1 data/tiny/tiny/tumor/tiny_t_L001_R1_xxx.fastq.gz data/tiny/tiny/tumor/tiny_t_L001_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_2 data/tiny/tiny/tumor/tiny_t_L002_R1_xxx.fastq.gz data/tiny/tiny/tumor/tiny_t_L002_R2_xxx.fastq.gz +1234 XX 1 9876T 9876T_3 data/tiny/tiny/tumor/tiny_t_L003_R1_xxx.fastq.gz data/tiny/tiny/tumor/tiny_t_L003_R2_xxx.fastq.gz +1234 XX 1 9876T DUMMY_T data/tiny/dummy/tumor/dummy_t_R1_xxx.fastq.gz data/tiny/dummy/tumor/dummy_t_R2_xxx.fastq.gz +1234 XX 1 9877R 9877R_5 data/tiny/tiny/tumor/tiny_t_L005_R1_xxx.fastq.gz data/tiny/tiny/tumor/tiny_t_L005_R2_xxx.fastq.gz +1234 XX 1 9877R 9877R_6 data/tiny/tiny/tumor/tiny_t_L006_R1_xxx.fastq.gz data/tiny/tiny/tumor/tiny_t_L006_R2_xxx.fastq.gz +1234 XX 1 9877R 9877R_7 data/tiny/tiny/tumor/tiny_t_L007_R1_xxx.fastq.gz data/tiny/tiny/tumor/tiny_t_L007_R2_xxx.fastq.gz diff --git a/doc/CONTAINERS.md b/doc/CONTAINERS.md index ff7943f63c..aacd91d0f2 100644 --- a/doc/CONTAINERS.md +++ b/doc/CONTAINERS.md @@ -58,8 +58,8 @@ A container named after the process is made for each process. If a container can ## multiqc [![multiqc-docker status][multiqc-docker-badge]][multiqc-docker-link] -- Based on `python:2.7-slim` -- Contain **[MultiQC][multiqc-link]** 1.1 +- Based on `ewels/multiqc:v1.3` +- Contain **[MultiQC][multiqc-link]** 1.3 ## mutect1 [![mutect1-docker status][mutect1-docker-badge]][mutect1-docker-link] diff --git a/doc/USAGE.md b/doc/USAGE.md index 316693ec04..64f281ac8e 100644 --- a/doc/USAGE.md +++ b/doc/USAGE.md @@ -26,6 +26,10 @@ Display help Disable QC tools and MultiQC to generate a HTML report. +### --outDir + +Choose an output directory + ### --project `ProjectID` Specify a project number ID on a UPPMAX cluster. (optionnal if not on such a cluster) diff --git a/main.nf b/main.nf index 079f8890b2..16718090c5 100644 --- a/main.nf +++ b/main.nf @@ -62,23 +62,28 @@ kate: syntax groovy; space-indent on; indent-width 2; ================================================================================ */ -version = '1.2.4' +version = '1.2.5' // Check that Nextflow version is up to date enough // try / throw / catch works for NF versions < 0.25 when this was implemented nf_required_version = '0.25.0' try { - if( ! nextflow.version.matches(">= $nf_required_version") ){ + if( ! nextflow.version.matches(">= ${nf_required_version}") ){ throw GroovyException('Nextflow version too old') } } catch (all) { - log.error "============================================================\n" + - " Nextflow version $nf_required_version required! You are running v$workflow.nextflow.version.\n" + + log.error "====================================================\n" + + " Nextflow version ${nf_required_version} required! You are running v${workflow.nextflow.version}.\n" + " Pipeline execution will continue, but things may break.\n" + " Please update Nextflow.\n" + "============================================================" } +if (params.help) exit 0, helpMessage() +if (params.version) exit 0, versionMessage() +if (!isAllowedParams(params)) exit 1, "params unknown, see --help for more information" +if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project " + // Default params: // Such params are overridden by command line or configuration definitions @@ -94,6 +99,10 @@ params.contactMail = '' params.noGVCF = false // Reports are generated params.noReports = false +// BAMQC is used +params.noBAMQC = false +// outDir is current directory +params.outDir = baseDir // No sample is defined params.sample = '' // No sampleDir is defined @@ -104,11 +113,10 @@ params.step = 'mapping' params.test = '' // No tools to be used params.tools = '' - -if (params.help) exit 0, helpMessage() -if (params.version) exit 0, versionMessage() -if (!isAllowedParams(params)) exit 1, "params is unknown, see --help for more information" -if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project " +// Params are defined in config files +params.containerPath = '' +params.repository = '' +params.tag = '' step = params.step.toLowerCase() if (step == 'preprocessing') step = 'mapping' @@ -152,10 +160,10 @@ if (params.sample) tsvPath = params.sample // No need for tsv file for step annotate if (!params.sample && !params.sampleDir) { tsvPaths = [ - 'mapping': "$workflow.projectDir/data/tsv/tiny.tsv", - 'realign': "$workflow.launchDir/$directoryMap.nonRealigned/nonRealigned.tsv", - 'recalibrate': "$workflow.launchDir/$directoryMap.nonRecalibrated/nonRecalibrated.tsv", - 'variantcalling': "$workflow.launchDir/$directoryMap.recalibrated/recalibrated.tsv" + 'mapping': "${workflow.projectDir}/data/tsv/tiny.tsv", + 'realign': "${params.outDir}/${directoryMap.nonRealigned}/nonRealigned.tsv", + 'recalibrate': "${params.outDir}/${directoryMap.nonRecalibrated}/nonRecalibrated.tsv", + 'variantcalling': "${params.outDir}/${directoryMap.recalibrated}/recalibrated.tsv" ] if (params.test || step != 'mapping') tsvPath = tsvPaths[step] } @@ -171,7 +179,7 @@ if (tsvPath) { case 'realign': bamFiles = extractBams(tsvFile); break case 'recalibrate': bamFiles = extractRecal(tsvFile); break case 'variantcalling': bamFiles = extractBams(tsvFile); break - default: exit 1, "Unknown step $step" + default: exit 1, "Unknown step ${step}" } } else if (params.sampleDir) { if (step != 'mapping') exit 1, '--sampleDir does not support steps other than "mapping"' @@ -185,11 +193,8 @@ if (tsvPath) { tsvFile = params.sampleDir // used in the reports } else if (step != 'annotate') exit 1, 'No sample were defined, see --help' -if (step == 'mapping') { - (patientGenders, fastqFiles) = extractGenders(fastqFiles) -} else { - (patientGenders, bamFiles) = extractGenders(bamFiles) -} +if (step == 'mapping') (patientGenders, fastqFiles) = extractGenders(fastqFiles) +else (patientGenders, bamFiles) = extractGenders(bamFiles) /* ================================================================================ @@ -216,7 +221,7 @@ if (verbose) bamFiles = bamFiles.view { process RunFastQC { tag {idPatient + "-" + idRun} - publishDir "$directoryMap.fastQC/$idRun", mode: 'copy' + publishDir "${params.outDir}/${directoryMap.fastQC}/${idRun}", mode: 'copy' input: set idPatient, status, idSample, idRun, file(fastqFile1), file(fastqFile2) from fastqFilesforFastQC @@ -228,7 +233,7 @@ process RunFastQC { script: """ - fastqc -q $fastqFile1 $fastqFile2 + fastqc -q ${fastqFile1} ${fastqFile2} """ } @@ -250,13 +255,13 @@ process MapReads { when: step == 'mapping' script: - readGroup = "@RG\\tID:$idRun\\tPU:$idRun\\tSM:$idSample\\tLB:$idSample\\tPL:illumina" + readGroup = "@RG\\tID:${idRun}\\tPU:${idRun}\\tSM:${idSample}\\tLB:${idSample}\\tPL:illumina" // adjust mismatch penalty for tumor samples extra = status == 1 ? "-B 3 " : "" """ - bwa mem -R \"$readGroup\" ${extra}-t $task.cpus -M \ - $genomeFile $fastqFile1 $fastqFile2 | \ - samtools sort --threads $task.cpus -m 4G - > ${idRun}.bam + bwa mem -R \"${readGroup}\" ${extra}-t ${task.cpus} -M \ + ${genomeFile} ${fastqFile1} ${fastqFile2} | \ + samtools sort --threads ${task.cpus} -m 4G - > ${idRun}.bam """ } @@ -291,7 +296,7 @@ process MergeBams { script: """ - samtools merge --threads $task.cpus ${idSample}.bam $bam + samtools merge --threads ${task.cpus} ${idSample}.bam ${bam} """ } @@ -318,7 +323,7 @@ if (verbose) mergedBam = mergedBam.view { process MarkDuplicates { tag {idPatient + "-" + idSample} - publishDir '.', saveAs: { it == "${bam}.metrics" ? "$directoryMap.markDuplicatesQC/$it" : "$directoryMap.nonRealigned/$it" }, mode: 'copy' + publishDir params.outDir, saveAs: { it == "${bam}.metrics" ? "${directoryMap.markDuplicatesQC}/${it}" : "${directoryMap.nonRealigned}/${it}" }, mode: 'copy' input: set idPatient, status, idSample, file(bam) from mergedBam @@ -347,23 +352,20 @@ process MarkDuplicates { // Creating a TSV file to restart from this step markDuplicatesTSV.map { idPatient, status, idSample, bam, bai -> gender = patientGenders[idPatient] - "$idPatient\t$gender\t$status\t$idSample\t$directoryMap.nonRealigned/$bam\t$directoryMap.nonRealigned/$bai\n" + "${idPatient}\t${gender}\t${status}\t${idSample}\t${params.outDir}/${directoryMap.nonRealigned}/${bam}\t${params.outDir}/${directoryMap.nonRealigned}/${bai}\n" }.collectFile( - name: 'nonRealigned.tsv', sort: true, storeDir: directoryMap.nonRealigned + name: 'nonRealigned.tsv', sort: true, storeDir: "${params.outDir}/${directoryMap.nonRealigned}" ) // Create intervals for realignement using both tumor+normal as input // Group the marked duplicates BAMs for intervals and realign by idPatient // Grouping also by gender, to make a nicer channel duplicatesGrouped = Channel.empty() -if (step == 'mapping') { - duplicatesGrouped = duplicates.groupTuple() -} else if (step == 'realign') { - duplicatesGrouped = bamFiles.map{ - idPatient, status, idSample, bam, bai -> - [idPatient, bam, bai] - }.groupTuple() -} +if (step == 'mapping') duplicatesGrouped = duplicates.groupTuple() +else if (step == 'realign') duplicatesGrouped = bamFiles.map{ + idPatient, status, idSample, bam, bai -> + [idPatient, bam, bai] +}.groupTuple() // The duplicatesGrouped channel is duplicated // one copy goes to the RealignerTargetCreator process @@ -386,7 +388,7 @@ if (verbose) duplicatesRealign = duplicatesRealign.view { if (verbose) markDuplicatesReport = markDuplicatesReport.view { "MarkDuplicates report:\n\ - File : [$it.fileName]" + File : [${it.fileName}]" } // VCF indexes are added so they will be linked, and not re-created on the fly @@ -412,17 +414,17 @@ process RealignerTargetCreator { when: step == 'mapping' || step == 'realign' script: - bams = bam.collect{"-I $it"}.join(' ') - known = knownIndels.collect{"-known $it"}.join(' ') + bams = bam.collect{"-I ${it}"}.join(' ') + known = knownIndels.collect{"-known ${it}"}.join(' ') """ java -Xmx${task.memory.toGiga()}g \ -jar \$GATK_HOME/GenomeAnalysisTK.jar \ -T RealignerTargetCreator \ - $bams \ - -R $genomeFile \ - $known \ - -nt $task.cpus \ - -L $intervals \ + ${bams} \ + -R ${genomeFile} \ + ${known} \ + -nt ${task.cpus} \ + -L ${intervals} \ -o ${idPatient}.intervals """ } @@ -470,16 +472,16 @@ process IndelRealigner { when: step == 'mapping' || step == 'realign' script: - bams = bam.collect{"-I $it"}.join(' ') - known = knownIndels.collect{"-known $it"}.join(' ') + bams = bam.collect{"-I ${it}"}.join(' ') + known = knownIndels.collect{"-known ${it}"}.join(' ') """ java -Xmx${task.memory.toGiga()}g \ -jar \$GATK_HOME/GenomeAnalysisTK.jar \ -T IndelRealigner \ - $bams \ - -R $genomeFile \ - -targetIntervals $intervals \ - $known \ + ${bams} \ + -R ${genomeFile} \ + -targetIntervals ${intervals} \ + ${known} \ -nWayOut '.real.bam' """ } @@ -501,7 +503,7 @@ if (verbose) realignedBam = realignedBam.view { process CreateRecalibrationTable { tag {idPatient + "-" + idSample} - publishDir directoryMap.nonRecalibrated, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.nonRecalibrated}", mode: 'copy' input: set idPatient, status, idSample, file(bam), file(bai) from realignedBam @@ -523,19 +525,19 @@ process CreateRecalibrationTable { when: step == 'mapping' || step == 'realign' script: - known = knownIndels.collect{ "-knownSites $it" }.join(' ') + known = knownIndels.collect{ "-knownSites ${it}" }.join(' ') """ java -Xmx${task.memory.toGiga()}g \ -Djava.io.tmpdir="/tmp" \ -jar \$GATK_HOME/GenomeAnalysisTK.jar \ -T BaseRecalibrator \ - -R $genomeFile \ - -I $bam \ - -L $intervals \ + -R ${genomeFile} \ + -I ${bam} \ + -L ${intervals} \ --disable_auto_index_creation_and_locking_when_reading_rods \ - -knownSites $dbsnp \ - $known \ - -nct $task.cpus \ + -knownSites ${dbsnp} \ + ${known} \ + -nct ${task.cpus} \ -l INFO \ -o ${idSample}.recal.table """ @@ -543,9 +545,9 @@ process CreateRecalibrationTable { // Create a TSV file to restart from this step recalibrationTableTSV.map { idPatient, status, idSample, bam, bai, recalTable -> gender = patientGenders[idPatient] - "$idPatient\t$gender\t$status\t$idSample\t$directoryMap.nonRecalibrated/$bam\t$directoryMap.nonRecalibrated/$bai\t\t$directoryMap.nonRecalibrated/$recalTable\n" + "${idPatient}\t${gender}\t${status}\t${idSample}\t${params.outDir}/${directoryMap.nonRecalibrated}/${bam}\t${params.outDir}/${directoryMap.nonRecalibrated}/${bai}\t${params.outDir}/${directoryMap.nonRecalibrated}/${recalTable}\n" }.collectFile( - name: 'nonRecalibrated.tsv', sort: true, storeDir: directoryMap.nonRecalibrated + name: 'nonRecalibrated.tsv', sort: true, storeDir: "${params.outDir}/${directoryMap.nonRecalibrated}" ) if (step == 'recalibrate') recalibrationTable = bamFiles @@ -568,10 +570,10 @@ recalTables = recalTables.map { [it[0]] + it[2..-1] } // remove status process RecalibrateBam { tag {idPatient + "-" + idSample} - publishDir directoryMap.recalibrated, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.recalibrated}", mode: 'copy' input: - set idPatient, status, idSample, file(bam), file(bai), recalibrationReport from recalibrationTable + set idPatient, status, idSample, file(bam), file(bai), file(recalibrationReport) from recalibrationTable set file(genomeFile), file(genomeIndex), file(genomeDict), file(intervals) from Channel.value([ referenceMap.genomeFile, referenceMap.genomeIndex, @@ -592,19 +594,19 @@ process RecalibrateBam { java -Xmx${task.memory.toGiga()}g \ -jar \$GATK_HOME/GenomeAnalysisTK.jar \ -T PrintReads \ - -R $genomeFile \ - -I $bam \ - -L $intervals \ - --BQSR $recalibrationReport \ + -R ${genomeFile} \ + -I ${bam} \ + -L ${intervals} \ + --BQSR ${recalibrationReport} \ -o ${idSample}.recal.bam """ } // Creating a TSV file to restart from this step recalibratedBamTSV.map { idPatient, status, idSample, bam, bai -> gender = patientGenders[idPatient] - "$idPatient\t$gender\t$status\t$idSample\t$directoryMap.recalibrated/$bam\t$directoryMap.recalibrated/$bai\n" + "${idPatient}\t${gender}\t${status}\t${idSample}\t${params.outDir}/${directoryMap.recalibrated}/${bam}\t${params.outDir}/${directoryMap.recalibrated}/${bai}\n" }.collectFile( - name: 'recalibrated.tsv', sort: true, storeDir: directoryMap.recalibrated + name: 'recalibrated.tsv', sort: true, storeDir: "${params.outDir}/${directoryMap.recalibrated}" ) if (step == 'variantcalling') { @@ -615,9 +617,7 @@ if (step == 'variantcalling') { (recalTables, recalibrationTableForHC) = recalTables.into(2) recalTables = recalTables.map { [it[0]] + it[2..-1] } // remove status -} else if (!explicitBqsrNeeded) { - (bamForBamQC, bamForSamToolsStats, recalibratedBam) = recalibrationTableForHC.map { it[0..-2] }.into(3) -} +} else if (!explicitBqsrNeeded) (bamForBamQC, bamForSamToolsStats, recalibratedBam) = recalibrationTableForHC.map { it[0..-2] }.into(3) if (verbose) recalibratedBam = recalibratedBam.view { "Recalibrated BAM for variant Calling:\n\ @@ -628,7 +628,7 @@ if (verbose) recalibratedBam = recalibratedBam.view { process RunSamtoolsStats { tag {idPatient + "-" + idSample} - publishDir directoryMap.samtoolsStats, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.samtoolsStats}", mode: 'copy' input: set idPatient, status, idSample, file(bam), file(bai) from bamForSamToolsStats @@ -636,12 +636,12 @@ process RunSamtoolsStats { output: file ("${bam}.samtools.stats.out") into samtoolsStatsReport - when: reports + when: reports - script: - """ - samtools stats $bam > ${bam}.samtools.stats.out - """ + script: + """ + samtools stats ${bam} > ${bam}.samtools.stats.out + """ } if (verbose) samtoolsStatsReport = samtoolsStatsReport.view { @@ -652,24 +652,24 @@ if (verbose) samtoolsStatsReport = samtoolsStatsReport.view { process RunBamQC { tag {idPatient + "-" + idSample} - publishDir directoryMap.bamQC, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.bamQC}", mode: 'copy' input: set idPatient, status, idSample, file(bam), file(bai) from bamForBamQC output: - file("$idSample") into bamQCreport + file("${idSample}") into bamQCreport - when: reports + when: reports && !params.noBAMQC - script: - """ - qualimap --java-mem-size=${task.memory.toGiga()}G \ - bamqc \ - -bam $bam \ - -outdir $idSample \ - -outformat HTML - """ + script: + """ + qualimap --java-mem-size=${task.memory.toGiga()}G \ + bamqc \ + -bam ${bam} \ + -outdir ${idSample} \ + -outformat HTML + """ } if (verbose) bamQCreport = bamQCreport.view { @@ -744,14 +744,14 @@ process CreateIntervalBeds { longest = t chunk += t print \$0 > name - }' $intervals + }' ${intervals} """ else """ awk -vFS="[:-]" '{ name = sprintf("%s_%d-%d", \$1, \$2, \$3); printf("%s\\t%d\\t%d\\n", \$1, \$2-1, \$3) > name ".bed" - }' $intervals + }' ${intervals} """ } @@ -760,9 +760,8 @@ bedIntervals = bedIntervals final duration = 0.0 for (line in intervalFile.readLines()) { final fields = line.split('\t') - if (fields.size() >= 5) { - duration += fields[4].toFloat() - } else { + if (fields.size() >= 5) duration += fields[4].toFloat() + else { start = fields[1].toInteger() end = fields[2].toInteger() duration += (end - start) / nucleotidesPerSecond @@ -839,18 +838,18 @@ process RunHaplotypecaller { -T HaplotypeCaller \ --emitRefConfidence GVCF \ -pairHMM LOGLESS_CACHING \ - -R $genomeFile \ - --dbsnp $dbsnp \ - $BQSR \ - -I $bam \ - -L $intervalBed \ + -R ${genomeFile} \ + --dbsnp ${dbsnp} \ + ${BQSR} \ + -I ${bam} \ + -L ${intervalBed} \ --disable_auto_index_creation_and_locking_when_reading_rods \ -o ${intervalBed.baseName}_${idSample}.g.vcf """ } hcGenomicVCF = hcGenomicVCF.groupTuple(by:[0,1,2,3]) -if (!gvcf) {hcGenomicVCF.close()} +if (!gvcf) hcGenomicVCF.close() process RunGenotypeGVCFs { tag {idSample + "-" + intervalBed.baseName} @@ -876,10 +875,10 @@ process RunGenotypeGVCFs { java -Xmx${task.memory.toGiga()}g \ -jar \$GATK_HOME/GenomeAnalysisTK.jar \ -T GenotypeGVCFs \ - -R $genomeFile \ - -L $intervalBed \ - --dbsnp $dbsnp \ - --variant $gvcf \ + -R ${genomeFile} \ + -L ${intervalBed} \ + --dbsnp ${dbsnp} \ + --variant ${gvcf} \ --disable_auto_index_creation_and_locking_when_reading_rods \ -o ${intervalBed.baseName}_${idSample}.vcf """ @@ -911,12 +910,12 @@ process RunMutect1 { java -Xmx${task.memory.toGiga()}g \ -jar \$MUTECT_HOME/muTect.jar \ -T MuTect \ - -R $genomeFile \ - --cosmic $cosmic \ - --dbsnp $dbsnp \ - -I:normal $bamNormal \ - -I:tumor $bamTumor \ - -L $intervalBed \ + -R ${genomeFile} \ + --cosmic ${cosmic} \ + --dbsnp ${dbsnp} \ + -I:normal ${bamNormal} \ + -I:tumor ${bamTumor} \ + -L ${intervalBed} \ --disable_auto_index_creation_and_locking_when_reading_rods \ --out ${intervalBed.baseName}_${idSampleTumor}_vs_${idSampleNormal}.call_stats.out \ --vcf ${intervalBed.baseName}_${idSampleTumor}_vs_${idSampleNormal}.vcf @@ -950,13 +949,13 @@ process RunMutect2 { java -Xmx${task.memory.toGiga()}g \ -jar \$GATK_HOME/GenomeAnalysisTK.jar \ -T MuTect2 \ - -R $genomeFile \ - --cosmic $cosmic \ - --dbsnp $dbsnp \ - -I:normal $bamNormal \ - -I:tumor $bamTumor \ + -R ${genomeFile} \ + --cosmic ${cosmic} \ + --dbsnp ${dbsnp} \ + -I:normal ${bamNormal} \ + -I:tumor ${bamTumor} \ --disable_auto_index_creation_and_locking_when_reading_rods \ - -L $intervalBed \ + -L ${intervalBed} \ -o ${intervalBed.baseName}_${idSampleTumor}_vs_${idSampleNormal}.vcf """ } @@ -978,7 +977,7 @@ process RunFreeBayes { script: """ freebayes \ - -f $genomeFile \ + -f ${genomeFile} \ --pooled-continuous \ --pooled-discrete \ --genotype-qualities \ @@ -987,9 +986,9 @@ process RunFreeBayes { --min-alternate-fraction 0.03 \ --min-repeat-entropy 1 \ --min-alternate-count 2 \ - -t $intervalBed \ - $bamTumor \ - $bamNormal > ${intervalBed.baseName}_${idSampleTumor}_vs_${idSampleNormal}.vcf + -t ${intervalBed} \ + ${bamTumor} \ + ${bamNormal} > ${intervalBed.baseName}_${idSampleTumor}_vs_${idSampleNormal}.vcf """ } @@ -1008,7 +1007,7 @@ if (verbose) vcfsToMerge = vcfsToMerge.view { process ConcatVCF { tag {variantCaller in ['gvcf-hc', 'haplotypecaller'] ? variantCaller + "-" + idSampleNormal : variantCaller + "_" + idSampleTumor + "_vs_" + idSampleNormal} - publishDir "${directoryMap."$variantCaller"}", mode: 'copy' + publishDir "${params.outDir}/${directoryMap."$variantCaller"}", mode: 'copy' input: set variantCaller, idPatient, idSampleNormal, idSampleTumor, file(vcFiles) from vcfsToMerge @@ -1021,14 +1020,9 @@ process ConcatVCF { when: 'haplotypecaller' in tools || 'mutect1' in tools || 'mutect2' in tools || 'freebayes' in tools script: - if (variantCaller == 'haplotypecaller') { - outputFile = "${variantCaller}_${idSampleNormal}.vcf" - } else if (variantCaller == 'gvcf-hc') { - outputFile = "haplotypecaller_${idSampleNormal}.g.vcf" - } else { - outputFile = "${variantCaller}_${idSampleTumor}_vs_${idSampleNormal}.vcf" - } - vcfFiles = vcFiles.collect{" $it"}.join(' ') + if (variantCaller == 'haplotypecaller') outputFile = "${variantCaller}_${idSampleNormal}.vcf" + else if (variantCaller == 'gvcf-hc') outputFile = "haplotypecaller_${idSampleNormal}.g.vcf" + else outputFile = "${variantCaller}_${idSampleTumor}_vs_${idSampleNormal}.vcf" """ # first make a header from one of the VCF intervals @@ -1081,7 +1075,7 @@ if (verbose) vcfConcatenated = vcfConcatenated.view { process RunStrelka { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.strelka, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.strelka}", mode: 'copy' input: set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor) from bamsForStrelka @@ -1099,17 +1093,21 @@ process RunStrelka { script: """ \$STRELKA_INSTALL_PATH/bin/configureStrelkaSomaticWorkflow.py \ - --tumor $bamTumor \ - --normal $bamNormal \ - --referenceFasta $genomeFile \ + --tumor ${bamTumor} \ + --normal ${bamNormal} \ + --referenceFasta ${genomeFile} \ --runDir Strelka - python Strelka/runWorkflow.py -m local -j $task.cpus + python Strelka/runWorkflow.py -m local -j ${task.cpus} - mv Strelka/results/variants/somatic.indels.vcf.gz Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_indels.vcf.gz - mv Strelka/results/variants/somatic.indels.vcf.gz.tbi Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_indels.vcf.gz.tbi - mv Strelka/results/variants/somatic.snvs.vcf.gz Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_snvs.vcf.gz - mv Strelka/results/variants/somatic.snvs.vcf.gz.tbi Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_snvs.vcf.gz.tbi + mv Strelka/results/variants/somatic.indels.vcf.gz \ + Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_indels.vcf.gz + mv Strelka/results/variants/somatic.indels.vcf.gz.tbi \ + Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_indels.vcf.gz.tbi + mv Strelka/results/variants/somatic.snvs.vcf.gz \ + Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_snvs.vcf.gz + mv Strelka/results/variants/somatic.snvs.vcf.gz.tbi \ + Strelka_${idSampleTumor}_vs_${idSampleNormal}_somatic_snvs.vcf.gz.tbi """ } @@ -1123,7 +1121,7 @@ if (verbose) strelkaOutput = strelkaOutput.view { process RunSingleStrelka { tag {idSample} - publishDir directoryMap.strelka, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.strelka}", mode: 'copy' input: set idPatient, status, idSample, file(bam), file(bai) from bamsForSingleStrelka @@ -1140,16 +1138,20 @@ process RunSingleStrelka { script: """ \$STRELKA_INSTALL_PATH/bin/configureStrelkaGermlineWorkflow.py \ - --bam $bam \ - --referenceFasta $genomeFile \ + --bam ${bam} \ + --referenceFasta ${genomeFile} \ --runDir Strelka - python Strelka/runWorkflow.py -m local -j $task.cpus + python Strelka/runWorkflow.py -m local -j ${task.cpus} - mv Strelka/results/variants/genome.*.vcf.gz Strelka_${idSample}_genome.vcf.gz - mv Strelka/results/variants/genome.*.vcf.gz.tbi Strelka_${idSample}_genome.vcf.gz.tbi - mv Strelka/results/variants/variants.vcf.gz Strelka_${idSample}_variants.vcf.gz - mv Strelka/results/variants/variants.vcf.gz.tbi Strelka_${idSample}_variants.vcf.gz.tbi + mv Strelka/results/variants/genome.*.vcf.gz \ + Strelka_${idSample}_genome.vcf.gz + mv Strelka/results/variants/genome.*.vcf.gz.tbi \ + Strelka_${idSample}_genome.vcf.gz.tbi + mv Strelka/results/variants/variants.vcf.gz \ + Strelka_${idSample}_variants.vcf.gz + mv Strelka/results/variants/variants.vcf.gz.tbi \ + Strelka_${idSample}_variants.vcf.gz.tbi """ } @@ -1163,7 +1165,7 @@ if (verbose) singleStrelkaOutput = singleStrelkaOutput.view { process RunManta { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.manta, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.manta}", mode: 'copy' input: set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor) from bamsForManta @@ -1180,21 +1182,29 @@ process RunManta { script: """ \$MANTA_INSTALL_PATH/bin/configManta.py \ - --normalBam $bamNormal \ - --tumorBam $bamTumor \ - --reference $genomeFile \ + --normalBam ${bamNormal} \ + --tumorBam ${bamTumor} \ + --reference ${genomeFile} \ --runDir Manta - python Manta/runWorkflow.py -m local -j $task.cpus - - mv Manta/results/variants/candidateSmallIndels.vcf.gz Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSmallIndels.vcf.gz - mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSmallIndels.vcf.gz.tbi - mv Manta/results/variants/candidateSV.vcf.gz Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSV.vcf.gz - mv Manta/results/variants/candidateSV.vcf.gz.tbi Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSV.vcf.gz.tbi - mv Manta/results/variants/diploidSV.vcf.gz Manta_${idSampleTumor}_vs_${idSampleNormal}.diploidSV.vcf.gz - mv Manta/results/variants/diploidSV.vcf.gz.tbi Manta_${idSampleTumor}_vs_${idSampleNormal}.diploidSV.vcf.gz.tbi - mv Manta/results/variants/somaticSV.vcf.gz Manta_${idSampleTumor}_vs_${idSampleNormal}.somaticSV.vcf.gz - mv Manta/results/variants/somaticSV.vcf.gz.tbi Manta_${idSampleTumor}_vs_${idSampleNormal}.somaticSV.vcf.gz.tbi + python Manta/runWorkflow.py -m local -j ${task.cpus} + + mv Manta/results/variants/candidateSmallIndels.vcf.gz \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSmallIndels.vcf.gz + mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSmallIndels.vcf.gz.tbi + mv Manta/results/variants/candidateSV.vcf.gz \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSV.vcf.gz + mv Manta/results/variants/candidateSV.vcf.gz.tbi \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.candidateSV.vcf.gz.tbi + mv Manta/results/variants/diploidSV.vcf.gz \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.diploidSV.vcf.gz + mv Manta/results/variants/diploidSV.vcf.gz.tbi \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.diploidSV.vcf.gz.tbi + mv Manta/results/variants/somaticSV.vcf.gz \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.somaticSV.vcf.gz + mv Manta/results/variants/somaticSV.vcf.gz.tbi \ + Manta_${idSampleTumor}_vs_${idSampleNormal}.somaticSV.vcf.gz.tbi """ } @@ -1208,7 +1218,7 @@ if (verbose) mantaOutput = mantaOutput.view { process RunSingleManta { tag {status == 0 ? idSample + " - Single Diploid" : idSample + " - Tumor-Only"} - publishDir directoryMap.manta, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.manta}", mode: 'copy' input: set idPatient, status, idSample, file(bam), file(bai) from bamsForSingleManta @@ -1226,34 +1236,46 @@ process RunSingleManta { if ( status == 0 ) // If Normal Sample """ \$MANTA_INSTALL_PATH/bin/configManta.py \ - --bam $bam \ - --reference $genomeFile \ + --bam ${bam} \ + --reference ${genomeFile} \ --runDir Manta - python Manta/runWorkflow.py -m local -j $task.cpus - - mv Manta/results/variants/candidateSmallIndels.vcf.gz Manta_${idSample}.candidateSmallIndels.vcf.gz - mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi Manta_${idSample}.candidateSmallIndels.vcf.gz.tbi - mv Manta/results/variants/candidateSV.vcf.gz Manta_${idSample}.candidateSV.vcf.gz - mv Manta/results/variants/candidateSV.vcf.gz.tbi Manta_${idSample}.candidateSV.vcf.gz.tbi - mv Manta/results/variants/diploidSV.vcf.gz Manta_${idSample}.diploidSV.vcf.gz - mv Manta/results/variants/diploidSV.vcf.gz.tbi Manta_${idSample}.diploidSV.vcf.gz.tbi + python Manta/runWorkflow.py -m local -j ${task.cpus} + + mv Manta/results/variants/candidateSmallIndels.vcf.gz \ + Manta_${idSample}.candidateSmallIndels.vcf.gz + mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi \ + Manta_${idSample}.candidateSmallIndels.vcf.gz.tbi + mv Manta/results/variants/candidateSV.vcf.gz \ + Manta_${idSample}.candidateSV.vcf.gz + mv Manta/results/variants/candidateSV.vcf.gz.tbi \ + Manta_${idSample}.candidateSV.vcf.gz.tbi + mv Manta/results/variants/diploidSV.vcf.gz \ + Manta_${idSample}.diploidSV.vcf.gz + mv Manta/results/variants/diploidSV.vcf.gz.tbi \ + Manta_${idSample}.diploidSV.vcf.gz.tbi """ else // Tumor Sample """ \$MANTA_INSTALL_PATH/bin/configManta.py \ - --tumorBam $bam \ - --reference $genomeFile \ + --tumorBam ${bam} \ + --reference ${genomeFile} \ --runDir Manta - python Manta/runWorkflow.py -m local -j $task.cpus - - mv Manta/results/variants/candidateSmallIndels.vcf.gz Manta_${idSample}.candidateSmallIndels.vcf.gz - mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi Manta_${idSample}.candidateSmallIndels.vcf.gz.tbi - mv Manta/results/variants/candidateSV.vcf.gz Manta_${idSample}.candidateSV.vcf.gz - mv Manta/results/variants/candidateSV.vcf.gz.tbi Manta_${idSample}.candidateSV.vcf.gz.tbi - mv Manta/results/variants/tumorSV.vcf.gz Manta_${idSample}.tumorSV.vcf.gz - mv Manta/results/variants/tumorSV.vcf.gz.tbi Manta_${idSample}.tumorSV.vcf.gz.tbi + python Manta/runWorkflow.py -m local -j ${task.cpus} + + mv Manta/results/variants/candidateSmallIndels.vcf.gz \ + Manta_${idSample}.candidateSmallIndels.vcf.gz + mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi \ + Manta_${idSample}.candidateSmallIndels.vcf.gz.tbi + mv Manta/results/variants/candidateSV.vcf.gz \ + Manta_${idSample}.candidateSV.vcf.gz + mv Manta/results/variants/candidateSV.vcf.gz.tbi \ + Manta_${idSample}.candidateSV.vcf.gz.tbi + mv Manta/results/variants/tumorSV.vcf.gz \ + Manta_${idSample}.tumorSV.vcf.gz + mv Manta/results/variants/tumorSV.vcf.gz.tbi \ + Manta_${idSample}.tumorSV.vcf.gz.tbi """ } @@ -1285,7 +1307,11 @@ process RunAlleleCount { script: """ - alleleCounter -l $acLoci -r $genomeFile -b $bam -o ${idSample}.alleleCount; + alleleCounter \ + -l ${acLoci} \ + -r ${genomeFile} \ + -b ${bam} \ + -o ${idSample}.alleleCount; """ } @@ -1308,7 +1334,7 @@ alleleCountOutput = alleleCountOutput.map { process RunConvertAlleleCounts { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.ascat, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.ascat}", mode: 'copy' input: set idPatient, idSampleNormal, idSampleTumor, file(alleleCountNormal), file(alleleCountTumor) from alleleCountOutput @@ -1321,7 +1347,7 @@ process RunConvertAlleleCounts { script: gender = patientGenders[idPatient] """ - convertAlleleCounts.r $idSampleTumor $alleleCountTumor $idSampleNormal $alleleCountNormal $gender + convertAlleleCounts.r ${idSampleTumor} ${alleleCountTumor} ${idSampleNormal} ${alleleCountNormal} ${gender} """ } @@ -1330,7 +1356,7 @@ process RunConvertAlleleCounts { process RunAscat { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.ascat, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.ascat}", mode: 'copy' input: set idPatient, idSampleNormal, idSampleTumor, file(bafNormal), file(logrNormal), file(bafTumor), file(logrTumor) from convertAlleleCountsOutput @@ -1344,7 +1370,7 @@ process RunAscat { """ # get rid of "chr" string if there is any for f in *BAF *LogR; do sed 's/chr//g' \$f > tmpFile; mv tmpFile \$f;done - run_ascat.r $bafTumor $logrTumor $bafNormal $logrNormal $idSampleTumor $baseDir + run_ascat.r ${bafTumor} ${logrTumor} ${bafNormal} ${logrNormal} ${idSampleTumor} ${baseDir} """ } @@ -1359,23 +1385,23 @@ vcfNotToAnnotate = Channel.create() if (step == 'annotate' && annotateVCF == []) { Channel.empty().mix( - Channel.fromPath('VariantCalling/HaplotypeCaller/*.vcf.gz') + Channel.fromPath("${params.outDir}/VariantCalling/HaplotypeCaller/*.vcf.gz") .flatten().map{vcf -> ['haplotypecaller',vcf]}, - Channel.fromPath('VariantCalling/Manta/*SV.vcf.gz') + Channel.fromPath("${params.outDir}/VariantCalling/Manta/*SV.vcf.gz") .flatten().map{vcf -> ['manta',vcf]}, - Channel.fromPath('VariantCalling/MuTect1/*.vcf.gz') + Channel.fromPath("${params.outDir}/VariantCalling/MuTect1/*.vcf.gz") .flatten().map{vcf -> ['mutect1',vcf]}, - Channel.fromPath('VariantCalling/MuTect2/*.vcf.gz') + Channel.fromPath("${params.outDir}/VariantCalling/MuTect2/*.vcf.gz") .flatten().map{vcf -> ['mutect2',vcf]}, - Channel.fromPath('VariantCalling/Strelka/*{somatic,variants}*.vcf.gz') + Channel.fromPath("${params.outDir}/VariantCalling/Strelka/*{somatic,variants}*.vcf.gz") .flatten().map{vcf -> ['strelka',vcf]} ).choice(vcfToAnnotate, vcfNotToAnnotate) { annotateTools == [] || (annotateTools != [] && it[0] in annotateTools) ? 0 : 1 } } else if (step == 'annotate' && annotateTools == [] && annotateVCF != []) { list = "" - annotateVCF.each{ list += ",$it" } + annotateVCF.each{ list += ",${it}" } list = list.substring(1) - if (StringUtils.countMatches("$list", ",") == 0) vcfToAnnotate = Channel.fromPath("$list") + if (StringUtils.countMatches("${list}", ",") == 0) vcfToAnnotate = Channel.fromPath("${list}") .map{vcf -> ['userspecified',vcf]} else vcfToAnnotate = Channel.fromPath("{$list}") .map{vcf -> ['userspecified',vcf]} @@ -1423,7 +1449,7 @@ vcfNotToAnnotate.close() process RunBcftoolsStats { tag {vcf} - publishDir directoryMap.bcftoolsStats, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.bcftoolsStats}", mode: 'copy' input: set variantCaller, file(vcf) from vcfForBCFtools @@ -1435,7 +1461,7 @@ process RunBcftoolsStats { script: """ - bcftools stats $vcf > ${vcf.baseName}.bcf.tools.stats.out + bcftools stats ${vcf} > ${vcf.baseName}.bcf.tools.stats.out """ } @@ -1447,7 +1473,7 @@ if (verbose) bcfReport = bcfReport.view { process RunSnpeff { tag {vcf} - publishDir directoryMap.snpeff, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.snpeff}", mode: 'copy' input: set variantCaller, file(vcf) from vcfForSnpeff @@ -1462,7 +1488,7 @@ process RunSnpeff { """ java -Xmx${task.memory.toGiga()}g \ -jar \$SNPEFF_HOME/snpEff.jar \ - $snpeffDb \ + ${snpeffDb} \ -csvStats ${vcf.baseName}.snpEff.csv \ -nodownload \ -cancer \ @@ -1482,7 +1508,7 @@ if (verbose) snpeffReport = snpeffReport.view { process RunVEP { tag {vcf} - publishDir directoryMap.vep, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.vep}", mode: 'copy' input: set variantCaller, file(vcf) from vcfForVep @@ -1496,7 +1522,7 @@ process RunVEP { genome = params.genome == 'smallGRCh37' ? 'GRCh37' : params.genome """ vep \ - -i $vcf \ + -i ${vcf} \ -o ${vcf.baseName}.vep.ann.vcf \ --stats_file ${vcf.baseName}.vep.summary.html \ --cache \ @@ -1505,6 +1531,7 @@ process RunVEP { --format vcf \ --offline \ --pick \ + --fork ${task.cpus} \ --total_length \ --vcf """ @@ -1516,7 +1543,7 @@ if (verbose) vepReport = vepReport.view { } process GenerateMultiQCconfig { - publishDir directoryMap.multiQC, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.multiQC}", mode: 'copy' input: @@ -1531,11 +1558,11 @@ process GenerateMultiQCconfig { tsvString = step != 'annotate' ? "- TSV file: ${tsvFile}" : '' """ touch multiqc_config.yaml - echo "custom_logo: $baseDir/doc/images/CAW_logo.png" >> multiqc_config.yaml + echo "custom_logo: ${baseDir}/doc/images/CAW_logo.png" >> multiqc_config.yaml echo "custom_logo_url: http://opensource.scilifelab.se/projects/caw" >> multiqc_config.yaml echo "custom_logo_title: 'Cancer Analysis Workflow'" >> multiqc_config.yaml echo "report_header_info:" >> multiqc_config.yaml - echo "- CAW version: $version" >> multiqc_config.yaml + echo "- CAW version: ${version}" >> multiqc_config.yaml echo "- Contact Name: ${params.callName}" >> multiqc_config.yaml echo "- Contact E-mail: ${params.contactMail}" >> multiqc_config.yaml echo "- Command Line: ${workflow.commandLine}" >> multiqc_config.yaml @@ -1546,17 +1573,17 @@ process GenerateMultiQCconfig { echo "- Tools: "${tools.join(", ")} >> multiqc_config.yaml echo ${annotateToolString} >> multiqc_config.yaml echo ${annotateVCFstring} >> multiqc_config.yaml - echo " acLoci : $referenceMap.acLoci" >> multiqc_config.yaml + echo " acLoci : ${referenceMap.acLoci}" >> multiqc_config.yaml echo " bwaIndex : "${referenceMap.bwaIndex.join(", ")} >> multiqc_config.yaml - echo " cosmic : $referenceMap.cosmic" >> multiqc_config.yaml - echo " cosmicIndex : $referenceMap.cosmicIndex" >> multiqc_config.yaml - echo " dbsnp : $referenceMap.dbsnp" >> multiqc_config.yaml - echo " dbsnpIndex : $referenceMap.dbsnpIndex" >> multiqc_config.yaml - echo " genomeDict : $referenceMap.genomeDict" >> multiqc_config.yaml - echo " genomeFile : $referenceMap.genomeFile" >> multiqc_config.yaml - echo " genomeIndex : $referenceMap.genomeIndex" >> multiqc_config.yaml - echo " intervals : $referenceMap.intervals" >> multiqc_config.yaml - echo " knownIndels : "${referenceMap.knownIndels.join(", ")} >> multiqc_config.yaml + echo " cosmic : ${referenceMap.cosmic}" >> multiqc_config.yaml + echo " cosmicIndex : ${referenceMap.cosmicIndex}" >> multiqc_config.yaml + echo " dbsnp : ${referenceMap.dbsnp}" >> multiqc_config.yaml + echo " dbsnpIndex : ${referenceMap.dbsnpIndex}" >> multiqc_config.yaml + echo " genomeDict : ${referenceMap.genomeDict}" >> multiqc_config.yaml + echo " genomeFile : ${referenceMap.genomeFile}" >> multiqc_config.yaml + echo " genomeIndex : ${referenceMap.genomeIndex}" >> multiqc_config.yaml + echo " intervals : ${referenceMap.intervals}" >> multiqc_config.yaml + echo " knownIndels : " ${referenceMap.knownIndels.join(", ")} >> multiqc_config.yaml echo " knownIndelsIndex: "${referenceMap.knownIndelsIndex.join(", ")} >> multiqc_config.yaml echo " snpeffDb : ${params.genomes[params.genome].snpeffDb}" >> multiqc_config.yaml echo "top_modules:" >> multiqc_config.yaml @@ -1576,8 +1603,6 @@ if (verbose && reports) multiQCconfig = multiQCconfig.view { reportsForMultiQC = Channel.empty() .mix( - Channel.fromPath('Reports/{BCFToolsStats,MarkDuplicates,SamToolsStats}/*'), - Channel.fromPath('Reports/{bamQC,FastQC}/*/*'), bamQCreport, bcfReport, fastQCreport, @@ -1589,7 +1614,7 @@ reportsForMultiQC = Channel.empty() ).collect() process RunMultiQC { - publishDir directoryMap.multiQC, mode: 'copy' + publishDir "${params.outDir}/${directoryMap.multiQC}", mode: 'copy' input: file ('*') from reportsForMultiQC @@ -1597,7 +1622,7 @@ process RunMultiQC { output: set file("*multiqc_report.html"), file("*multiqc_data") into multiQCReport - when: reports + when: reports script: """ @@ -1619,20 +1644,18 @@ if (verbose) multiQCReport = multiQCReport.view { def cawMessage() { // Display CAW message - log.info "CANCER ANALYSIS WORKFLOW ~ $version - " + this.grabRevision() + (workflow.commitId ? " [$workflow.commitId]" : "") + log.info "CANCER ANALYSIS WORKFLOW ~ ${version} - " + this.grabRevision() + (workflow.commitId ? " [${workflow.commitId}]" : "") } def checkFileExtension(it, extension) { // Check file extension - if (!it.toString().toLowerCase().endsWith(extension.toLowerCase())) { - exit 1, "File: $it has the wrong extension: $extension see --help for more information" - } + if (!it.toString().toLowerCase().endsWith(extension.toLowerCase())) exit 1, "File: ${it} has the wrong extension: ${extension} see --help for more information" } def checkParameterExistence(it, list) { // Check parameter existence if (!list.contains(it)) { - println("Unknown parameter: $it") + println("Unknown parameter: ${it}") return false } return true @@ -1644,8 +1667,8 @@ def checkParameterList(list, realList) { } def checkParamReturnFile(item) { - params."$item" = params.genomes[params.genome]."$item" - return file(params."$item") + params."${item}" = params.genomes[params.genome]."${item}" + return file(params."${item}") } def checkParams(it) { @@ -1673,6 +1696,7 @@ def checkParams(it) { 'dbsnp-index', 'dbsnp', 'docker', + 'genome_base', 'genome-dict', 'genome-file', 'genome-index', @@ -1687,10 +1711,15 @@ def checkParams(it) { 'known-indels', 'knownIndels', 'knownIndelsIndex', + 'no-BAMQC', 'no-GVCF', 'no-reports', + 'noBAMQC', 'noGVCF', 'noReports', + 'out-dir', + 'outDir', + 'params', 'project', 'push', 'repository', @@ -1722,15 +1751,12 @@ def checkReferenceMap(referenceMap) { } def checkRefExistence(referenceFile, fileToCheck) { - if (fileToCheck instanceof List) { - return fileToCheck.every{ checkRefExistence(referenceFile, it) } - } + if (fileToCheck instanceof List) return fileToCheck.every{ checkRefExistence(referenceFile, it) } def f = file(fileToCheck) - if (f instanceof List && f.size() > 0) { - // this is an expanded wildcard: we can assume all files exist - return true - } else if (!f.exists()) { - log.info "Missing references: $referenceFile $fileToCheck" + // this is an expanded wildcard: we can assume all files exist + if (f instanceof List && f.size() > 0) return true + else if (!f.exists()) { + log.info "Missing references: ${referenceFile} ${fileToCheck}" return false } return true @@ -1772,9 +1798,7 @@ def defineDirectoryMap() { } def defineReferenceMap() { - if (!(params.genome in params.genomes)) { - exit 1, "Genome $params.genome not found in configuration" - } + if (!(params.genome in params.genomes)) exit 1, "Genome ${params.genome} not found in configuration" return [ // loci file for ascat 'acLoci' : checkParamReturnFile("acLoci"), @@ -1860,8 +1884,8 @@ def extractFastq(tsvFile) { // Normally path to files starts from workflow.launchDir // But when executing workflow from Github // Path to hosted FASTQ files starts from workflow.projectDir - def fastqFile1 = workflow.commitId && params.test ? returnFile("$workflow.projectDir/${list[5]}") : returnFile("${list[5]}") - def fastqFile2 = workflow.commitId && params.test ? returnFile("$workflow.projectDir/${list[6]}") : returnFile("${list[6]}") + def fastqFile1 = workflow.commitId && params.test ? returnFile("${workflow.projectDir}/${list[5]}") : returnFile(list[5]) + def fastqFile2 = workflow.commitId && params.test ? returnFile("${workflow.projectDir}/${list[6]}") : returnFile(list[6]) checkFileExtension(fastqFile1,".fastq.gz") checkFileExtension(fastqFile2,".fastq.gz") @@ -1881,7 +1905,7 @@ def extractFastqFromDir(pattern) { // a temporary channel does all the work Channel .fromPath(pattern, type: 'dir') - .ifEmpty { error "No directories found matching pattern '$pattern'" } + .ifEmpty { error "No directories found matching pattern '${pattern}'" } .subscribe onNext: { sampleDir -> // the last name of the sampleDir is assumed to be a unique sample id sampleId = sampleDir.getFileName().toString() @@ -1889,9 +1913,7 @@ def extractFastqFromDir(pattern) { for (path1 in file("${sampleDir}/**_R1_*.fastq.gz")) { assert path1.getName().contains('_R1_') path2 = file(path1.toString().replace('_R1_', '_R2_')) - if (!path2.exists()) { - error "Path '${path2}' not found" - } + if (!path2.exists()) error "Path '${path2}' not found" (flowcell, lane) = flowcellLaneFromFastq(path1) patient = sampleId gender = 'ZZ' // unused @@ -2062,44 +2084,48 @@ def isAllowedParams(params) { def minimalInformationMessage() { // Minimal information message - log.info "Command Line: $workflow.commandLine" - log.info "Project Dir : $workflow.projectDir" - log.info "Launch Dir : $workflow.launchDir" - log.info "Work Dir : $workflow.workDir" - if (step != 'annotate') log.info "TSV file : $tsvFile" + log.info "Command Line: " + workflow.commandLine + log.info "Profile : " + workflow.profile + log.info "Project Dir : " + workflow.projectDir + log.info "Launch Dir : " + workflow.launchDir + log.info "Work Dir : " + workflow.workDir + log.info "Out Dir : " + params.outDir + if (step != 'annotate') log.info "TSV file : ${tsvFile}" log.info "Genome : " + params.genome + log.info "Genome_base : " + params.genome_base log.info "Step : " + step if (tools) log.info "Tools : " + tools.join(', ') if (annotateTools) log.info "Annotate on : " + annotateTools.join(', ') if (annotateVCF) log.info "VCF files : " +annotateVCF.join(',\n ') + log.info "Containers :" + if (params.repository) log.info " Repository : ${params.repository}" + else log.info " ContainerPath: " + params.containerPath + log.info " Tag : " + params.tag log.info "Reference files used:" - log.info " acLoci : $referenceMap.acLoci" - log.info " bwaIndex : " + referenceMap.bwaIndex.join(',\n ') - log.info " cosmic : $referenceMap.cosmic" - log.info " cosmicIndex : $referenceMap.cosmicIndex" - log.info " dbsnp : $referenceMap.dbsnp" - log.info " dbsnpIndex : $referenceMap.dbsnpIndex" - log.info " genomeDict : $referenceMap.genomeDict" - log.info " genomeFile : $referenceMap.genomeFile" - log.info " genomeIndex : $referenceMap.genomeIndex" - log.info " intervals : $referenceMap.intervals" - log.info " knownIndels : " + referenceMap.knownIndels.join(',\n ') - log.info " knownIndelsIndex: " + referenceMap.knownIndelsIndex.join(',\n ') - log.info " snpeffDb : ${params.genomes[params.genome].snpeffDb}" + log.info " acLoci :\n\t" + referenceMap.acLoci + log.info " cosmic :\n\t" + referenceMap.cosmic + log.info "\t" + referenceMap.cosmicIndex + log.info " dbsnp :\n\t" + referenceMap.dbsnp + log.info "\t" + referenceMap.dbsnpIndex + log.info " genome :\n\t" + referenceMap.genomeFile + log.info "\t" + referenceMap.genomeDict + log.info "\t" + referenceMap.genomeIndex + log.info " bwa indexes :\n\t" + referenceMap.bwaIndex.join(',\n\t') + log.info " intervals :\n\t" + referenceMap.intervals + log.info " knownIndels :\n\t" + referenceMap.knownIndels.join(',\n\t') + log.info "\t" + referenceMap.knownIndelsIndex.join(',\n\t') + log.info " snpeffDb :\n\t" + params.genomes[params.genome].snpeffDb } def nextflowMessage() { // Nextflow message (version + build) - log.info "N E X T F L O W ~ version $workflow.nextflow.version $workflow.nextflow.build" + log.info "N E X T F L O W ~ version ${workflow.nextflow.version} ${workflow.nextflow.build}" } def returnFile(it) { // return file if it exists - final f = file(it) - if (!f.exists()) { - exit 1, "Missing file in TSV file: $it, see --help for more information" - } - return f + if (!file(it).exists()) exit 1, "Missing file in TSV file: ${it}, see --help for more information" + return file(it) } def returnStatus(it) { @@ -2107,17 +2133,13 @@ def returnStatus(it) { // Status should be only 0 or 1 // 0 being normal // 1 being tumor (or relapse or anything that is not normal...) - if (!(it in [0, 1])) { - exit 1, "Status is not recognized in TSV file: $it, see --help for more information" - } + if (!(it in [0, 1])) exit 1, "Status is not recognized in TSV file: ${it}, see --help for more information" return it } def returnTSV(it, number) { // return TSV if it has the correct number of items in row - if (it.size() != number) { - exit 1, "Malformed row in TSV file: $it, see --help for more information" - } + if (it.size() != number) exit 1, "Malformed row in TSV file: ${it}, see --help for more information" return it } @@ -2130,8 +2152,8 @@ def startMessage() { def versionMessage() { // Display version message log.info "CANCER ANALYSIS WORKFLOW" - log.info " version : $version" - log.info workflow.commitId ? "Git info : $workflow.repository - $workflow.revision [$workflow.commitId]" : " revision : " + this.grabRevision() + log.info " version : " + version + log.info workflow.commitId ? "Git info : ${workflow.repository} - ${workflow.revision} [${workflow.commitId}]" : " revision : " + this.grabRevision() } workflow.onComplete { @@ -2139,10 +2161,10 @@ workflow.onComplete { this.nextflowMessage() this.cawMessage() this.minimalInformationMessage() - log.info "Completed at: $workflow.complete" - log.info "Duration : $workflow.duration" - log.info "Success : $workflow.success" - log.info "Exit status : $workflow.exitStatus" + log.info "Completed at: " + workflow.complete + log.info "Duration : " + workflow.duration + log.info "Success : " + workflow.success + log.info "Exit status : " + workflow.exitStatus log.info "Error report: " + (workflow.errorReport ?: '-') } @@ -2150,5 +2172,6 @@ workflow.onError { // Display error message this.nextflowMessage() this.cawMessage() - log.info "Workflow execution stopped with the following message: " + workflow.errorMessage + log.info "Workflow execution stopped with the following message:" + log.info " " + workflow.errorMessage } diff --git a/nextflow.config b/nextflow.config index a688e115cd..c3d901930c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -23,54 +23,76 @@ env { NXF_OPTS="-Xms1g -Xmx4g" } +includeConfig 'configuration/base.config' + profiles { // Runs the pipeline locally on a single 16-core node // Singularity images need to be set up standard { // Default profile for UPPMAX secure clusters includeConfig 'configuration/uppmax-localhost.config' includeConfig 'configuration/singularity-path.config' + includeConfig 'configuration/genomes.config' } // Runs the pipeline using the job scheduler // Singularity images need to be set up slurm { // slurm profile for UPPMAX secure clusters includeConfig 'configuration/uppmax-slurm.config' includeConfig 'configuration/singularity-path.config' + includeConfig 'configuration/genomes.config' } // Runs the pipeline locally on a single 16-core node // Singularity images will be pulled automatically download { // profile for UPPMAX clusters includeConfig 'configuration/uppmax-localhost.config' includeConfig 'configuration/singularity.config' + includeConfig 'configuration/genomes.config' } // Runs the pipeline using the job scheduler // Singularity images will be pulled automatically slurm-download { // slurm profile for UPPMAX clusters includeConfig 'configuration/uppmax-slurm.config' includeConfig 'configuration/singularity.config' + includeConfig 'configuration/genomes.config' } // Docker images will be pulled automatically docker { // For small testing testing with Docker includeConfig 'configuration/travis.config' includeConfig 'configuration/docker.config' includeConfig 'configuration/containers.config' + includeConfig 'configuration/genomes.config' } + // Docker images will be pulled automatically + awsbatch { // For running on AWS Batch with Docker + includeConfig 'configuration/aws-batch.config' + includeConfig 'configuration/docker.config' + includeConfig 'configuration/containers.config' + includeConfig 'configuration/genomes.config' + } // Singularity images will be pulled automatically singularity { // For small testing includeConfig 'configuration/travis.config' includeConfig 'configuration/singularity.config' includeConfig 'configuration/containers.config' + includeConfig 'configuration/genomes.config' + } + // Singularity images need to be set up + singularityPath { // For small testing + includeConfig 'configuration/travis.config' + includeConfig 'configuration/singularity-path.config' + includeConfig 'configuration/genomes.config' } } -// Path to references files -includeConfig 'configuration/genomes.config' - process.shell = ['/bin/bash', '-euo', 'pipefail'] timeline { // Turning on timeline tracking by default enabled = true } +report { // Turning on execution report by default + enabled = true +} + trace { // Turning on trace tracking by default enabled = true fields = 'process,task_id,hash,name,attempt,status,exit,realtime,%cpu,vmem,rss,submit,start,complete,duration,realtime,rchar,wchar' diff --git a/scripts/do_all.sh b/scripts/do_all.sh index 6da2761064..d8b813f204 100755 --- a/scripts/do_all.sh +++ b/scripts/do_all.sh @@ -4,7 +4,7 @@ set -xeuo pipefail PROFILE=singularity PUSH='' REPOSITORY=maxulysse -TAG=1.2.3 +TAG=1.2.5 TOOL=docker while [[ $# -gt 0 ]] diff --git a/scripts/test.sh b/scripts/test.sh index ddbba94fd7..fd6358e287 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,7 +4,7 @@ set -xeuo pipefail GENOME=smallGRCh37 PROFILE=singularity SAMPLE=data/tsv/tiny.tsv -TAG=1.2.3 +TAG=1.2.5 TEST=ALL TRAVIS=${TRAVIS:-false} @@ -45,13 +45,13 @@ done function nf_test() { echo "$(tput setaf 1)nextflow run $@ -profile $PROFILE --genome $GENOME -resume --verbose$(tput sgr0)" - nextflow run $@ -profile $PROFILE --genome $GENOME -resume --verbose + nextflow run $@ -profile $PROFILE --genome $GENOME -resume --genome_base $PWD/References/$GENOME --verbose } # Build references only for smallGRCh37 if [[ $GENOME == smallGRCh37 ]] && [[ $TEST != BUILDCONTAINERS ]] then - nf_test buildReferences.nf --download + nf_test buildReferences.nf --download --outDir References/$GENOME # Remove images only on TRAVIS if [[ $PROFILE == docker ]] && [[ $TRAVIS == true ]] then @@ -85,7 +85,7 @@ fi if [[ ALL,ANNOTATESNPEFF,ANNOTATEVEP =~ $TEST ]] then - nf_test . --step mapping --sample data/tsv/tiny-single-manta.tsv --tools Manta,Strelka + nf_test . --step mapping --sampleDir data/tiny/manta/normal --tools Manta,Strelka nf_test . --step mapping --sample data/tsv/tiny-manta.tsv --tools Manta,Strelka nf_test . --step mapping --sample $SAMPLE --tools MuTect2 @@ -112,7 +112,7 @@ then nf_test . --step annotate --tools ${ANNOTATOR} --annotateVCF VariantCalling/Manta/Manta_9876T_vs_1234N.diploidSV.vcf.gz --noReports fi -if [[ ALL,BUILDCONTAINERS =~ $TEST ]] +if [[ ALL,BUILDCONTAINERS =~ $TEST ]] && [[ $PROFILE == docker ]] then nf_test buildContainers.nf --docker --containers caw,fastqc,gatk,igvtools,multiqc,mutect1,picard,qualimap,runallelecount,r-base,snpeff fi