diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c238e9..2afb85c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#425]](https://github.com/nf-core/smrnaseq/pull/425) - Raise [minimum required NXF version for pipeline](https://github.com/nf-core/smrnaseq/issues/424) - usage of `arity` in some modules now requires this - [[#426]](https://github.com/nf-core/smrnaseq/pull/426) - Add [nf-core mirtop](https://github.com/nf-core/smrnaseq/issues/417) - replace local `mirtop` - [[#427]](https://github.com/nf-core/smrnaseq/pull/427) - Add [nf-core pigz uncompress](https://github.com/nf-core/smrnaseq/issues/422) - replace local `mirdeep_pigz` +- [[#429]](https://github.com/nf-core/smrnaseq/pull/429) - Make [saving of intermediate files optional](https://github.com/nf-core/smrnaseq/issues/424) - Allows user to choose whether to save intermediate files or not. Replaces several params that referred to the same such as `params.save_aligned` and `params.save_aligned_mirna_quant`. ## v2.3.1 - 2024-04-18 - Gray Zinc Dalmation Patch diff --git a/conf/modules.config b/conf/modules.config index 5dc3c678..f27533af 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -42,6 +42,14 @@ process { ] } + withName: 'UNTAR' { + publishDir = [ + mode: params.publish_dir_mode, + enabled: params.save_intermediates, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + // // FASTQ_FASTQC_UMITOOLS_FASTP // @@ -110,6 +118,7 @@ process { publishDir = [ path: { "${params.outdir}/fastqc/raw" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -119,6 +128,7 @@ process { publishDir = [ path: { "${params.outdir}/fastqc/trimmed" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -161,6 +171,7 @@ process { publishDir = [ path: { "${params.outdir}/bowtie_index/genome" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -249,6 +260,7 @@ process { publishDir = [ path: { "${params.outdir}/mirna_quant/reference" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -256,6 +268,7 @@ process { publishDir = [ path: { "${params.outdir}/mirna_quant/reference" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -263,6 +276,7 @@ process { publishDir = [ path: { "${params.outdir}/bowtie_index/mirna_mature" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -270,6 +284,7 @@ process { publishDir = [ path: { "${params.outdir}/bowtie_index/mirna_hairpin" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -278,7 +293,7 @@ process { path: { "${params.outdir}/mirna_quant/bam/mature" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, - enabled: params.save_aligned_mirna_quant + enabled: params.save_intermediates ] } withName: 'NFCORE_SMRNASEQ:MIRNA_QUANT:BAM_STATS_MATURE:.*' { @@ -286,6 +301,7 @@ process { publishDir = [ path: { "${params.outdir}/mirna_quant/bam/mature" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -294,7 +310,7 @@ process { path: { "${params.outdir}/mirna_quant/bam/hairpin" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, - enabled: params.save_aligned_mirna_quant + enabled: params.save_intermediates ] } withName: 'NFCORE_SMRNASEQ:MIRNA_QUANT:BAM_STATS_HAIRPIN:.*' { @@ -302,6 +318,7 @@ process { publishDir = [ path: { "${params.outdir}/mirna_quant/bam/hairpin" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -316,6 +333,7 @@ process { publishDir = [ path: { "${params.outdir}/mirna_quant/seqcluster" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] ext.args = "-m 1 --min_size 15" @@ -326,7 +344,7 @@ process { path: { "${params.outdir}/mirna_quant/bam/seqcluster" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, - enabled: params.save_aligned_mirna_quant + enabled: params.save_intermediates ] } @@ -371,12 +389,16 @@ process { publishDir = [ path: { "${params.outdir}/genome_quant/bam" }, mode: params.publish_dir_mode, + enabled: params.save_intermediates, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } withName: 'SAMTOOLS_INDEX' { ext.args = '-c' + publishDir = [ + enabled: params.save_intermediates, + ] } withName: 'NFCORE_SMRNASEQ:GENOME_QUANT:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' { @@ -392,7 +414,7 @@ process { path: { "${params.outdir}/genome_quant/bam" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename }, - enabled: params.save_aligned + enabled: params.save_intermediates ] } diff --git a/docs/output.md b/docs/output.md index 59bfed29..0fdba8f9 100644 --- a/docs/output.md +++ b/docs/output.md @@ -27,6 +27,8 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d - [MultiQC](#multiqc) - aggregate report, describing results of the whole pipeline - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution +If `--save_intermediates` is specified, intermediate files generated by each process will be saved in the output directory. + ### FastQC
@@ -102,6 +104,8 @@ MultiQC reports the number of reads that were removed by each of the contaminant - `sample_mature_hairpin_unmapped.fq.gz`: Unmapped reads against miRNA precursors (hairpins) - `sample_mature_hairpin_genome.bam`: The aligned BAM file of alignment against that didn't map to the precursor. +If `--save_intermediates` is specified, these files will be placed in this directory. + ## SAMtools [SAMtools](http://samtools.sourceforge.net/) is used for sorting and indexing the output BAM files from Bowtie. In addition, the numbers of features are counted with the `idxstats` option. diff --git a/docs/usage.md b/docs/usage.md index 7bf35ed5..06115967 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -164,6 +164,10 @@ genome: 'GRCh37' You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). +## Optional parameters + +If `--save_intermediates` is specified, the intermediate files generated in the pipeline will be saved in the output directory. + ### Updating the pipeline When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: diff --git a/nextflow.config b/nextflow.config index 8652c99e..0c7f979f 100644 --- a/nextflow.config +++ b/nextflow.config @@ -10,7 +10,6 @@ params { // Input options - input = null // Protocol default (override via config profile - NOT directly via parameter!) @@ -28,10 +27,11 @@ params { mirgenedb_hairpin = null mirgenedb_gff = null mirgenedb_species = null - save_aligned = false - save_aligned_mirna_quant = true bowtie_index = null + // General pipeline configuration + save_intermediates = false + // UMI handling with_umi = false // skips umitools extract in FASTQ_FASTQC_UMITOOLS_FASTP subworkflow. Needs to be true for fastq mode in collapsing reads @@ -50,7 +50,7 @@ params { fastp_min_length = 17 fastp_known_mirna_adapters = "$projectDir/assets/known_adapters.fa" save_trimmed_fail = false - save_merged = true + save_merged = false skip_fastqc = false skip_multiqc = false skip_mirdeep = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 765c231a..88cea8a9 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -47,6 +47,11 @@ "default": "custom", "fa_icon": "fas fa-atom", "enum": ["custom", "illumina", "nextflex", "qiaseq", "cats"] + }, + "save_intermediates": { + "type": "boolean", + "description": "Save all intermediate files (e.g. fastq, bams) to output directory", + "fa_icon": "fas fa-save" } } }, @@ -185,19 +190,6 @@ "help_text": "Saving generated references means that you can use them for future pipeline runs, reducing processing times.", "fa_icon": "fas fa-save" }, - "save_aligned": { - "type": "boolean", - "fa_icon": "fas fa-save", - "help_text": "Save aligned reads of initial bowtie mapping.", - "description": "Save aligned reads of initial mapping in bam format." - }, - "save_aligned_mirna_quant": { - "type": "boolean", - "fa_icon": "fas fa-save", - "default": true, - "help_text": "Save aligned reads of the bowtie runs in BOWTIE_MAP_MATURE, BOWTIE_MAP_HAIRPIN, and BOWTIE_MAP_SEQCLUSTER.", - "description": "Save aligned reads of miRNA quant subworkflow in bam format." - }, "igenomes_ignore": { "type": "boolean", "description": "Do not load the iGenomes reference config.", @@ -270,7 +262,7 @@ "save_merged": { "type": "boolean", "description": "Save merged reads.", - "default": true + "default": false }, "phred_offset": { "type": "integer",