Add support for generating taxprofiler/funcscan input samplesheets for preprocessed FASTQs/FASTAs

conf/test_hybrid.config

@@ -27,4 +27,8 @@ params {
     skip_gtdbtk                 = true
     gtdbtk_min_completeness     = 0
     skip_concoct                = true
+
+    // Generate downstream samplesheets
+    generate_downstream_samplesheets = true
+    generate_pipeline_samplesheets   = null
 }

docs/output.md

@@ -26,6 +26,9 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d
 
 Note that when specifying the parameter `--coassemble_group`, for the corresponding output filenames/directories of the assembly or downsteam processes the group ID, or more precisely the term `group-[group_id]`, will be used instead of the sample ID.
 
+The pipeline can also generate downstream pipeline input samplesheets.
+These are stored in `<outdir>/downstream_samplesheets`.
+
 ## Quality control
 
 These steps trim away the adapter sequences present in input reads, trims away bad quality bases and sicard reads that are too short.
@@ -707,6 +710,9 @@ Because of aDNA damage, _de novo_ assemblers sometimes struggle to call a correc
 
 </details>
 
+The pipeline can also generate input samplesheets for downstream pipelines.
+These are stored in `<outdir>/downstream_samplesheets`.
+
 ### MultiQC
 
 <details markdown="1">
@@ -751,3 +757,24 @@ Summary tool-specific plots and tables of following tools are currently displaye
 </details>
 
 [Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage.
+
+### Downstream samplesheets
+
+The pipeline can also generate input files for the following downstream pipelines:
+
+- [nf-core/funcscan](https://nf-co.re/funcscan)
+- [nf-core/taxprofiler](https://nf-co.re/taxprofiler)
+
+<details markdown="1">
+<summary>Output files</summary>
+
+- `downstream_samplesheets/`
+  - `taxprofiler.csv`: Partially filled out nf-core/taxprofiler `--input` csv with paths to preprocessed reads (adapter trimmed, host removed etc.) in `.fastq.gz` formats. I.e., the direct input into MEGAHIT, SPAdes, SPAdesHybrid.
+  - `funcscan.csv`: Filled out nf-core/funcscan `--input` csv with absolute paths to the assembled contig FASTA files produced by nf-core/mag (i.e., the direct output from MEGAHIT, SPAdes, SPAdesHybrid - not bins).
+
+</details>
+
+:::warning
+Any generated downstream samplesheet is provided as 'best effort' and are not guaranteed to work straight out of the box!
+They may not be complete (e.g. some columns may need to be manually filled in).
+:::

nextflow.config

@@ -194,6 +194,9 @@ params {
     validationShowHiddenParams       = false
     validate_params                  = true
 
+    // Generate downstream samplesheets
+    generate_downstream_samplesheets = false
+    generate_pipeline_samplesheets   = null
 }
 
 // Load base.config by default for all pipelines

nextflow_schema.json

@@ -83,6 +83,26 @@
                 }
             }
         },
+        "generate_samplesheet_options": {
+            "title": "Downstream pipeline samplesheet generation options",
+            "type": "object",
+            "fa_icon": "fas fa-align-justify",
+            "description": "Options for generating input samplesheets for complementary downstream pipelines.",
+            "properties": {
+                "generate_downstream_samplesheets": {
+                    "type": "boolean",
+                    "description": "Turn on generation of samplesheets for downstream pipelines.",
+                    "fa_icon": "fas fa-toggle-on"
+                },
+                "generate_pipeline_samplesheets": {
+                    "type": "string",
+                    "description": "Specify which pipeline to generate a samplesheet for.",
+                    "help": "Note that the nf-core/funcscan samplesheet will only include paths to raw assemblies, not bins\n\nThe nf-core/taxprofiler samplesheet will include of paths the pre-processed reads that are used are used as input for _de novo_ assembly.",
+                    "fa_icon": "fas fa-toolbox",
+                    "pattern": "^(taxprofiler|funcscan)(?:,(taxprofiler|funcscan)){0,1}"
+                }
+            }
+        },
         "institutional_config_options": {
             "title": "Institutional config options",
             "type": "object",
@@ -914,6 +934,9 @@
         {
             "$ref": "#/definitions/reference_genome_options"
         },
+        {
+            "$ref": "#/definitions/generate_samplesheet_options"
+        },
         {
             "$ref": "#/definitions/institutional_config_options"
         },

subworkflows/local/generate_downstream_samplesheets/main.nf

@@ -0,0 +1,92 @@
+//
+// Subworkflow with functionality specific to the nf-core/mag pipeline
+//
+
+workflow SAMPLESHEET_TAXPROFILER {
+    take:
+    ch_reads
+
+    main:
+    format     = 'csv'
+
+    def fastq_rel_path = '/'
+    if (params.bbnorm) {
+        fastq_rel_path = '/bbmap/bbnorm/'
+    } else if (!params.keep_phix) {
+        fastq_rel_path = '/QC_shortreads/remove_phix/'
+    }
+    else if (params.host_fasta) {
+        fastq_rel_path = '/QC_shortreads/remove_host/'
+    }
+    else if (!params.skip_clipping) {
+        fastq_rel_path = '/QC_shortreads/fastp/'
+    }
+
+    ch_list_for_samplesheet = ch_reads
+        .map {
+            meta, fastq ->
+                def sample              = meta.id
+                def run_accession       = meta.id
+                def instrument_platform = ""
+                def fastq_1             = file(params.outdir).toString() + fastq_rel_path + meta.id + '/' + fastq[0].getName()
+                def fastq_2             = file(params.outdir).toString() + fastq_rel_path + meta.id + '/' + fastq[1].getName()
+                def fasta               = ""
+            [ sample: sample, run_accession: run_accession, instrument_platform: instrument_platform, fastq_1: fastq_1, fastq_2: fastq_2, fasta: fasta ]
+        }
+        .tap{ ch_colnames }
+
+    channelToSamplesheet(ch_list_for_samplesheet, "${params.outdir}/downstream_samplesheets/mag", format)
+
+}
+
+workflow SAMPLESHEET_FUNCSCAN {
+    take:
+    ch_assemblies
+
+    main:
+    format     = 'csv'
+
+    ch_list_for_samplesheet = ch_assemblies
+        .map {
+            meta, filename ->
+                def sample = meta.id
+                def fasta  = file(params.outdir).toString() + '/Assembly/' + meta.assembler + '/' + filename.getName()
+            [ sample: sample, fasta: fasta ]
+        }
+        .tap{ ch_colnames }
+
+    channelToSamplesheet(ch_list_for_samplesheet, "${params.outdir}/downstream_samplesheets/funcscan", format)
+}
+
+workflow GENERATE_DOWNSTREAM_SAMPLESHEETS {
+    take:
+    ch_reads
+    ch_assemblies
+
+    main:
+    def downstreampipeline_names = params.generate_pipeline_samplesheets.split(",")
+
+    if ( downstreampipeline_names.contains('taxprofiler') && params.save_clipped_reads ) { // save_clipped_reads must be true
+        SAMPLESHEET_TAXPROFILER(ch_reads)
+    }
+
+    if ( downstreampipeline_names.contains('funcscan') ) {
+        SAMPLESHEET_FUNCSCAN(ch_assemblies)
+    }
+}
+
+def channelToSamplesheet(ch_list_for_samplesheet, path, format) {
+    def format_sep = [csv: ",", tsv: "\t", txt: "\t"][format]
+
+    def ch_header = ch_list_for_samplesheet
+
+    ch_header
+        .first()
+        .map { it.keySet().join(format_sep) }
+        .concat(ch_list_for_samplesheet.map { it.values().join(format_sep) })
+        .collectFile(
+            name: "${path}.${format}",
+            newLine: true,
+            sort: false
+        )
+}

subworkflows/local/utils_nfcore_mag_pipeline/main.nf

@@ -118,6 +118,11 @@ workflow PIPELINE_INITIALISATION {
     //
     validateInputParameters(
         hybrid
+
+        // Validate samplesheet generation parameters
+        if (params.generate_downstream_samplesheets && !params.generate_pipeline_samplesheets) {
+            error('[nf-core/createtaxdb] If supplying `--generate_downstream_samplesheets`, you must also specify which pipeline to generate for with `--generate_pipeline_samplesheets! Check input.')
+        }
     )
 
     // Validate PRE-ASSEMBLED CONTIG input when supplied
@@ -330,6 +335,19 @@ def validateInputParameters(hybrid) {
     if (params.save_mmseqs_db && !params.metaeuk_mmseqs_db) {
         error('[nf-core/mag] ERROR: Invalid parameter combination: --save_mmseqs_db supplied but no database has been requested for download with --metaeuk_mmseqs_db!')
     }
+
+    // Validate samplesheet generation parameters
+    if (params.generate_downstream_samplesheets && !params.generate_pipeline_samplesheets) {
+        error('[nf-core/mag] If supplying `--generate_downstream_samplesheets`, you must also specify which pipeline to generate for with `--generate_pipeline_samplesheets! Check input.')
+    }
+
+    if (params.generate_downstream_samplesheets && !params.save_clipped_reads) {
+        error('[nf-core/mag] Supplied --generate_downstream_samplesheets but missing --save_clipped_reads (mandatory for reads.gz output).')
+    }
+
+    if (params.generate_downstream_samplesheets && params.save_clipped_reads && (params.bbnorm || !params.keep_phix || params.host_fasta || params.skip_clipping)) {
+        error('[nf-core/mag] Supplied --generate_downstream_samplesheets and --save_clipped_reads is true, but also need one of the following: --bbnorm true, or --keep_phix false, or --host_fasta true, or skip_clipping true.')
+    }
 }
 
 //

workflows/mag.nf

@@ -13,17 +13,18 @@ include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_mag_
 //
 // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules
 //
-include { BINNING_PREPARATION             } from '../subworkflows/local/binning_preparation'
-include { BINNING                         } from '../subworkflows/local/binning'
-include { BINNING_REFINEMENT              } from '../subworkflows/local/binning_refinement'
-include { BUSCO_QC                        } from '../subworkflows/local/busco_qc'
-include { VIRUS_IDENTIFICATION            } from '../subworkflows/local/virus_identification'
-include { CHECKM_QC                       } from '../subworkflows/local/checkm_qc'
-include { GUNC_QC                         } from '../subworkflows/local/gunc_qc'
-include { GTDBTK                          } from '../subworkflows/local/gtdbtk'
-include { ANCIENT_DNA_ASSEMBLY_VALIDATION } from '../subworkflows/local/ancient_dna'
-include { DOMAIN_CLASSIFICATION           } from '../subworkflows/local/domain_classification'
-include { DEPTHS                          } from '../subworkflows/local/depths'
+include { BINNING_PREPARATION              } from '../subworkflows/local/binning_preparation'
+include { BINNING                          } from '../subworkflows/local/binning'
+include { BINNING_REFINEMENT               } from '../subworkflows/local/binning_refinement'
+include { BUSCO_QC                         } from '../subworkflows/local/busco_qc'
+include { VIRUS_IDENTIFICATION             } from '../subworkflows/local/virus_identification'
+include { CHECKM_QC                        } from '../subworkflows/local/checkm_qc'
+include { GUNC_QC                          } from '../subworkflows/local/gunc_qc'
+include { GTDBTK                           } from '../subworkflows/local/gtdbtk'
+include { ANCIENT_DNA_ASSEMBLY_VALIDATION  } from '../subworkflows/local/ancient_dna'
+include { DOMAIN_CLASSIFICATION            } from '../subworkflows/local/domain_classification'
+include { DEPTHS                           } from '../subworkflows/local/depths'
+include { GENERATE_DOWNSTREAM_SAMPLESHEETS } from '../subworkflows/local/generate_downstream_samplesheets/main.nf'
 
 //
 // MODULE: Installed directly from nf-core/modules
@@ -1002,6 +1003,13 @@ workflow MAG {
         }
     }
 
+    //
+    // Samplesheet generation
+    //
+    if ( params.generate_downstream_samplesheets ) {
+        GENERATE_DOWNSTREAM_SAMPLESHEETS ( ch_short_reads_assembly, ch_assemblies )
+    }
+
     //
     // Collate and save software versions
     //