Merge pull request #229 from atrigila/fix_dir_structure

Fix directory structure for nf-tests
nf-core · Aug 7, 2024 · 9938c0a · 9938c0a
2 parents af9862f + 14b0bb0
commit 9938c0a
Show file tree

Hide file tree

Showing 10 changed files with 234 additions and 45 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -27,6 +27,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#219](https://github.com/nf-core/demultiplex/pull/219) Modified workflow to store samplesheet in results folder.
 - [#217](https://github.com/nf-core/demultiplex/pull/217) Update all nf-core modules and tests.
 
+### `Fixed`
+
+- [#224](https://github.com/nf-core/demultiplex/pull/217) Fix input filename collision.
+
 ## 1.4.1 - 2024-02-27
 
 ### `Changed`

diff --git a/conf/test.config b/conf/test.config
@@ -44,12 +44,14 @@ process {
                 // Gather and write Reports
                 path: { meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}" },
                 mode: params.publish_dir_mode,
-                pattern: "Reports",
-                saveAs: {filename -> filename.split("/")[-1] }
+                pattern: "**/{Reports}",
+                saveAs: { filename -> filename.minus("output/") }
             ],
             [
                 path: { "${params.outdir}/${meta.id}" },
+                mode: params.publish_dir_mode,
                 pattern: "**.fastq.gz",
+                saveAs: { filename -> filename.minus("output/") }
             ]
         ]
     }
@@ -70,12 +72,13 @@ process {
                 // Gather and write Reports and Stats
                 path: { meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}" },
                 mode: params.publish_dir_mode,
-                pattern: "{Reports,Stats}",
-                saveAs: {filename -> filename.split("/")[-1] }
+                pattern: "**/{Reports,Stats}",
+                saveAs: { filename -> filename.minus("output/") }
             ],
             [
                 path: { "${params.outdir}/${meta.id}" },
                 pattern: "**.fastq.gz",
+                saveAs: { filename -> filename.minus("output/") }
             ]
         ]
     }

diff --git a/tests/pipeline/bcl2fastq.nf.test b/tests/pipeline/bcl2fastq.nf.test
@@ -28,10 +28,10 @@ nextflow_pipeline {
                     ).match("multiqc") },
                 { assert snapshot(
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_summary.txt"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Stats/")
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Stats/").list()
                     ).match("bcl2fastq") }
             )
         }

diff --git a/tests/pipeline/bcl2fastq.nf.test.snap b/tests/pipeline/bcl2fastq.nf.test.snap
@@ -13,13 +13,62 @@
         "content": [
             "Sample1_S1_L001.fastp.fastq.gz:md5,64639ff33d4b07d62a9079f88cbbd373",
             "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
-            "Sample1_S1_L001_summary.txt:md5,510c6b8a26ff8d530950c05b3ca5a1c6"
+            "Sample1_S1_L001_summary.txt:md5,510c6b8a26ff8d530950c05b3ca5a1c6",
+            [
+                [
+                    [
+                        [
+                            [
+                                [
+                                    "lane.html:md5,794e48287f47a9f22dcb6b6d0c22c3eb",
+                                    "laneBarcode.html:md5,2bbdae3ee57151eab520c966597d7438"
+                                ],
+                                [
+
+                                ]
+                            ]
+                        ],
+                        [
+                            [
+                                [
+                                    "lane.html:md5,f741870307050dcea79a838f5971770f",
+                                    "laneBarcode.html:md5,ffe2e863811c76cb9da27d5d124e0611"
+                                ],
+                                [
+                                    "lane.html:md5,cca97e771d3491dbc8cd3fe389595b09",
+                                    "laneBarcode.html:md5,cca97e771d3491dbc8cd3fe389595b09"
+                                ]
+                            ],
+                            [
+                                [
+                                    "lane.html:md5,c383b0768d9978733d3f5d3b91c15af0",
+                                    "laneBarcode.html:md5,48842c23b9a2816aec540177df870968"
+                                ],
+                                [
+
+                                ]
+                            ]
+                        ]
+                    ],
+                    "Report.css:md5,eb7d3eb68fc1539f411404987246b59b",
+                    "index.html:md5,5747c407854ae2c358d0ec201ce622d8",
+                    "tree.html:md5,a1b9bf592973ca829ec69ddf888b7e34"
+                ]
+            ],
+            [
+                "AdapterTrimming.txt:md5,48ed2b914b1246c0b5d8667525550946",
+                "ConversionStats.xml:md5,8fe0f57f3f5d256a0762dba75ac62d05",
+                "DemultiplexingStats.xml:md5,2047ff18f5b9107c084de06e9ff943ad",
+                "DemuxSummaryF1L1.txt:md5,03e5fd0c1e3079c5f8c7b4d0501b37ff",
+                "FastqSummaryF1L1.txt:md5,0c6f2d87ee183b84d1051cde9a5643d1",
+                "Stats.json:md5,8e5f038b8aa9e465599d3575f930e604"
+            ]
         ],
         "meta": {
             "nf-test": "0.9.0",
             "nextflow": "23.10.1"
         },
-        "timestamp": "2024-08-06T15:23:06.503332"
+        "timestamp": "2024-08-07T14:59:00.778899"
     },
     "multiqc": {
         "content": [

diff --git a/tests/pipeline/bclconvert.nf.test b/tests/pipeline/bclconvert.nf.test
@@ -27,10 +27,10 @@ nextflow_pipeline {
                     ).match("multiqc") },
                 { assert snapshot(
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_summary.txt"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Undetermined_S0_L001_R1_001.fastq.gz"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/")
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list()
                     ).match("bclconvert") }
             )
         }

diff --git a/tests/pipeline/bclconvert.nf.test.snap b/tests/pipeline/bclconvert.nf.test.snap
@@ -4,13 +4,27 @@
             "Sample1_S1_L001.fastp.fastq.gz:md5,64639ff33d4b07d62a9079f88cbbd373",
             "Sample1_S1_L001_R1_001.fastq.gz:md5,0675fb6365322eaafb33c0f8e862b54b",
             "Sample1_S1_L001_summary.txt:md5,510c6b8a26ff8d530950c05b3ca5a1c6",
-            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e"
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e",
+            [
+                "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
+                "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
+                "Demultiplex_Stats.csv:md5,93949a8cd96f907d83e0808c1ec2a04b",
+                "Demultiplex_Tile_Stats.csv:md5,83120160b0f22a1303fa1db31c19f6e9",
+                "IndexMetricsOut.bin:md5,9e688c58a5487b8eaf69c9e1005ad0bf",
+                "Index_Hopping_Counts.csv:md5,1059369e375fd8f8423c0f6c934be978",
+                "Quality_Metrics.csv:md5,6614accb1bb414fe312b17b81f5521f7",
+                "Quality_Tile_Metrics.csv:md5,cdc89fd2962bdd4a24f71e186112118a",
+                "RunInfo.xml:md5,03038959f4dd181c86bc97ae71fe270a",
+                "SampleSheet.csv:md5,ee5db2e12754e069998b0a96e535238c",
+                "Top_Unknown_Barcodes.csv:md5,2e2faba761137f228e56bd3428453ccc",
+                "fastq_list.csv:md5,ae1470cd741e33b87a4ad26f849a50f9"
+            ]
         ],
         "meta": {
             "nf-test": "0.9.0",
             "nextflow": "23.10.1"
         },
-        "timestamp": "2024-08-06T15:29:26.612324"
+        "timestamp": "2024-08-07T15:04:02.812091"
     },
     "software_versions": {
         "content": [

diff --git a/tests/pipeline/skip_tools.nf.test b/tests/pipeline/skip_tools.nf.test
@@ -24,10 +24,10 @@ nextflow_pipeline {
                 { assert workflow.trace.succeeded().size() == 5 },
                 { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Undetermined_S0_L001_R1_001.fastq.gz"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
                     ).match("skip_trimming") }
             )
         }
@@ -52,10 +52,10 @@ nextflow_pipeline {
                 { assert workflow.trace.succeeded().size() == 4 },
                 { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Undetermined_S0_L001_R1_001.fastq.gz"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
                     ).match("skip_fastp") }
             )
         }
@@ -80,10 +80,10 @@ nextflow_pipeline {
                 { assert workflow.trace.succeeded().size() == 5 },
                 { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz.md5"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Undetermined_S0_L001_R1_001.fastq.gz"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
                     ).match("skip_fastqc") }
             )
         }
@@ -108,10 +108,10 @@ nextflow_pipeline {
                 { assert workflow.trace.succeeded().size() == 4 },
                 { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Undetermined_S0_L001_R1_001.fastq.gz"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
                     ).match("skip_fastp_fastqc") }
             )
         }
@@ -136,10 +136,10 @@ nextflow_pipeline {
                 { assert workflow.trace.succeeded().size() == 4 },
                 { assert !path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz.md5"),
-                        path("$outputDir/220422_M11111_0222_000000000-K9H97/output/Undetermined_S0_L001_R1_001.fastq.gz"),
-                        //path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
                     ).match("skip_multiqc") }
             )
         }