nf-core · LouisLeNezet · Nov 24, 2024 · Nov 10, 2024 · Nov 11, 2024 · Nov 11, 2024
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -64,6 +64,7 @@ Special thanks to [Matthias Hörtenhuber](https://github.com/mashehu) and [Mazza
 - [#153](https://github.com/nf-core/phaseimpute/pull/153) - Update and simplify subworkflows snapshot and check only for files names (no md5sum for bam and vcf files due to timestamp).
 - [#157](https://github.com/nf-core/phaseimpute/pull/157) - Add `chunk_model` as parameter for better control over `GLIMPSE2_CHUNK` and set window size in `GLIMPSE1_CHUNK` and `GLIMPSE2_chunk` to 4mb to reduce number of chunks (empirical).
 - [#160](https://github.com/nf-core/phaseimpute/pull/160) - Improve `CHANGELOG.md` and add details to `usage.md`
+- [#158](https://github.com/nf-core/phaseimpute/pull/158) - Remove frequency computation and phasing from full test to reduce cost and computational time.
 
 ### `Fixed`
 
@@ -73,6 +74,7 @@ Special thanks to [Matthias Hörtenhuber](https://github.com/mashehu) and [Mazza
 - [#75](https://github.com/nf-core/phaseimpute/pull/75) - Set frequency computation with `VCFFIXUP` process as optional with `--compute_freq`. Use `GLIMPSE_CHUNK` on panel vcf to compute the chunk and not makewindows on fasta.
 - [#117](https://github.com/nf-core/phaseimpute/pull/117) - Fix directories in CSV.
 - [#151](https://github.com/nf-core/phaseimpute/pull/151) - Fix `Type not supported: class org.codehaus.groovy.runtime.GStringImpl` error due to `String` test in `getFileExtension()`.
+- [#158](https://github.com/nf-core/phaseimpute/pull/158) - Fix contigs usage when regions is only a subset of the given contigs (e.g. if panel file has the 22 chr and the region file only 2 then only the 2 common will be processed). Fix `multiQC` samples names for better comprehension. Fix `-resume` errors when `ch_fasta` is use by adding `cache = 'lenient'` in necessary processes. Fix `--window-size` of `GLIMPSE_CHUNK` from `4` to `4000000`.
 - [#153](https://github.com/nf-core/phaseimpute/pull/153) - Fix getFileExtension function. Fix image in `usage.md`. Fix small warnings and errors with updated language server. `def` has been added when necessary, `:` use instead of `,` in assertions, `_` added to variables not used in closures, `for` loop replaced by `.each{}`, remove unused code / input.
 - [#161](https://github.com/nf-core/phaseimpute/pull/161) - Fix `VCF_SPLIT_BCFTOOLS` when only one sample present by updating `BCFTOOLS_PLUGINSPLIT` and adding `BCFTOOLS_QUERY` to get truth samples names for renaming the resulting files.
 

diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml
@@ -17,3 +17,10 @@ top_modules:
 export_plots: true
 
 disable_version_detection: true
+
+fn_clean_trim:
+  - ".panel"
+
+extra_fn_clean_exts:
+  - type: regex
+    pattern: "\\.batch[0-9]+"
diff --git a/conf/modules.config b/conf/modules.config
@@ -31,11 +31,13 @@ process {
     // Simulation
     // Coverage process
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:SAMTOOLS_COVERAGE_INP' {
-        ext.prefix = { "${meta.id}.A-truth" }
+        cache = "lenient"
+        ext.prefix = { "${meta.id}.truth" }
         publishDir = [ enabled: false ]
     }
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:SAMTOOLS_COVERAGE_DWN' {
-        ext.prefix = { "${meta.id}.B-sim" }
+        cache = "lenient"
+        ext.prefix = { "${meta.id}" }
         publishDir = [ enabled: false ]
     }
 
@@ -75,7 +77,7 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:LIST_TO_FILE' {
         tag = { "${meta.id}" }
-        ext.prefix = { "${meta.id}.${meta.batch}" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}" }
         publishDir = [
             path: { "${params.outdir}/imputation/batch" },
             mode: params.publish_dir_mode,

diff --git a/conf/steps/imputation_glimpse1.config b/conf/steps/imputation_glimpse1.config
@@ -26,6 +26,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_GLIMPSE1:BCFTOOLS_MPILEUP' {
+        cache = "lenient"
         ext.args = [
             "-I",
             "-E",
@@ -39,6 +40,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_GLIMPSE1:BCFTOOLS_MERGE' {
+        cache = "lenient"
         ext.args = [
             "--write-index=tbi",
         ].join(' ')
@@ -58,7 +60,7 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:GLIMPSE_PHASE' {
         ext.args   = ["--impute-reference-only-variants"].join(' ')
-        ext.prefix = { "${meta.id}_batch${meta.batch}_${meta.chunk.replace(':','_')}_phase" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.${meta.chunk.replace(':','_')}.glimpse1" }
         ext.suffix = "bcf"
         publishDir = [ enabled: false ]
     }
@@ -69,7 +71,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_IMPUTE_GLIMPSE1:GLIMPSE_LIGATE' {
-        ext.prefix = { "${meta.id}_batch${meta.batch}_${meta.chr}_ligate" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.${meta.chr}.ligate.glimpse1" }
         publishDir = [ enabled: false ]
     }
 
@@ -89,6 +91,6 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_GLIMPSE1:BCFTOOLS_CONCAT' {
         ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
-        ext.prefix = { "${meta.id}_batch${meta.batch}.glimpse1" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.glimpse1" }
     }
 }
diff --git a/conf/steps/imputation_glimpse2.config b/conf/steps/imputation_glimpse2.config
@@ -18,7 +18,8 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_GLIMPSE2:GLIMPSE2_PHASE' {
-        ext.prefix   = { "${meta.id}_batch${meta.batch}_${meta.chunk.replace(':','_')}_glimpse2" }
+        cache = "lenient"
+        ext.prefix   = { "${meta.id}.batch${meta.batch}.${meta.chunk.replace(':','_')}.glimpse2" }
         ext.args     = { "--keep-monomorphic-ref-sites" }
         ext.suffix   = "bcf"
         publishDir   = [ enabled: false ]
@@ -30,7 +31,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_GLIMPSE2:GLIMPSE2_LIGATE' {
-        ext.prefix = { "${meta.id}_batch${meta.batch}_${meta.chr}_ligate" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.${meta.chr}.ligate" }
         publishDir = [ enabled: false ]
     }
 
@@ -51,6 +52,6 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_GLIMPSE2:BCFTOOLS_CONCAT' {
         ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
-        ext.prefix = { "${meta.id}_batch${meta.batch}.glimpse2" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.glimpse2" }
     }
 }
diff --git a/conf/steps/imputation_quilt.config b/conf/steps/imputation_quilt.config
@@ -23,14 +23,14 @@ process {
     // Impute quilt
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:QUILT_QUILT' {
         ext.args = "--seed=${params.seed}"
-        ext.prefix = { "${meta.id}_batch${meta.batch}_${meta.chr}_${meta.chunk}.impute" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.${meta.chunk.replace(':','_')}.quilt" }
         publishDir = [enabled: false]
     }
 
     // Annotate quilt imputed VCFs
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:BCFTOOLS_ANNOTATE' {
         ext.args     = ["--set-id '%CHROM:%POS:%REF:%ALT'", "-Oz", "--write-index=tbi"].join(' ')
-        ext.prefix = { "${meta.id}_batch${meta.batch}_${meta.chr}_${meta.chunk}.impute.annotate" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.${meta.chunk.replace(':','_')}.quilt.annotate" }
         publishDir = [ enabled: false ]
     }
 
@@ -45,6 +45,6 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_QUILT:BCFTOOLS_CONCAT' {
         ext.args = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
-        ext.prefix = { "${meta.id}_batch${meta.batch}.quilt" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.quilt" }
     }
 }
diff --git a/conf/steps/imputation_stitch.config b/conf/steps/imputation_stitch.config
@@ -18,14 +18,15 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_STITCH:GAWK' {
         ext.args2  = "'BEGIN { OFS=\"\\t\" } NR>1  { split(\$1, a, \"[:-_]\"); print a[1], \$2, \$3, \$4 }'" // Remove duplicates
-        ext.prefix = { "${meta.id}_batch${meta.batch}_${meta.chr}_posfile_stitch" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.${meta.chr}.posfile.stitch" }
         ext.suffix = "txt"
         publishDir = [enabled: false]
         tag = {"${meta.id} ${meta.chr}"}
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_STITCH:STITCH' {
-        ext.prefix = { "${meta.id}_batch${meta.batch}.stitch" }
+        cache = "lenient"
+        ext.prefix = { "${meta.id}.batch${meta.batch}.${meta.chr}.stitch" }
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_STITCH:BCFTOOLS_INDEX' {
@@ -44,6 +45,6 @@ process {
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_STITCH:BCFTOOLS_CONCAT' {
         ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
-        ext.prefix = { "${meta.id}_batch${meta.batch}.stitch" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.stitch" }
     }
 }
diff --git a/conf/steps/panel_prep.config b/conf/steps/panel_prep.config
@@ -185,7 +185,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE_CHUNK' {
-        ext.args   = "--window-size 4"
+        ext.args   = "--window-size 4000000"
         ext.prefix = { "${meta.id}_${meta.chr}_chunks_glimpse1" }
         publishDir = [
             path: { "${params.outdir}/prep_panel/chunks/glimpse1/" },

diff --git a/conf/steps/validation.config b/conf/steps/validation.config
@@ -35,7 +35,7 @@ process {
             "-Aim",
             "-C alleles"
         ].join(' ')
-        ext.prefix = { "${meta.id}_truth.call" }
+        ext.prefix = { "${meta.id}.truth.call" }
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_TRUTH:BCFTOOLS_MERGE' {
@@ -52,7 +52,7 @@ process {
 
     // Concatenate the truth set
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_TRUTH:.*' {
-        ext.prefix = { "${meta.id}.A-truth" }
+        ext.prefix = { "${meta.id}.batch${meta.batch}.truth" }
         publishDir = [
             path: { "${params.outdir}/validation/concat" },
             mode: params.publish_dir_mode,
@@ -84,7 +84,7 @@ process {
         publishDir = [
             path: { "${params.outdir}/validation/samples" },
             mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename}
         ]
     }
 

diff --git a/conf/test.config b/conf/test.config
@@ -13,7 +13,7 @@
 process {
     resourceLimits = [
         cpus: 4,
-        memory: '15.GB',
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_all.config b/conf/test_all.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '10.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_batch.config b/conf/test_batch.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_dog.config b/conf/test_dog.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_full.config b/conf/test_full.config
@@ -36,8 +36,8 @@ params {
     // Panelprep optional args
     remove_samples = "NA12878,NA12891,NA12892"
     normalize      = true
-    compute_freq   = true
-    phase          = true
+    compute_freq   = false
+    phase          = false
 
     // Impute tools
     tools = "glimpse1"

diff --git a/conf/test_glimpse2.config b/conf/test_glimpse2.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_panelprep.config b/conf/test_panelprep.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_quilt.config b/conf/test_quilt.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_sim.config b/conf/test_sim.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_stitch.config b/conf/test_stitch.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/conf/test_validate.config b/conf/test_validate.config
@@ -12,8 +12,8 @@
 
 process {
     resourceLimits = [
-        cpus: 2,
-        memory: '2.GB',
+        cpus: 4,
+        memory: '4.GB',
         time: '1.h'
     ]
 }

diff --git a/docs/usage.md b/docs/usage.md
@@ -241,6 +241,18 @@ The required flags for this mode are:
 - `--remove_samples`: (optional) A comma-separated list of samples to remove from the reference during the normalization process.
 - `--compute_freq`: (optional) Whether the frequency (AC/AN field) for each variants needs to be computed or not (true/false). This can be the case if the frequency is absent from the reference panel or if individuals have been removed.
 
+The panel will be chunked using the `GLIMPSE_CHUNKS` process. The size of the chunks can be optimized according to your needs (e.g. cluster resources, specie chromosomes size, ...) using the following config. The 4mb size (default value) is empirically determined to be a good value in humans (i.e. enough parallelization but not too much).
+
+```config title="panel.config"
+withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE_CHUNK' {
+    ext.args   = "--window-size 4000000"
+}
+
+withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE2_CHUNK' {
+    ext.args   = "--window-mb 4"
+}
+```
+
 You can find an overview of the results produced by this steps in the [Output](output.md).
 
 ## Start with imputation `--steps impute`

diff --git a/subworkflows/local/bam_impute_quilt/main.nf b/subworkflows/local/bam_impute_quilt/main.nf
@@ -37,7 +37,7 @@ workflow BAM_IMPUTE_QUILT {
         .map {
             metaI, bam, bai, bamlist, metaPC, hap, legend, chr, start, end, ngen, buffer, gmap ->
             [
-                metaI + [panel: metaPC.id, chr: metaPC.chr, chunk: start + "-" + end],
+                metaI + [panel: metaPC.id, chr: metaPC.chr, chunk: metaPC.chr + ":" + start + "-" + end],
                 bam, bai, bamlist, hap, legend, chr, start, end, ngen, buffer, gmap
             ]
         }

diff --git a/subworkflows/local/utils_nfcore_phaseimpute_pipeline/main.nf b/subworkflows/local/utils_nfcore_phaseimpute_pipeline/main.nf
@@ -275,6 +275,25 @@ workflow PIPELINE_INITIALISATION {
         .collect()
         .subscribe { log.info "The following contigs will be processed: ${it}" }
 
+    // Remove other contigs from panel and posfile files
+    ch_panel = ch_panel
+        .combine(ch_regions.collect{ it[0]["chr"]}.toList())
+        .filter { meta, _vcf, _index, chrs ->
+            meta.chr in chrs
+        }
+        .map {meta, vcf, index, _chrs ->
+            [meta, vcf, index]
+        }
+
+    ch_posfile = ch_posfile
+        .combine(ch_regions.collect{ it[0]["chr"]}.toList())
+        .filter { meta, _vcf, _index, _hap, _legend, chrs ->
+            meta.chr in chrs
+        }
+        .map {meta, vcf, index, hap, legend, _chrs ->
+            [meta, vcf, index, hap, legend]
+        }
+
     // Check that all input files have the correct index
     checkFileIndex(ch_input.mix(ch_input_truth, ch_ref_gen, ch_panel))