From 3149533049278246b1cf4acc28c83567bb0bf845 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 30 Aug 2022 13:37:56 +0000 Subject: [PATCH 01/34] Template update for nf-core/tools version 2.5 --- .editorconfig | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .github/workflows/ci.yml | 23 ++------ .github/workflows/linting.yml | 38 +++++++++++-- CHANGELOG.md | 2 +- CITATION.cff | 56 +++++++++++++++++++ README.md | 21 +++---- assets/email_template.txt | 1 - bin/check_samplesheet.py | 41 +++++++------- conf/base.config | 5 ++ docs/usage.md | 12 ++-- lib/WorkflowMain.groovy | 9 ++- lib/WorkflowRnavar.groovy | 5 +- main.nf | 2 +- modules.json | 22 +++++--- .../templates/dumpsoftwareversions.py | 14 +++-- nextflow.config | 23 +++++++- 17 files changed, 186 insertions(+), 93 deletions(-) create mode 100644 CITATION.cff diff --git a/.editorconfig b/.editorconfig index b6b31907..b78de6e6 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,7 +8,7 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js}] +[*.{md,yml,yaml,html,css,scss,js,cff}] indent_size = 2 # These files are edited and tested upstream in nf-core/modules diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ab5c01d7..5c26fb9c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,8 +15,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/rnav - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/rnavar/tree/master/.github/CONTRIBUTING.md) - - [ ] If necessary, also make a PR on the nf-core/rnavar _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/rnavar/tree/master/.github/CONTRIBUTING.md)- [ ] If necessary, also make a PR on the nf-core/rnavar _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cdf128e..67225b7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,6 @@ on: env: NXF_ANSI_LOG: false - CAPSULE_LOG: none jobs: test: @@ -20,27 +19,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Nextflow versions - include: - # Test pipeline minimum Nextflow version - - NXF_VER: "21.10.3" - NXF_EDGE: "" - # Test latest edge release of Nextflow - - NXF_VER: "" - NXF_EDGE: "1" + NXF_VER: + - "21.10.3" + - "latest-everything" steps: - name: Check out pipeline code uses: actions/checkout@v2 - name: Install Nextflow - env: - NXF_VER: ${{ matrix.NXF_VER }} - # Uncomment only if the edge release is more recent than the latest stable release - # See https://github.com/nextflow-io/nextflow/issues/2467 - # NXF_EDGE: ${{ matrix.NXF_EDGE }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" - name: Run pipeline with test data # TODO nf-core: You can customise CI pipeline run tests as required diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 77358dee..8a5ce69b 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -35,6 +35,36 @@ jobs: - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} + PythonBlack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Check code lints with Black + uses: psf/black@stable + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Python linting (`black`) is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` + * Fix formatting errors in your pipeline: `black .` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false + nf-core: runs-on: ubuntu-latest steps: @@ -42,15 +72,11 @@ jobs: uses: actions/checkout@v2 - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 - uses: actions/setup-python@v3 with: - python-version: "3.6" + python-version: "3.7" architecture: "x64" - name: Install dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index cb967546..9eb1e73a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v1.0dev - [date] +## v1.1.0dev - [date] Initial release of nf-core/rnavar, created with the [nf-core](https://nf-co.re/) template. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..4533e2f2 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,56 @@ +cff-version: 1.2.0 +message: "If you use `nf-core tools` in your work, please cite the `nf-core` publication" +authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven +title: "The nf-core framework for community-curated bioinformatics pipelines." +version: 2.4.1 +doi: 10.1038/s41587-020-0439-x +date-released: 2022-05-16 +url: https://github.com/nf-core/tools +prefered-citation: + type: article + authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven + doi: 10.1038/s41587-020-0439-x + journal: nature biotechnology + start: 276 + end: 278 + title: "The nf-core framework for community-curated bioinformatics pipelines." + issue: 3 + volume: 38 + year: 2020 + url: https://dx.doi.org/10.1038/s41587-020-0439-x diff --git a/README.md b/README.md index 7ba93db3..d80e7ac0 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,14 @@ # ![nf-core/rnavar](docs/images/nf-core-rnavar_logo_light.png#gh-light-mode-only) ![nf-core/rnavar](docs/images/nf-core-rnavar_logo_dark.png#gh-dark-mode-only) -[![GitHub Actions CI Status](https://github.com/nf-core/rnavar/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/rnavar/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/nf-core/rnavar/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/rnavar/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?logo=Amazon%20AWS)](https://nf-co.re/rnavar/results) -[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/rnavar/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) -[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?logo=anaconda)](https://docs.conda.io/en/latest/) -[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?logo=docker)](https://www.docker.com/) -[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg)](https://sylabs.io/docs/) +[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) +[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) +[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) [![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/rnavar) -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23rnavar-4A154B?logo=slack)](https://nfcore.slack.com/channels/rnavar) -[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?logo=twitter)](https://twitter.com/nf_core) -[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?logo=youtube)](https://www.youtube.com/c/nf-core) +[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23rnavar-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/rnavar)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction @@ -25,7 +20,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/rnavar/results). +On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources.The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/rnavar/results). ## Pipeline summary @@ -42,7 +37,7 @@ On release, automated continuous integration tests run the pipeline on a full-si 3. Download the pipeline and test it on a minimal dataset with a single command: - ```console + ```bash nextflow run nf-core/rnavar -profile test,YOURPROFILE --outdir ``` @@ -57,7 +52,7 @@ On release, automated continuous integration tests run the pipeline on a full-si - ```console + ```bash nextflow run nf-core/rnavar --input samplesheet.csv --outdir --genome GRCh37 -profile ``` diff --git a/assets/email_template.txt b/assets/email_template.txt index 2c43ec61..cbc1a918 100644 --- a/assets/email_template.txt +++ b/assets/email_template.txt @@ -6,7 +6,6 @@ `._,._,' nf-core/rnavar v${version} ---------------------------------------------------- - Run Name: $runName <% if (success){ diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 3652c63c..9a8b8962 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -11,7 +11,6 @@ from collections import Counter from pathlib import Path - logger = logging.getLogger() @@ -79,13 +78,15 @@ def validate_and_transform(self, row): def _validate_sample(self, row): """Assert that the sample name exists and convert spaces to underscores.""" - assert len(row[self._sample_col]) > 0, "Sample input is required." + if len(row[self._sample_col]) <= 0: + raise AssertionError("Sample input is required.") # Sanitize samples slightly. row[self._sample_col] = row[self._sample_col].replace(" ", "_") def _validate_first(self, row): """Assert that the first FASTQ entry is non-empty and has the right format.""" - assert len(row[self._first_col]) > 0, "At least the first FASTQ file is required." + if len(row[self._first_col]) <= 0: + raise AssertionError("At least the first FASTQ file is required.") self._validate_fastq_format(row[self._first_col]) def _validate_second(self, row): @@ -97,36 +98,34 @@ def _validate_pair(self, row): """Assert that read pairs have the same file extension. Report pair status.""" if row[self._first_col] and row[self._second_col]: row[self._single_col] = False - assert ( - Path(row[self._first_col]).suffixes[-2:] == Path(row[self._second_col]).suffixes[-2:] - ), "FASTQ pairs must have the same file extensions." + if Path(row[self._first_col]).suffixes[-2:] != Path(row[self._second_col]).suffixes[-2:]: + raise AssertionError("FASTQ pairs must have the same file extensions.") else: row[self._single_col] = True def _validate_fastq_format(self, filename): """Assert that a given filename has one of the expected FASTQ extensions.""" - assert any(filename.endswith(extension) for extension in self.VALID_FORMATS), ( - f"The FASTQ file has an unrecognized extension: {filename}\n" - f"It should be one of: {', '.join(self.VALID_FORMATS)}" - ) + if not any(filename.endswith(extension) for extension in self.VALID_FORMATS): + raise AssertionError( + f"The FASTQ file has an unrecognized extension: {filename}\n" + f"It should be one of: {', '.join(self.VALID_FORMATS)}" + ) def validate_unique_samples(self): """ Assert that the combination of sample name and FASTQ filename is unique. - In addition to the validation, also rename the sample if more than one sample, - FASTQ file combination exists. + In addition to the validation, also rename all samples to have a suffix of _T{n}, where n is the + number of times the same sample exist, but with different FASTQ files, e.g., multiple runs per experiment. """ - assert len(self._seen) == len(self.modified), "The pair of sample name and FASTQ must be unique." - if len({pair[0] for pair in self._seen}) < len(self._seen): - counts = Counter(pair[0] for pair in self._seen) - seen = Counter() - for row in self.modified: - sample = row[self._sample_col] - seen[sample] += 1 - if counts[sample] > 1: - row[self._sample_col] = f"{sample}_T{seen[sample]}" + if len(self._seen) != len(self.modified): + raise AssertionError("The pair of sample name and FASTQ must be unique.") + seen = Counter() + for row in self.modified: + sample = row[self._sample_col] + seen[sample] += 1 + row[self._sample_col] = f"{sample}_T{seen[sample]}" def read_head(handle, num_lines=10): diff --git a/conf/base.config b/conf/base.config index 71e76362..fcccdad9 100644 --- a/conf/base.config +++ b/conf/base.config @@ -26,6 +26,11 @@ process { // adding in your local modules too. // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors + withLabel:process_single { + cpus = { check_max( 1 , 'cpus' ) } + memory = { check_max( 6.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } diff --git a/docs/usage.md b/docs/usage.md index 3d9775d0..3878607a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -12,7 +12,7 @@ You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row as shown in the examples below. -```console +```bash --input '[path to samplesheet file]' ``` @@ -56,7 +56,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: -```console +```bash nextflow run nf-core/rnavar --input samplesheet.csv --outdir --genome GRCh37 -profile docker ``` @@ -64,9 +64,9 @@ This will launch the pipeline with the `docker` configuration profile. See below Note that the pipeline will create the following files in your working directory: -```console +```bash work # Directory containing the nextflow working files - # Finished results in specified location (defined with --outdir) + # Finished results in specified location (defined with --outdir) .nextflow_log # Log file from Nextflow # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` @@ -75,7 +75,7 @@ work # Directory containing the nextflow working files When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: -```console +```bash nextflow pull nf-core/rnavar ``` @@ -251,6 +251,6 @@ Some HPC setups also allow you to run nextflow within a cluster job submitted yo In some cases, the Nextflow Java virtual machines can start to request a large amount of memory. We recommend adding the following line to your environment to limit this (typically in `~/.bashrc` or `~./bash_profile`): -```console +```bash NXF_OPTS='-Xms1g -Xmx4g' ``` diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 43e5694a..35bf164f 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -59,6 +59,7 @@ class WorkflowMain { } // Print parameter summary log to screen + log.info paramsSummaryLog(workflow, params, log) // Check that a -profile or Nextflow config has been provided to run the pipeline @@ -78,17 +79,15 @@ class WorkflowMain { System.exit(1) } } - // // Get attribute from genome config file e.g. fasta // - public static String getGenomeAttribute(params, attribute) { - def val = '' + public static Object getGenomeAttribute(params, attribute) { if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { if (params.genomes[ params.genome ].containsKey(attribute)) { - val = params.genomes[ params.genome ][ attribute ] + return params.genomes[ params.genome ][ attribute ] } } - return val + return null } } diff --git a/lib/WorkflowRnavar.groovy b/lib/WorkflowRnavar.groovy index cf3ff3b6..162b1db3 100755 --- a/lib/WorkflowRnavar.groovy +++ b/lib/WorkflowRnavar.groovy @@ -10,6 +10,7 @@ class WorkflowRnavar { public static void initialise(params, log) { genomeExistsError(params, log) + if (!params.fasta) { log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." System.exit(1) @@ -41,9 +42,7 @@ class WorkflowRnavar { yaml_file_text += "data: |\n" yaml_file_text += "${summary_section}" return yaml_file_text - } - - // + }// // Exit pipeline if incorrect --genome key provided // private static void genomeExistsError(params, log) { diff --git a/main.nf b/main.nf index 6fc4d58a..22dbf6ca 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,7 @@ nf-core/rnavar ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/rnavar - Website: https://nf-co.re/rnavar +Website: https://nf-co.re/rnavar Slack : https://nfcore.slack.com/channels/rnavar ---------------------------------------------------------------------------------------- */ diff --git a/modules.json b/modules.json index f3f58353..1fa2f0b1 100644 --- a/modules.json +++ b/modules.json @@ -3,14 +3,20 @@ "homePage": "https://github.com/nf-core/rnavar", "repos": { "nf-core/modules": { - "custom/dumpsoftwareversions": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "fastqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "multiqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_url": "https://github.com/nf-core/modules.git", + "modules": { + "custom/dumpsoftwareversions": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + }, + "fastqc": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + }, + "multiqc": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + } } } } diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py index d1390392..787bdb7b 100644 --- a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -import yaml import platform from textwrap import dedent +import yaml + def _make_versions_html(versions): html = [ @@ -58,11 +59,12 @@ def _make_versions_html(versions): for process, process_versions in versions_by_process.items(): module = process.split(":")[-1] try: - assert versions_by_module[module] == process_versions, ( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) except KeyError: versions_by_module[module] = process_versions diff --git a/nextflow.config b/nextflow.config index 7199e3d5..7655f1b4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -13,11 +13,11 @@ params { // Input options input = null + // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false - // MultiQC options multiqc_config = null multiqc_title = null @@ -37,6 +37,7 @@ params { schema_ignore_params = 'genomes' enable_conda = false + // Config options custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" @@ -45,6 +46,7 @@ params { config_profile_url = null config_profile_name = null + // Max resource options // Defaults only, expecting to be overwritten max_memory = '128.GB' @@ -72,6 +74,7 @@ try { // } + profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { @@ -82,6 +85,15 @@ profiles { shifter.enabled = false charliecloud.enabled = false } + mamba { + params.enable_conda = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } docker { docker.enabled = true docker.userEmulation = true @@ -119,10 +131,16 @@ profiles { podman.enabled = false shifter.enabled = false } + gitpod { + executor.name = 'local' + executor.cpus = 16 + executor.memory = 60.GB + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } + // Load igenomes.config if required if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' @@ -130,6 +148,7 @@ if (!params.igenomes_ignore) { params.genomes = [:] } + // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. // See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. @@ -169,7 +188,7 @@ manifest { description = 'GATK4 RNA variant calling pipeline' mainScript = 'main.nf' nextflowVersion = '!>=21.10.3' - version = '1.0dev' + version = '1.1.0dev' } // Load modules.config for DSL2 module specific options From c0049b9ce49e2fe8862789df8e5d498608106c11 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 1 Sep 2022 13:31:48 +0000 Subject: [PATCH 02/34] Template update for nf-core/tools version 2.5.1 --- bin/check_samplesheet.py | 9 ++++++--- pyproject.toml | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 pyproject.toml diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 9a8b8962..11b15572 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -98,7 +98,9 @@ def _validate_pair(self, row): """Assert that read pairs have the same file extension. Report pair status.""" if row[self._first_col] and row[self._second_col]: row[self._single_col] = False - if Path(row[self._first_col]).suffixes[-2:] != Path(row[self._second_col]).suffixes[-2:]: + first_col_suffix = Path(row[self._first_col]).suffixes[-2:] + second_col_suffix = Path(row[self._second_col]).suffixes[-2:] + if first_col_suffix != second_col_suffix: raise AssertionError("FASTQ pairs must have the same file extensions.") else: row[self._single_col] = True @@ -157,7 +159,7 @@ def sniff_format(handle): handle.seek(0) sniffer = csv.Sniffer() if not sniffer.has_header(peek): - logger.critical(f"The given sample sheet does not appear to contain a header.") + logger.critical("The given sample sheet does not appear to contain a header.") sys.exit(1) dialect = sniffer.sniff(peek) return dialect @@ -195,7 +197,8 @@ def check_samplesheet(file_in, file_out): reader = csv.DictReader(in_handle, dialect=sniff_format(in_handle)) # Validate the existence of the expected header columns. if not required_columns.issubset(reader.fieldnames): - logger.critical(f"The sample sheet **must** contain the column headers: {', '.join(required_columns)}.") + req_cols = ", ".join(required_columns) + logger.critical(f"The sample sheet **must** contain these column headers: {req_cols}.") sys.exit(1) # Validate each row. checker = RowChecker() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0d62beb6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. +[tool.black] +line-length = 120 +target_version = ["py37", "py38", "py39", "py310"] + +[tool.isort] +profile = "black" +known_first_party = ["nf_core"] +multi_line_output = 3 From dca413e39c4a01b4aa862c512c6110ed48e4db60 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 4 Oct 2022 22:04:09 +0000 Subject: [PATCH 03/34] Template update for nf-core/tools version 2.6 --- .github/workflows/awsfulltest.yml | 4 ++ .github/workflows/awstest.yml | 4 ++ .prettierignore | 1 + CITATION.cff | 8 +-- assets/adaptivecard.json | 67 +++++++++++++++++++ assets/methods_description_template.yml | 25 +++++++ assets/multiqc_config.yml | 6 +- docs/usage.md | 8 +++ lib/NfcoreTemplate.groovy | 55 +++++++++++++++ lib/Utils.groovy | 21 ++++-- lib/WorkflowRnavar.groovy | 19 ++++++ main.nf | 3 +- modules.json | 27 ++++---- .../custom/dumpsoftwareversions/main.nf | 8 +-- .../custom/dumpsoftwareversions/meta.yml | 0 .../templates/dumpsoftwareversions.py | 0 modules/nf-core/{modules => }/fastqc/main.nf | 12 ++++ modules/nf-core/{modules => }/fastqc/meta.yml | 0 modules/nf-core/modules/multiqc/main.nf | 31 --------- modules/nf-core/multiqc/main.nf | 53 +++++++++++++++ .../nf-core/{modules => }/multiqc/meta.yml | 15 +++++ nextflow.config | 5 +- nextflow_schema.json | 18 +++++ workflows/rnavar.nf | 26 ++++--- 24 files changed, 345 insertions(+), 71 deletions(-) create mode 100644 assets/adaptivecard.json create mode 100644 assets/methods_description_template.yml mode change 100755 => 100644 lib/Utils.groovy rename modules/nf-core/{modules => }/custom/dumpsoftwareversions/main.nf (79%) rename modules/nf-core/{modules => }/custom/dumpsoftwareversions/meta.yml (100%) rename modules/nf-core/{modules => }/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py (100%) rename modules/nf-core/{modules => }/fastqc/main.nf (85%) rename modules/nf-core/{modules => }/fastqc/meta.yml (100%) delete mode 100644 modules/nf-core/modules/multiqc/main.nf create mode 100644 modules/nf-core/multiqc/main.nf rename modules/nf-core/{modules => }/multiqc/meta.yml (73%) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 7b3d239a..50a7398e 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -28,3 +28,7 @@ jobs: "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnavar/results-${{ github.sha }}" } profiles: test_full,aws_tower + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index a05594d9..84a22495 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -23,3 +23,7 @@ jobs: "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnavar/results-test-${{ github.sha }}" } profiles: test,aws_tower + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log diff --git a/.prettierignore b/.prettierignore index d0e7ae58..eb74a574 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ email_template.html +adaptivecard.json .nextflow* work/ data/ diff --git a/CITATION.cff b/CITATION.cff index 4533e2f2..017666c0 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -13,8 +13,8 @@ authors: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen @@ -39,8 +39,8 @@ prefered-citation: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen diff --git a/assets/adaptivecard.json b/assets/adaptivecard.json new file mode 100644 index 00000000..2b1d36cb --- /dev/null +++ b/assets/adaptivecard.json @@ -0,0 +1,67 @@ +{ + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "contentUrl": null, + "content": { + "\$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "msteams": { + "width": "Full" + }, + "type": "AdaptiveCard", + "version": "1.2", + "body": [ + { + "type": "TextBlock", + "size": "Large", + "weight": "Bolder", + "color": "<% if (success) { %>Good<% } else { %>Attention<%} %>", + "text": "nf-core/rnavar v${version} - ${runName}", + "wrap": true + }, + { + "type": "TextBlock", + "spacing": "None", + "text": "Completed at ${dateComplete} (duration: ${duration})", + "isSubtle": true, + "wrap": true + }, + { + "type": "TextBlock", + "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors. The full error message was: ${errorReport}.<% } %>", + "wrap": true + }, + { + "type": "TextBlock", + "text": "The command used to launch the workflow was as follows:", + "wrap": true + }, + { + "type": "TextBlock", + "text": "${commandLine}", + "isSubtle": true, + "wrap": true + } + ], + "actions": [ + { + "type": "Action.ShowCard", + "title": "Pipeline Configuration", + "card": { + "type": "AdaptiveCard", + "\$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "body": [ + { + "type": "FactSet", + "facts": [<% out << summary.collect{ k,v -> "{\"title\": \"$k\", \"value\" : \"$v\"}"}.join(",\n") %> + ] + } + ] + } + } + ] + } + } + ] +} diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml new file mode 100644 index 00000000..f2bc183b --- /dev/null +++ b/assets/methods_description_template.yml @@ -0,0 +1,25 @@ +id: "nf-core-rnavar-methods-description" +description: "Suggested text and references to use when describing pipeline usage within the methods section of a publication." +section_name: "nf-core/rnavar Methods Description" +section_href: "https://github.com/nf-core/rnavar" +plot_type: "html" +## TODO nf-core: Update the HTML below to your prefered methods description, e.g. add publication citation for this pipeline +## You inject any metadata in the Nextflow '${workflow}' object +data: | +

Methods

+

Data was processed using nf-core/rnavar v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (Ewels et al., 2020).

+

The pipeline was executed with Nextflow v${workflow.nextflow.version} (Di Tommaso et al., 2017) with the following command:

+ 
${workflow.commandLine}
+

References

+
    +
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. https://doi.org/10.1038/nbt.3820
    • +
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. https://doi.org/10.1038/s41587-020-0439-x
    • +
+
+
Notes:
+
    + ${nodoi_text} +
  • The command above does not include parameters contained in any configs or profiles that may have been used. Ensure the config file is also uploaded with your publication!
    • +
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
    • +
+
diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index f3b0ac1d..2689c2b9 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -3,9 +3,11 @@ report_comment: > analysis pipeline. For information about how to interpret these results, please see the documentation. report_section_order: - software_versions: + "nf-core-rnavar-methods-description": order: -1000 - "nf-core-rnavar-summary": + software_versions: order: -1001 + "nf-core-rnavar-summary": + order: -1002 export_plots: true diff --git a/docs/usage.md b/docs/usage.md index 3878607a..db4e6f4d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -237,6 +237,14 @@ See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). +## Azure Resource Requests + +To be used with the `azurebatch` profile by specifying the `-profile azurebatch`. +We recommend providing a compute `params.vm_type` of `Standard_D16_v3` VMs by default but these options can be changed if required. + +Note that the choice of VM size depends on your quota and the overall workload during the analysis. +For a thorough list, please refer the [Azure Sizes for virtual machines in Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes). + ## Running in the background Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy index 2fc0a9b9..27feb009 100755 --- a/lib/NfcoreTemplate.groovy +++ b/lib/NfcoreTemplate.groovy @@ -145,6 +145,61 @@ class NfcoreTemplate { output_tf.withWriter { w -> w << email_txt } } + // + // Construct and send adaptive card + // https://adaptivecards.io + // + public static void adaptivecard(workflow, params, summary_params, projectDir, log) { + def hook_url = params.hook_url + + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) misc_fields['repository'] = workflow.repository + if (workflow.commitId) misc_fields['commitid'] = workflow.commitId + if (workflow.revision) misc_fields['revision'] = workflow.revision + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = workflow.manifest.version + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + def hf = new File("$projectDir/assets/adaptivecard.json") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection(); + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")); + def postRC = post.getResponseCode(); + if (! postRC.equals(200)) { + log.warn(post.getErrorStream().getText()); + } + } + // // Print pipeline summary on completion // diff --git a/lib/Utils.groovy b/lib/Utils.groovy old mode 100755 new mode 100644 index 28567bd7..8d030f4e --- a/lib/Utils.groovy +++ b/lib/Utils.groovy @@ -21,19 +21,26 @@ class Utils { } // Check that all channels are present - def required_channels = ['conda-forge', 'bioconda', 'defaults'] - def conda_check_failed = !required_channels.every { ch -> ch in channels } + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order - conda_check_failed |= !(channels.indexOf('conda-forge') < channels.indexOf('bioconda')) - conda_check_failed |= !(channels.indexOf('bioconda') < channels.indexOf('defaults')) + def channel_priority_violation = false + def n = required_channels_in_order.size() + for (int i = 0; i < n - 1; i++) { + channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + } - if (conda_check_failed) { + if (channels_missing | channel_priority_violation) { log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " There is a problem with your Conda configuration!\n\n" + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/user/install.html#set-up-channels\n" + - " NB: The order of the channels matters!\n" + + " Please refer to https://bioconda.github.io/\n" + + " The observed channel order is \n" + + " ${channels}\n" + + " but the following channel order is required:\n" + + " ${required_channels_in_order}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" } } diff --git a/lib/WorkflowRnavar.groovy b/lib/WorkflowRnavar.groovy index 162b1db3..6f2ab32d 100755 --- a/lib/WorkflowRnavar.groovy +++ b/lib/WorkflowRnavar.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the workflow/rnavar.nf in the nf-core/rnavar pipeline // +import groovy.text.SimpleTemplateEngine + class WorkflowRnavar { // @@ -42,6 +44,23 @@ class WorkflowRnavar { yaml_file_text += "data: |\n" yaml_file_text += "${summary_section}" return yaml_file_text + } + + public static String methodsDescriptionText(run_workflow, mqc_methods_yaml) { + // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file + def meta = [:] + meta.workflow = run_workflow.toMap() + meta["manifest_map"] = run_workflow.manifest.toMap() + + meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" + meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + + def methods_text = mqc_methods_yaml.text + + def engine = new SimpleTemplateEngine() + def description_html = engine.createTemplate(methods_text).make(meta) + + return description_html }// // Exit pipeline if incorrect --genome key provided // diff --git a/main.nf b/main.nf index 22dbf6ca..bd3b64d2 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,8 @@ nf-core/rnavar ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/rnavar -Website: https://nf-co.re/rnavar + + Website: https://nf-co.re/rnavar Slack : https://nfcore.slack.com/channels/rnavar ---------------------------------------------------------------------------------------- */ diff --git a/modules.json b/modules.json index 1fa2f0b1..b4e3abcc 100644 --- a/modules.json +++ b/modules.json @@ -2,20 +2,21 @@ "name": "nf-core/rnavar", "homePage": "https://github.com/nf-core/rnavar", "repos": { - "nf-core/modules": { - "git_url": "https://github.com/nf-core/modules.git", + "https://github.com/nf-core/modules.git": { "modules": { - "custom/dumpsoftwareversions": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" - }, - "fastqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" - }, - "multiqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" + "nf-core": { + "custom/dumpsoftwareversions": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "fastqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "multiqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + } } } } diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf similarity index 79% rename from modules/nf-core/modules/custom/dumpsoftwareversions/main.nf rename to modules/nf-core/custom/dumpsoftwareversions/main.nf index 327d5100..cebb6e05 100644 --- a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -1,11 +1,11 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_low' + label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? "bioconda::multiqc=1.11" : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.11--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml similarity index 100% rename from modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml rename to modules/nf-core/custom/dumpsoftwareversions/meta.yml diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py similarity index 100% rename from modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py rename to modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/modules/nf-core/modules/fastqc/main.nf b/modules/nf-core/fastqc/main.nf similarity index 85% rename from modules/nf-core/modules/fastqc/main.nf rename to modules/nf-core/fastqc/main.nf index ed6b8c50..05730368 100644 --- a/modules/nf-core/modules/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -44,4 +44,16 @@ process FASTQC { END_VERSIONS """ } + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.html + touch ${prefix}.zip + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ } diff --git a/modules/nf-core/modules/fastqc/meta.yml b/modules/nf-core/fastqc/meta.yml similarity index 100% rename from modules/nf-core/modules/fastqc/meta.yml rename to modules/nf-core/fastqc/meta.yml diff --git a/modules/nf-core/modules/multiqc/main.nf b/modules/nf-core/modules/multiqc/main.nf deleted file mode 100644 index 1264aac1..00000000 --- a/modules/nf-core/modules/multiqc/main.nf +++ /dev/null @@ -1,31 +0,0 @@ -process MULTIQC { - label 'process_medium' - - conda (params.enable_conda ? 'bioconda::multiqc=1.12' : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.12--pyhdfd78af_0' }" - - input: - path multiqc_files - - output: - path "*multiqc_report.html", emit: report - path "*_data" , emit: data - path "*_plots" , optional:true, emit: plots - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - """ - multiqc -f $args . - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ -} diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf new file mode 100644 index 00000000..a8159a57 --- /dev/null +++ b/modules/nf-core/multiqc/main.nf @@ -0,0 +1,53 @@ +process MULTIQC { + label 'process_single' + + conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + + input: + path multiqc_files, stageAs: "?/*" + path(multiqc_config) + path(extra_multiqc_config) + path(multiqc_logo) + + output: + path "*multiqc_report.html", emit: report + path "*_data" , emit: data + path "*_plots" , optional:true, emit: plots + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def config = multiqc_config ? "--config $multiqc_config" : '' + def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + """ + multiqc \\ + --force \\ + $args \\ + $config \\ + $extra_config \\ + . + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS + """ + + stub: + """ + touch multiqc_data + touch multiqc_plots + touch multiqc_report.html + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml similarity index 73% rename from modules/nf-core/modules/multiqc/meta.yml rename to modules/nf-core/multiqc/meta.yml index 6fa891ef..ebc29b27 100644 --- a/modules/nf-core/modules/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -12,11 +12,25 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] + input: - multiqc_files: type: file description: | List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - multiqc_config: + type: file + description: Optional config yml for MultiQC + pattern: "*.{yml,yaml}" + - extra_multiqc_config: + type: file + description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. + pattern: "*.{yml,yaml}" + - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" + output: - report: type: file @@ -38,3 +52,4 @@ authors: - "@abhi18av" - "@bunop" - "@drpatelh" + - "@jfy133" diff --git a/nextflow.config b/nextflow.config index 7655f1b4..c80668dd 100644 --- a/nextflow.config +++ b/nextflow.config @@ -21,7 +21,9 @@ params { // MultiQC options multiqc_config = null multiqc_title = null + multiqc_logo = null max_multiqc_email_size = '25.MB' + multiqc_methods_description = null // Boilerplate options outdir = null @@ -31,6 +33,7 @@ params { email_on_fail = null plaintext_email = false monochrome_logs = false + hook_url = null help = false validate_params = true show_hidden_params = false @@ -74,7 +77,6 @@ try { // } - profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { @@ -189,6 +191,7 @@ manifest { mainScript = 'main.nf' nextflowVersion = '!>=21.10.3' version = '1.1.0dev' + doi = '' } // Load modules.config for DSL2 module specific options diff --git a/nextflow_schema.json b/nextflow_schema.json index 62e73f72..51870fa6 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -213,12 +213,30 @@ "fa_icon": "fas fa-palette", "hidden": true }, + "hook_url": { + "type": "string", + "description": "Incoming hook URL for messaging service", + "fa_icon": "fas fa-people-group", + "help_text": "Incoming hook URL for messaging service. Currently, only MS Teams is supported.", + "hidden": true + }, "multiqc_config": { "type": "string", "description": "Custom config file to supply to MultiQC.", "fa_icon": "fas fa-cog", "hidden": true }, + "multiqc_logo": { + "type": "string", + "description": "Custom logo file to supply to MultiQC. File name must also be set in the MultiQC config file", + "fa_icon": "fas fa-image", + "hidden": true + }, + "multiqc_methods_description": { + "type": "string", + "description": "Custom MultiQC yaml file containing HTML including a methods description.", + "fa_icon": "fas fa-cog" + }, "tracedir": { "type": "string", "description": "Directory to keep pipeline Nextflow logs and reports.", diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 4132f35a..1042cc76 100644 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -23,8 +23,10 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() +ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) +ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() +ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() +ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -46,9 +48,9 @@ include { INPUT_CHECK } from '../subworkflows/local/input_check' // // MODULE: Installed directly from nf-core/modules // -include { FASTQC } from '../modules/nf-core/modules/fastqc/main' -include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' +include { FASTQC } from '../modules/nf-core/fastqc/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -89,15 +91,20 @@ workflow RNAVAR { workflow_summary = WorkflowRnavar.paramsSummaryMultiqc(workflow, summary_params) ch_workflow_summary = Channel.value(workflow_summary) + methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) + ch_methods_description = Channel.value(methods_description) + ch_multiqc_files = Channel.empty() - ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) MULTIQC ( - ch_multiqc_files.collect() + ch_multiqc_files.collect(), + ch_multiqc_config.collect().ifEmpty([]), + ch_multiqc_custom_config.collect().ifEmpty([]), + ch_multiqc_logo.collect().ifEmpty([]) ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) @@ -114,6 +121,9 @@ workflow.onComplete { NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) } NfcoreTemplate.summary(workflow, params, log) + if (params.hook_url) { + NfcoreTemplate.adaptivecard(workflow, params, summary_params, projectDir, log) + } } /* From 681ca7e527fcc9cbc1f48d16b7887a6b63a5eea6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 8 Dec 2022 13:14:26 +0000 Subject: [PATCH 04/34] Template update for nf-core/tools version 2.7.1 --- .devcontainer/devcontainer.json | 27 +++++++++++++ .gitattributes | 1 + .github/CONTRIBUTING.md | 16 ++++++++ .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/ci.yml | 8 +++- .github/workflows/fix-linting.yml | 6 +-- .github/workflows/linting.yml | 18 +++++---- .github/workflows/linting_comment.yml | 2 +- .prettierignore | 2 + CITATION.cff | 56 --------------------------- README.md | 4 +- assets/slackreport.json | 34 ++++++++++++++++ docs/usage.md | 24 +++++++----- lib/NfcoreSchema.groovy | 1 - lib/NfcoreTemplate.groovy | 41 +++++++++++++++----- lib/WorkflowMain.groovy | 18 ++++++--- modules.json | 9 +++-- modules/local/samplesheet_check.nf | 4 ++ nextflow.config | 12 ++++-- nextflow_schema.json | 8 +++- workflows/rnavar.nf | 11 +++--- 21 files changed, 193 insertions(+), 111 deletions(-) create mode 100644 .devcontainer/devcontainer.json delete mode 100644 CITATION.cff create mode 100644 assets/slackreport.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..ea27a584 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,27 @@ +{ + "name": "nfcore", + "image": "nfcore/gitpod:latest", + "remoteUser": "gitpod", + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/opt/conda/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", + "python.formatting.yapfPath": "/opt/conda/bin/yapf", + "python.linting.flake8Path": "/opt/conda/bin/flake8", + "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", + "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", + "python.linting.pylintPath": "/opt/conda/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"] + } + } +} diff --git a/.gitattributes b/.gitattributes index 050bb120..7a2dabc2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ *.config linguist-language=nextflow +*.nf.test linguist-language=nextflow modules/nf-core/** linguist-generated subworkflows/nf-core/** linguist-generated diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 7a76df36..dae092ad 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -101,3 +101,19 @@ If you are using a new feature from core Nextflow, you may bump the minimum requ ### Images and figures For overview images and other documents we follow the nf-core [style guidelines and examples](https://nf-co.re/developers/design_guidelines). + +## GitHub Codespaces + +This repo includes a devcontainer configuration which will create a GitHub Codespaces for Nextflow development! This is an online developer environment that runs in your browser, complete with VSCode and a terminal. + +To get started: + +- Open the repo in [Codespaces](https://github.com/nf-core/rnavar/codespaces) +- Tools installed + - nf-core + - Nextflow + +Devcontainer specs: + +- [DevContainer config](.devcontainer/devcontainer.json) +- [Dockerfile](.devcontainer/Dockerfile) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index fb5c200e..212481be 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -42,7 +42,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 21.10.3)_ + * Nextflow version _(eg. 22.10.1)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67225b7a..9a080ef2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,10 @@ on: env: NXF_ANSI_LOG: false +concurrency: + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" + cancel-in-progress: true + jobs: test: name: Run pipeline with test data @@ -20,11 +24,11 @@ jobs: strategy: matrix: NXF_VER: - - "21.10.3" + - "22.10.1" - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index e67cdc60..34ea8bd5 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -24,7 +24,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install Prettier run: npm install -g prettier @prettier/plugin-php @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "::set-output name=result::pass" + echo "name=result::pass" >> $GITHUB_OUTPUT else - echo "::set-output name=result::fail" + echo "name=result::fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 8a5ce69b..858d622e 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -4,6 +4,8 @@ name: nf-core linting # that the code meets the nf-core guidelines. on: push: + branches: + - dev pull_request: release: types: [published] @@ -12,9 +14,9 @@ jobs: EditorConfig: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -25,9 +27,9 @@ jobs: Prettier: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install Prettier run: npm install -g prettier @@ -38,7 +40,7 @@ jobs: PythonBlack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Check code lints with Black uses: psf/black@stable @@ -69,12 +71,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v4 with: python-version: "3.7" architecture: "x64" @@ -97,7 +99,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 04758f61..39635186 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "::set-output name=pr_number::$(cat linting-logs/PR_number.txt)" + run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 diff --git a/.prettierignore b/.prettierignore index eb74a574..437d763d 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,6 @@ email_template.html adaptivecard.json +slackreport.json .nextflow* work/ data/ @@ -8,3 +9,4 @@ results/ testing/ testing* *.pyc +bin/ diff --git a/CITATION.cff b/CITATION.cff deleted file mode 100644 index 017666c0..00000000 --- a/CITATION.cff +++ /dev/null @@ -1,56 +0,0 @@ -cff-version: 1.2.0 -message: "If you use `nf-core tools` in your work, please cite the `nf-core` publication" -authors: - - family-names: Ewels - given-names: Philip - - family-names: Peltzer - given-names: Alexander - - family-names: Fillinger - given-names: Sven - - family-names: Patel - given-names: Harshil - - family-names: Alneberg - given-names: Johannes - - family-names: Wilm - given-names: Andreas - - family-names: Garcia - given-names: Maxime Ulysse - - family-names: Di Tommaso - given-names: Paolo - - family-names: Nahnsen - given-names: Sven -title: "The nf-core framework for community-curated bioinformatics pipelines." -version: 2.4.1 -doi: 10.1038/s41587-020-0439-x -date-released: 2022-05-16 -url: https://github.com/nf-core/tools -prefered-citation: - type: article - authors: - - family-names: Ewels - given-names: Philip - - family-names: Peltzer - given-names: Alexander - - family-names: Fillinger - given-names: Sven - - family-names: Patel - given-names: Harshil - - family-names: Alneberg - given-names: Johannes - - family-names: Wilm - given-names: Andreas - - family-names: Garcia - given-names: Maxime Ulysse - - family-names: Di Tommaso - given-names: Paolo - - family-names: Nahnsen - given-names: Sven - doi: 10.1038/s41587-020-0439-x - journal: nature biotechnology - start: 276 - end: 278 - title: "The nf-core framework for community-curated bioinformatics pipelines." - issue: 3 - volume: 38 - year: 2020 - url: https://dx.doi.org/10.1038/s41587-020-0439-x diff --git a/README.md b/README.md index d80e7ac0..d4fab95f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/rnavar/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) @@ -31,7 +31,7 @@ On release, automated continuous integration tests run the pipeline on a full-si ## Quick Start -1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.10.3`) +1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`) 2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_. diff --git a/assets/slackreport.json b/assets/slackreport.json new file mode 100644 index 00000000..043d02f2 --- /dev/null +++ b/assets/slackreport.json @@ -0,0 +1,34 @@ +{ + "attachments": [ + { + "fallback": "Plain-text summary of the attachment.", + "color": "<% if (success) { %>good<% } else { %>danger<%} %>", + "author_name": "sanger-tol/readmapping v${version} - ${runName}", + "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", + "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", + "fields": [ + { + "title": "Command used to launch the workflow", + "value": "```${commandLine}```", + "short": false + } + <% + if (!success) { %> + , + { + "title": "Full error message", + "value": "```${errorReport}```", + "short": false + }, + { + "title": "Pipeline configuration", + "value": "<% out << summary.collect{ k,v -> k == "hook_url" ? "_${k}_: (_hidden_)" : ( ( v.class.toString().contains('Path') || ( v.class.toString().contains('String') && v.contains('/') ) ) ? "_${k}_: `${v}`" : (v.class.toString().contains('DateTime') ? ("_${k}_: " + v.format(java.time.format.DateTimeFormatter.ofLocalizedDateTime(java.time.format.FormatStyle.MEDIUM))) : "_${k}_: ${v}") ) }.join(",\n") %>", + "short": false + } + <% } + %> + ], + "footer": "Completed at <% out << dateComplete.format(java.time.format.DateTimeFormatter.ofLocalizedDateTime(java.time.format.FormatStyle.MEDIUM)) %> (duration: ${duration})" + } + ] +} diff --git a/docs/usage.md b/docs/usage.md index db4e6f4d..8f140452 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -83,9 +83,9 @@ nextflow pull nf-core/rnavar It is a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. -First, go to the [nf-core/rnavar releases page](https://github.com/nf-core/rnavar/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. +First, go to the [nf-core/rnavar releases page](https://github.com/nf-core/rnavar/releases) and find the latest pipeline version - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. -This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. +This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. ## Core Nextflow arguments @@ -95,7 +95,7 @@ This version number will be logged in reports when you run the pipeline, so that Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. When using Biocontainers, most of these software packaging methods pull Docker containers from quay.io e.g [FastQC](https://quay.io/repository/biocontainers/fastqc) except for Singularity which directly downloads Singularity images via https hosted by the [Galaxy project](https://depot.galaxyproject.org/singularity/) and Conda which downloads and installs software locally from [Bioconda](https://bioconda.github.io/). +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -104,8 +104,11 @@ The pipeline also dynamically loads configurations from [https://github.com/nf-c Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. -If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended. +If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended, since it can lead to different results on different machines dependent on the computer enviroment. +- `test` + - A profile with a complete configuration for automated testing + - Includes links to test data so needs no other parameters - `docker` - A generic configuration profile to be used with [Docker](https://docker.com/) - `singularity` @@ -118,9 +121,6 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. -- `test` - - A profile with a complete configuration for automated testing - - Includes links to test data so needs no other parameters ### `-resume` @@ -169,8 +169,14 @@ Work dir: Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` ``` +#### For beginners + +A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. + +#### Advanced option on process level + To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). -We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/software/star/align/main.nf`. +We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/star/align/main.nf`. If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. @@ -189,7 +195,7 @@ process { > > If you get a warning suggesting that the process selector isn't recognised check that the process name has been specified correctly. -### Updating containers +### Updating containers (advanced users) The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. If for some reason you need to use a different version of a particular tool with the pipeline then you just need to identify the `process` name and override the Nextflow `container` definition for that process using the `withName` declaration. For example, in the [nf-core/viralrecon](https://nf-co.re/viralrecon) pipeline a tool called [Pangolin](https://github.com/cov-lineages/pangolin) has been used during the COVID-19 pandemic to assign lineages to SARS-CoV-2 genome sequenced samples. Given that the lineage assignments change quite frequently it doesn't make sense to re-release the nf-core/viralrecon everytime a new version of Pangolin has been released. However, you can override the default container used by the pipeline by creating a custom config file and passing it as a command-line argument via `-c custom.config`. diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index b3d092f8..33cd4f6e 100755 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -46,7 +46,6 @@ class NfcoreSchema { 'quiet', 'syslog', 'v', - 'version', // Options for `nextflow run` command 'ansi', diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy index 27feb009..25a0a74a 100755 --- a/lib/NfcoreTemplate.groovy +++ b/lib/NfcoreTemplate.groovy @@ -32,6 +32,25 @@ class NfcoreTemplate { } } + // + // Generate version string + // + public static String version(workflow) { + String version_string = "" + + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string + } + // // Construct and send completion email // @@ -61,7 +80,7 @@ class NfcoreTemplate { misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp def email_fields = [:] - email_fields['version'] = workflow.manifest.version + email_fields['version'] = NfcoreTemplate.version(workflow) email_fields['runName'] = workflow.runName email_fields['success'] = workflow.success email_fields['dateComplete'] = workflow.complete @@ -146,10 +165,10 @@ class NfcoreTemplate { } // - // Construct and send adaptive card - // https://adaptivecards.io + // Construct and send a notification to a web server as JSON + // e.g. Microsoft Teams and Slack // - public static void adaptivecard(workflow, params, summary_params, projectDir, log) { + public static void IM_notification(workflow, params, summary_params, projectDir, log) { def hook_url = params.hook_url def summary = [:] @@ -170,7 +189,7 @@ class NfcoreTemplate { misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp def msg_fields = [:] - msg_fields['version'] = workflow.manifest.version + msg_fields['version'] = NfcoreTemplate.version(workflow) msg_fields['runName'] = workflow.runName msg_fields['success'] = workflow.success msg_fields['dateComplete'] = workflow.complete @@ -178,13 +197,16 @@ class NfcoreTemplate { msg_fields['exitStatus'] = workflow.exitStatus msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine + msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") msg_fields['projectDir'] = workflow.projectDir msg_fields['summary'] = summary << misc_fields // Render the JSON template def engine = new groovy.text.GStringTemplateEngine() - def hf = new File("$projectDir/assets/adaptivecard.json") + // Different JSON depending on the service provider + // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format + def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" + def hf = new File("$projectDir/assets/${json_path}") def json_template = engine.createTemplate(hf).make(msg_fields) def json_message = json_template.toString() @@ -209,7 +231,7 @@ class NfcoreTemplate { if (workflow.stats.ignoredCount == 0) { log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" + log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" } } else { log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" @@ -297,6 +319,7 @@ class NfcoreTemplate { // public static String logo(workflow, monochrome_logs) { Map colors = logColours(monochrome_logs) + String workflow_version = NfcoreTemplate.version(workflow) String.format( """\n ${dashedLine(monochrome_logs)} @@ -305,7 +328,7 @@ class NfcoreTemplate { ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} v${workflow.manifest.version}${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${workflow_version}${colors.reset} ${dashedLine(monochrome_logs)} """.stripIndent() ) diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 35bf164f..8309481f 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -19,7 +19,7 @@ class WorkflowMain { } // - // Print help to screen if required + // Generate help string // public static String help(workflow, params, log) { def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" @@ -32,7 +32,7 @@ class WorkflowMain { } // - // Print parameter summary log to screen + // Generate parameter summary log string // public static String paramsSummaryLog(workflow, params, log) { def summary_log = '' @@ -53,15 +53,21 @@ class WorkflowMain { System.exit(0) } - // Validate workflow parameters via the JSON schema - if (params.validate_params) { - NfcoreSchema.validateParameters(workflow, params, log) + // Print workflow version and exit on --version + if (params.version) { + String workflow_version = NfcoreTemplate.version(workflow) + log.info "${workflow.manifest.name} ${workflow_version}" + System.exit(0) } // Print parameter summary log to screen - log.info paramsSummaryLog(workflow, params, log) + // Validate workflow parameters via the JSON schema + if (params.validate_params) { + NfcoreSchema.validateParameters(workflow, params, log) + } + // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) diff --git a/modules.json b/modules.json index b4e3abcc..48ebc0a1 100644 --- a/modules.json +++ b/modules.json @@ -7,15 +7,18 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed_by": ["modules"] } } } diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf index 553da8c9..08f6035c 100644 --- a/modules/local/samplesheet_check.nf +++ b/modules/local/samplesheet_check.nf @@ -1,5 +1,6 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" + label 'process_single' conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -13,6 +14,9 @@ process SAMPLESHEET_CHECK { path '*.csv' , emit: csv path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: // This script is bundled with the pipeline, in nf-core/rnavar/bin/ """ check_samplesheet.py \\ diff --git a/nextflow.config b/nextflow.config index c80668dd..bbdc1216 100644 --- a/nextflow.config +++ b/nextflow.config @@ -35,6 +35,7 @@ params { monochrome_logs = false hook_url = null help = false + version = false validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' @@ -81,6 +82,7 @@ profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { params.enable_conda = true + conda.enabled = true docker.enabled = false singularity.enabled = false podman.enabled = false @@ -89,6 +91,7 @@ profiles { } mamba { params.enable_conda = true + conda.enabled = true conda.useMamba = true docker.enabled = false singularity.enabled = false @@ -104,6 +107,9 @@ profiles { shifter.enabled = false charliecloud.enabled = false } + arm { + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + } singularity { singularity.enabled = true singularity.autoMounts = true @@ -185,11 +191,11 @@ dag { manifest { name = 'nf-core/rnavar' - author = '@praveenraj2018' + author = """@praveenraj2018""" homePage = 'https://github.com/nf-core/rnavar' - description = 'GATK4 RNA variant calling pipeline' + description = """GATK4 RNA variant calling pipeline""" mainScript = 'main.nf' - nextflowVersion = '!>=21.10.3' + nextflowVersion = '!>=22.10.1' version = '1.1.0dev' doi = '' } diff --git a/nextflow_schema.json b/nextflow_schema.json index 51870fa6..0e344fd0 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -176,6 +176,12 @@ "fa_icon": "fas fa-question-circle", "hidden": true }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, "publish_dir_mode": { "type": "string", "default": "copy", @@ -217,7 +223,7 @@ "type": "string", "description": "Incoming hook URL for messaging service", "fa_icon": "fas fa-people-group", - "help_text": "Incoming hook URL for messaging service. Currently, only MS Teams is supported.", + "help_text": "Incoming hook URL for messaging service. Currently, MS Teams and Slack are supported.", "hidden": true }, "multiqc_config": { diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 1042cc76..146f99b0 100644 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -82,7 +82,7 @@ workflow RNAVAR { ch_versions = ch_versions.mix(FASTQC.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') + ch_versions.unique{ it.text }.collectFile(name: 'collated_versions.yml') ) // @@ -102,12 +102,11 @@ workflow RNAVAR { MULTIQC ( ch_multiqc_files.collect(), - ch_multiqc_config.collect().ifEmpty([]), - ch_multiqc_custom_config.collect().ifEmpty([]), - ch_multiqc_logo.collect().ifEmpty([]) + ch_multiqc_config.toList(), + ch_multiqc_custom_config.toList(), + ch_multiqc_logo.toList() ) multiqc_report = MULTIQC.out.report.toList() - ch_versions = ch_versions.mix(MULTIQC.out.versions) } /* @@ -122,7 +121,7 @@ workflow.onComplete { } NfcoreTemplate.summary(workflow, params, log) if (params.hook_url) { - NfcoreTemplate.adaptivecard(workflow, params, summary_params, projectDir, log) + NfcoreTemplate.IM_notification(workflow, params, summary_params, projectDir, log) } } From e5294c1454e8d470b5eff3fe39c489cf60c06053 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 19 Dec 2022 12:08:15 +0000 Subject: [PATCH 05/34] Template update for nf-core/tools version 2.7.2 --- .github/workflows/fix-linting.yml | 4 +- .github/workflows/linting_comment.yml | 2 +- lib/WorkflowMain.groovy | 2 +- modules.json | 6 +- modules/local/samplesheet_check.nf | 2 +- .../custom/dumpsoftwareversions/main.nf | 2 +- .../templates/dumpsoftwareversions.py | 99 ++++++++++--------- modules/nf-core/fastqc/main.nf | 40 +++----- modules/nf-core/multiqc/main.nf | 2 +- nextflow.config | 3 - nextflow_schema.json | 6 -- workflows/rnavar.nf | 2 +- 12 files changed, 82 insertions(+), 88 deletions(-) mode change 100644 => 100755 modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 34ea8bd5..aedaa933 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "name=result::pass" >> $GITHUB_OUTPUT + echo "result=pass" >> $GITHUB_OUTPUT else - echo "name=result::fail" >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 39635186..0bbcd30f 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT + run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 8309481f..977ba948 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -72,7 +72,7 @@ class WorkflowMain { NfcoreTemplate.checkConfigProvided(workflow, log) // Check that conda channels are set-up correctly - if (params.enable_conda) { + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { Utils.checkCondaChannels(log) } diff --git a/modules.json b/modules.json index 48ebc0a1..02df1b22 100644 --- a/modules.json +++ b/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] } } diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf index 08f6035c..95d6727a 100644 --- a/modules/local/samplesheet_check.nf +++ b/modules/local/samplesheet_check.nf @@ -2,7 +2,7 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" label 'process_single' - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : 'quay.io/biocontainers/python:3.8.3' }" diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index cebb6e05..3df21765 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,7 +2,7 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100644 new mode 100755 index 787bdb7b..e55b8d43 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,5 +1,9 @@ #!/usr/bin/env python + +"""Provide functions to merge multiple versions.yml files.""" + + import platform from textwrap import dedent @@ -7,6 +11,7 @@ def _make_versions_html(versions): + """Generate a tabular HTML output of all versions for MultiQC.""" html = [ dedent( """\\ @@ -45,47 +50,53 @@ def _make_versions_html(versions): return "\\n".join(html) -versions_this_module = {} -versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, -} - -with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - -# aggregate versions by the module name (derived from fully-qualified process name) -versions_by_module = {} -for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - -versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", -} - -versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), -} - -with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) -with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - -with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) +def main(): + """Load all version files and generate merged output.""" + versions_this_module = {} + versions_this_module["${task.process}"] = { + "python": platform.python_version(), + "yaml": yaml.__version__, + } + + with open("$versions") as f: + versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + + # aggregate versions by the module name (derived from fully-qualified process name) + versions_by_module = {} + for process, process_versions in versions_by_process.items(): + module = process.split(":")[-1] + try: + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) + except KeyError: + versions_by_module[module] = process_versions + + versions_by_module["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version", + } + + versions_mqc = { + "id": "software_versions", + "section_name": "${workflow.manifest.name} Software Versions", + "section_href": "https://github.com/${workflow.manifest.name}", + "plot_type": "html", + "description": "are collected at run time from the software output.", + "data": _make_versions_html(versions_by_module), + } + + with open("software_versions.yml", "w") as f: + yaml.dump(versions_by_module, f, default_flow_style=False) + with open("software_versions_mqc.yml", "w") as f: + yaml.dump(versions_mqc, f, default_flow_style=False) + + with open("versions.yml", "w") as f: + yaml.dump(versions_this_module, f, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 05730368..9ae58381 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -2,7 +2,7 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : 'quay.io/biocontainers/fastqc:0.11.9--0' }" @@ -20,30 +20,22 @@ process FASTQC { script: def args = task.ext.args ?: '' - // Add soft-links to original FastQs for consistent naming in pipeline def prefix = task.ext.prefix ?: "${meta.id}" - if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $args --threads $task.cpus ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } + // Make list of old name and new name pairs to use for renaming in the bash while loop + def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } + def rename_to = old_new_pairs*.join(' ').join(' ') + def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + """ + printf "%s %s\\n" $rename_to | while read old_name new_name; do + [ -f "\${new_name}" ] || ln -s \$old_name \$new_name + done + fastqc $args --threads $task.cpus $renamed_files + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ stub: def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index a8159a57..68f66bea 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,7 +1,7 @@ process MULTIQC { label 'process_single' - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/nextflow.config b/nextflow.config index bbdc1216..f4a47634 100644 --- a/nextflow.config +++ b/nextflow.config @@ -39,7 +39,6 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - enable_conda = false // Config options @@ -81,7 +80,6 @@ try { profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { - params.enable_conda = true conda.enabled = true docker.enabled = false singularity.enabled = false @@ -90,7 +88,6 @@ profiles { charliecloud.enabled = false } mamba { - params.enable_conda = true conda.enabled = true conda.useMamba = true docker.enabled = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 0e344fd0..4395e63f 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -263,12 +263,6 @@ "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "enable_conda": { - "type": "boolean", - "description": "Run this workflow with Conda. You can also use '-profile conda' instead of providing this parameter.", - "hidden": true, - "fa_icon": "fas fa-bacon" } } } diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 146f99b0..6d42ac31 100644 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -82,7 +82,7 @@ workflow RNAVAR { ch_versions = ch_versions.mix(FASTQC.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique{ it.text }.collectFile(name: 'collated_versions.yml') + ch_versions.unique().collectFile(name: 'collated_versions.yml') ) // From feac5802049ff07b59b9390b66bd6488b1811240 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 28 Apr 2023 14:26:15 +0000 Subject: [PATCH 06/34] Template update for nf-core/tools version 2.8 --- .editorconfig | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .github/workflows/awsfulltest.yml | 2 +- .github/workflows/awstest.yml | 2 +- .github/workflows/branch.yml | 2 +- .github/workflows/clean-up.yml | 24 ++++ .github/workflows/linting.yml | 2 +- .pre-commit-config.yaml | 5 + README.md | 74 ++++++---- bin/check_samplesheet.py | 3 - conf/base.config | 2 +- conf/igenomes.config | 8 ++ conf/test_full.config | 2 + docs/usage.md | 130 +++++------------- lib/NfcoreSchema.groovy | 4 +- lib/WorkflowMain.groovy | 13 +- lib/WorkflowRnavar.groovy | 12 +- main.nf | 1 - modules.json | 4 +- modules/local/samplesheet_check.nf | 2 +- .../custom/dumpsoftwareversions/main.nf | 6 +- .../custom/dumpsoftwareversions/meta.yml | 2 + modules/nf-core/multiqc/main.nf | 6 +- modules/nf-core/multiqc/meta.yml | 3 +- nextflow.config | 29 +++- tower.yml | 5 + 27 files changed, 193 insertions(+), 157 deletions(-) create mode 100644 .github/workflows/clean-up.yml create mode 100644 .pre-commit-config.yaml create mode 100644 tower.yml diff --git a/.editorconfig b/.editorconfig index b78de6e6..b6b31907 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,7 +8,7 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js,cff}] +[*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 # These files are edited and tested upstream in nf-core/modules diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 212481be..c06c21f5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -45,6 +45,6 @@ body: * Nextflow version _(eg. 22.10.1)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ - * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_ + * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ * Version of nf-core/rnavar _(eg. 1.1, 1.5, 1.8.2)_ diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5c26fb9c..80d8e275 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,7 +15,8 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/rnav - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! -- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/rnavar/tree/master/.github/CONTRIBUTING.md)- [ ] If necessary, also make a PR on the nf-core/rnavar _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/rnavar/tree/master/.github/CONTRIBUTING.md) +- [ ] If necessary, also make a PR on the nf-core/rnavar _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 50a7398e..ed4f93c0 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index 84a22495..97d65891 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 377dde74..40e9c617 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -13,7 +13,7 @@ jobs: - name: Check PRs if: github.repository == 'nf-core/rnavar' run: | - { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/rnavar ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/rnavar ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] # If the above check failed, post a comment on the PR explaining the failure # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml new file mode 100644 index 00000000..694e90ec --- /dev/null +++ b/.github/workflows/clean-up.yml @@ -0,0 +1,24 @@ +name: "Close user-tagged issues and PRs" +on: + schedule: + - cron: "0 0 * * 0" # Once a week + +jobs: + clean-up: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v7 + with: + stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." + stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." + close-issue-message: "This issue was closed because it has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor and then staled for 20 days with no activity." + days-before-stale: 30 + days-before-close: 20 + days-before-pr-close: -1 + any-of-labels: "awaiting-changes,awaiting-feedback" + exempt-issue-labels: "WIP" + exempt-pr-labels: "WIP" + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 858d622e..888cb4bc 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.8" architecture: "x64" - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..0c31cdb9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.7.1" + hooks: + - id: prettier diff --git a/README.md b/README.md index d4fab95f..381dd4b4 100644 --- a/README.md +++ b/README.md @@ -8,57 +8,71 @@ [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) [![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/rnavar) -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23rnavar-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/rnavar)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) +[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23rnavar-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/rnavar)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction - +**nf-core/rnavar** is a bioinformatics pipeline that ... -**nf-core/rnavar** is a bioinformatics best-practice analysis pipeline for GATK4 RNA variant calling pipeline. - -The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It uses Docker/Singularity containers making installation trivial and results highly reproducible. The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. Where possible, these processes have been submitted to and installed from [nf-core/modules](https://github.com/nf-core/modules) in order to make them available to all nf-core pipelines, and to everyone within the Nextflow community! - - - -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources.The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/rnavar/results). - -## Pipeline summary + + 1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) 2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)) -## Quick Start +## Usage + +> **Note** +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how +> to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) +> with `-profile test` before running the workflow on actual data. + + - Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string. +Now, you can run the pipeline using: - > - The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`. - > - Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. - > - If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs. - > - If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs. + -4. Start running your own analysis! +```bash +nextflow run nf-core/rnavar \ + -profile \ + --input samplesheet.csv \ + --outdir +``` - +> **Warning:** +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those +> provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; +> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). - ```bash - nextflow run nf-core/rnavar --input samplesheet.csv --outdir --genome GRCh37 -profile - ``` +For more details, please refer to the [usage documentation](https://nf-co.re/rnavar/usage) and the [parameter documentation](https://nf-co.re/rnavar/parameters). -## Documentation +## Pipeline output -The nf-core/rnavar pipeline comes with documentation about the pipeline [usage](https://nf-co.re/rnavar/usage), [parameters](https://nf-co.re/rnavar/parameters) and [output](https://nf-co.re/rnavar/output). +To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/rnavar/results) tab on the nf-core website pipeline page. +For more details about the output files and reports, please refer to the +[output documentation](https://nf-co.re/rnavar/output). ## Credits diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 11b15572..4a758fe0 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -158,9 +158,6 @@ def sniff_format(handle): peek = read_head(handle) handle.seek(0) sniffer = csv.Sniffer() - if not sniffer.has_header(peek): - logger.critical("The given sample sheet does not appear to contain a header.") - sys.exit(1) dialect = sniffer.sniff(peek) return dialect diff --git a/conf/base.config b/conf/base.config index fcccdad9..89b498e5 100644 --- a/conf/base.config +++ b/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/conf/igenomes.config b/conf/igenomes.config index 7a1b3ac6..3f114377 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -36,6 +36,14 @@ params { macs_gsize = "2.7e9" blacklist = "${projectDir}/assets/blacklists/hg38-blacklist.bed" } + 'CHM13' { + fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAIndex/" + bwamem2 = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAmem2Index/" + gtf = "${params.igenomes_base}/Homo_sapiens/NCBI/CHM13/Annotation/Genes/genes.gtf" + gff = "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/009/914/755/GCF_009914755.1_T2T-CHM13v2.0/GCF_009914755.1_T2T-CHM13v2.0_genomic.gff.gz" + mito_name = "chrM" + } 'GRCm38' { fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/version0.6.0/" diff --git a/conf/test_full.config b/conf/test_full.config index 979f49f7..718622c4 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -10,6 +10,8 @@ ---------------------------------------------------------------------------------------- */ +cleanup = true + params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' diff --git a/docs/usage.md b/docs/usage.md index 8f140452..08da8b39 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -71,6 +71,29 @@ work # Directory containing the nextflow working files # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` +If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file. + +Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. + +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). +> The above pipeline run specified with a params file in yaml format: + +```bash +nextflow run nf-core/rnavar -profile docker -params-file params.yaml +``` + +with `params.yaml` containing: + +```yaml +input: './samplesheet.csv' +outdir: './results/' +genome: 'GRCh37' +input: 'data' +<...> +``` + +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). + ### Updating the pipeline When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: @@ -87,6 +110,10 @@ First, go to the [nf-core/rnavar releases page](https://github.com/nf-core/rnava This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. +To further assist in reproducbility, you can use share and re-use [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. + +> 💡 If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. + ## Core Nextflow arguments > **NB:** These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). @@ -95,7 +122,7 @@ This version number will be logged in reports when you run the pipeline, so that Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Apptainer, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -119,8 +146,10 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Shifter](https://nersc.gitlab.io/development/shifter/how-to-use/) - `charliecloud` - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) +- `apptainer` + - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `conda` - - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. + - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. ### `-resume` @@ -138,102 +167,19 @@ Specify the path to a specific config file (this is a core Nextflow command). Se Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher requests (2 x original, then 3 x original). If it still fails after the third attempt then the pipeline execution is stopped. -For example, if the nf-core/rnaseq pipeline is failing after multiple re-submissions of the `STAR_ALIGN` process due to an exit code of `137` this would indicate that there is an out of memory issue: - -```console -[62/149eb0] NOTE: Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) -- Execution is retried (1) -Error executing process > 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)' - -Caused by: - Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) - -Command executed: - STAR \ - --genomeDir star \ - --readFilesIn WT_REP1_trimmed.fq.gz \ - --runThreadN 2 \ - --outFileNamePrefix WT_REP1. \ - - -Command exit status: - 137 - -Command output: - (empty) - -Command error: - .command.sh: line 9: 30 Killed STAR --genomeDir star --readFilesIn WT_REP1_trimmed.fq.gz --runThreadN 2 --outFileNamePrefix WT_REP1. -Work dir: - /home/pipelinetest/work/9d/172ca5881234073e8d76f2a19c88fb - -Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` -``` - -#### For beginners - -A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. - -#### Advanced option on process level - -To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). -We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/star/align/main.nf`. -If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). -The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. -The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. -Providing you haven't set any other standard nf-core parameters to **cap** the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. -The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. - -```nextflow -process { - withName: 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN' { - memory = 100.GB - } -} -``` - -> **NB:** We specify the full process name i.e. `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN` in the config file because this takes priority over the short name (`STAR_ALIGN`) and allows existing configuration using the full process name to be correctly overridden. -> -> If you get a warning suggesting that the process selector isn't recognised check that the process name has been specified correctly. - -### Updating containers (advanced users) - -The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. If for some reason you need to use a different version of a particular tool with the pipeline then you just need to identify the `process` name and override the Nextflow `container` definition for that process using the `withName` declaration. For example, in the [nf-core/viralrecon](https://nf-co.re/viralrecon) pipeline a tool called [Pangolin](https://github.com/cov-lineages/pangolin) has been used during the COVID-19 pandemic to assign lineages to SARS-CoV-2 genome sequenced samples. Given that the lineage assignments change quite frequently it doesn't make sense to re-release the nf-core/viralrecon everytime a new version of Pangolin has been released. However, you can override the default container used by the pipeline by creating a custom config file and passing it as a command-line argument via `-c custom.config`. - -1. Check the default version used by the pipeline in the module file for [Pangolin](https://github.com/nf-core/viralrecon/blob/a85d5969f9025409e3618d6c280ef15ce417df65/modules/nf-core/software/pangolin/main.nf#L14-L19) -2. Find the latest version of the Biocontainer available on [Quay.io](https://quay.io/repository/biocontainers/pangolin?tag=latest&tab=tags) -3. Create the custom config accordingly: - - - For Docker: +To change the resource requests, please see the [max resources](https://nf-co.re/docs/usage/configuration#max-resources) and [tuning workflow resources](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) section of the nf-core website. - ```nextflow - process { - withName: PANGOLIN { - container = 'quay.io/biocontainers/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` +### Custom Containers - - For Singularity: +In some cases you may wish to change which container or conda environment a step of the pipeline uses for a particular tool. By default nf-core pipelines use containers and software from the [biocontainers](https://biocontainers.pro/) or [bioconda](https://bioconda.github.io/) projects. However in some cases the pipeline specified version maybe out of date. - ```nextflow - process { - withName: PANGOLIN { - container = 'https://depot.galaxyproject.org/singularity/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` +To use a different container from the default container or conda environment specified in a pipeline, please see the [updating tool versions](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. - - For Conda: +### Custom Tool Arguments - ```nextflow - process { - withName: PANGOLIN { - conda = 'bioconda::pangolin=3.0.5' - } - } - ``` +A pipeline might not always support every possible argument or option of a particular tool used in pipeline. Fortunately, nf-core pipelines provide some freedom to users to insert additional parameters that the pipeline does not include by default. -> **NB:** If you wish to periodically update individual tool-specific results (e.g. Pangolin) generated by the pipeline then you must ensure to keep the `work/` directory otherwise the `-resume` ability of the pipeline will be compromised and it will restart from scratch. +To learn how to provide additional arguments to a particular tool of the pipeline, please see the [customising tool arguments](https://nf-co.re/docs/usage/configuration#customising-tool-arguments) section of the nf-core website. ### nf-core/configs diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index 33cd4f6e..9b34804d 100755 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -2,6 +2,7 @@ // This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. // +import nextflow.Nextflow import org.everit.json.schema.Schema import org.everit.json.schema.loader.SchemaLoader import org.everit.json.schema.ValidationException @@ -83,6 +84,7 @@ class NfcoreSchema { 'stub-run', 'test', 'w', + 'with-apptainer', 'with-charliecloud', 'with-conda', 'with-dag', @@ -177,7 +179,7 @@ class NfcoreSchema { } if (has_error) { - System.exit(1) + Nextflow.error('Exiting!') } } diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 977ba948..f5bdc826 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the main.nf workflow in the nf-core/rnavar pipeline // +import nextflow.Nextflow + class WorkflowMain { // @@ -21,7 +23,7 @@ class WorkflowMain { // // Generate help string // - public static String help(workflow, params, log) { + public static String help(workflow, params) { def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" def help_string = '' help_string += NfcoreTemplate.logo(workflow, params.monochrome_logs) @@ -34,7 +36,7 @@ class WorkflowMain { // // Generate parameter summary log string // - public static String paramsSummaryLog(workflow, params, log) { + public static String paramsSummaryLog(workflow, params) { def summary_log = '' summary_log += NfcoreTemplate.logo(workflow, params.monochrome_logs) summary_log += NfcoreSchema.paramsSummaryLog(workflow, params) @@ -49,7 +51,7 @@ class WorkflowMain { public static void initialise(workflow, params, log) { // Print help to screen if required if (params.help) { - log.info help(workflow, params, log) + log.info help(workflow, params) System.exit(0) } @@ -61,7 +63,7 @@ class WorkflowMain { } // Print parameter summary log to screen - log.info paramsSummaryLog(workflow, params, log) + log.info paramsSummaryLog(workflow, params) // Validate workflow parameters via the JSON schema if (params.validate_params) { @@ -81,8 +83,7 @@ class WorkflowMain { // Check input has been provided if (!params.input) { - log.error "Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'" - System.exit(1) + Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") } } // diff --git a/lib/WorkflowRnavar.groovy b/lib/WorkflowRnavar.groovy index 6f2ab32d..52176347 100755 --- a/lib/WorkflowRnavar.groovy +++ b/lib/WorkflowRnavar.groovy @@ -2,6 +2,7 @@ // This file holds several functions specific to the workflow/rnavar.nf in the nf-core/rnavar pipeline // +import nextflow.Nextflow import groovy.text.SimpleTemplateEngine class WorkflowRnavar { @@ -14,8 +15,7 @@ class WorkflowRnavar { if (!params.fasta) { - log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - System.exit(1) + Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." } } @@ -61,17 +61,19 @@ class WorkflowRnavar { def description_html = engine.createTemplate(methods_text).make(meta) return description_html - }// + } + + // // Exit pipeline if incorrect --genome key provided // private static void genomeExistsError(params, log) { if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + " ${params.genomes.keySet().join(", ")}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - System.exit(1) + Nextflow.error(error_string) } } } diff --git a/main.nf b/main.nf index bd3b64d2..6fc4d58a 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,6 @@ nf-core/rnavar ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/rnavar - Website: https://nf-co.re/rnavar Slack : https://nfcore.slack.com/channels/rnavar ---------------------------------------------------------------------------------------- diff --git a/modules.json b/modules.json index 02df1b22..26009d50 100644 --- a/modules.json +++ b/modules.json @@ -7,7 +7,7 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", + "git_sha": "76cc4938c1f6ea5c7d83fed1eeffc146787f9543", "installed_by": ["modules"] }, "fastqc": { @@ -17,7 +17,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", + "git_sha": "f2d63bd5b68925f98f572eed70993d205cc694b7", "installed_by": ["modules"] } } diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf index 95d6727a..852dcbf9 100644 --- a/modules/local/samplesheet_check.nf +++ b/modules/local/samplesheet_check.nf @@ -5,7 +5,7 @@ process SAMPLESHEET_CHECK { conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : - 'quay.io/biocontainers/python:3.8.3' }" + 'biocontainers/python:3.8.3' }" input: path samplesheet diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 3df21765..800a6099 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.13" + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml index 60b546a0..c32657de 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,7 +1,9 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: - custom + - dump - version tools: - custom: diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 68f66bea..4b604749 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.13" + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index ebc29b27..f93b5ee5 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: MultiQC description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: @@ -37,7 +38,7 @@ output: description: MultiQC report file pattern: "multiqc_report.html" - data: - type: dir + type: directory description: MultiQC data dir pattern: "multiqc_data" - plots: diff --git a/nextflow.config b/nextflow.config index f4a47634..50f76a90 100644 --- a/nextflow.config +++ b/nextflow.config @@ -78,7 +78,11 @@ try { profiles { - debug { process.beforeScript = 'echo $HOSTNAME' } + debug { + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false + } conda { conda.enabled = true docker.enabled = false @@ -86,6 +90,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } mamba { conda.enabled = true @@ -95,14 +100,18 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } docker { docker.enabled = true + docker.registry = 'quay.io' docker.userEmulation = true + conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' @@ -110,31 +119,49 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + conda.enabled = false docker.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } podman { podman.enabled = true + podman.registry = 'quay.io' + conda.enabled = false docker.enabled = false singularity.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } shifter { shifter.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false charliecloud.enabled = false + apptainer.enabled = false } charliecloud { charliecloud.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false + apptainer.enabled = false + } + apptainer { + apptainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false } gitpod { executor.name = 'local' diff --git a/tower.yml b/tower.yml new file mode 100644 index 00000000..787aedfe --- /dev/null +++ b/tower.yml @@ -0,0 +1,5 @@ +reports: + multiqc_report.html: + display: "MultiQC HTML report" + samplesheet.csv: + display: "Auto-created samplesheet with collated metadata and FASTQ paths" From 259c1c47a8f290823f747457ea7eaf36b0ba7d92 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 30 Jun 2023 16:15:42 +0000 Subject: [PATCH 07/34] Template update for nf-core/tools version 2.9 --- .github/CONTRIBUTING.md | 1 - .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/awsfulltest.yml | 11 +- .github/workflows/awstest.yml | 10 +- .github/workflows/ci.yml | 2 +- .gitpod.yml | 5 + CITATIONS.md | 6 + README.md | 6 +- assets/methods_description_template.yml | 12 +- assets/multiqc_config.yml | 4 +- assets/nf-core-rnavar_logo_light.png | Bin 9910 -> 65097 bytes assets/slackreport.json | 2 +- conf/test_full.config | 2 - docs/usage.md | 6 +- lib/NfcoreSchema.groovy | 530 ------------------------ lib/NfcoreTemplate.groovy | 2 +- lib/WorkflowMain.groovy | 37 -- lib/WorkflowRnavar.groovy | 45 +- main.nf | 16 + nextflow.config | 52 ++- nextflow_schema.json | 36 +- workflows/rnavar.nf | 25 +- 22 files changed, 175 insertions(+), 637 deletions(-) delete mode 100755 lib/NfcoreSchema.groovy diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index dae092ad..90bd1ebb 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -116,4 +116,3 @@ To get started: Devcontainer specs: - [DevContainer config](.devcontainer/devcontainer.json) -- [Dockerfile](.devcontainer/Dockerfile) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index c06c21f5..b70618d8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -42,7 +42,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 22.10.1)_ + * Nextflow version _(eg. 23.04.0)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index ed4f93c0..3e2b977a 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1 + uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters @@ -22,13 +22,18 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/rnavar/work-${{ github.sha }} parameters: | { + "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnavar/results-${{ github.sha }}" } - profiles: test_full,aws_tower + profiles: test_full + - uses: actions/upload-artifact@v3 with: name: Tower debug log file - path: tower_action_*.log + path: | + tower_action_*.log + tower_action_*.json diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index 97d65891..ea0bd288 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -12,18 +12,22 @@ jobs: steps: # Launch workflow using Tower CLI tool action - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1 + uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/rnavar/work-${{ github.sha }} parameters: | { "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnavar/results-test-${{ github.sha }}" } - profiles: test,aws_tower + profiles: test + - uses: actions/upload-artifact@v3 with: name: Tower debug log file - path: tower_action_*.log + path: | + tower_action_*.log + tower_action_*.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a080ef2..f9eb2e39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" + - "23.04.0" - "latest-everything" steps: - name: Check out pipeline code diff --git a/.gitpod.yml b/.gitpod.yml index 85d95ecc..25488dcc 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,9 @@ image: nfcore/gitpod:latest +tasks: + - name: Update Nextflow and setup pre-commit + command: | + pre-commit install --install-hooks + nextflow self-update vscode: extensions: # based on nf-core.nf-core-extensionpack diff --git a/CITATIONS.md b/CITATIONS.md index 0eae2ebf..27fb294e 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -12,7 +12,10 @@ - [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/) + > Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. Available online https://www.bioinformatics.babraham.ac.uk/projects/fastqc/. + - [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) + > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. ## Software packaging/containerisation tools @@ -31,5 +34,8 @@ - [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241) + > Merkel, D. (2014). Docker: lightweight linux containers for consistent development and deployment. Linux Journal, 2014(239), 2. doi: 10.5555/2600239.2600241. + - [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/) + > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675. diff --git a/README.md b/README.md index 381dd4b4..5c7f0b4f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/rnavar/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) @@ -66,11 +66,11 @@ nextflow run nf-core/rnavar \ > provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; > see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). -For more details, please refer to the [usage documentation](https://nf-co.re/rnavar/usage) and the [parameter documentation](https://nf-co.re/rnavar/parameters). +For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/rnavar/usage) and the [parameter documentation](https://nf-co.re/rnavar/parameters). ## Pipeline output -To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/rnavar/results) tab on the nf-core website pipeline page. +To see the results of an example test run with a full size dataset refer to the [results](https://nf-co.re/rnavar/results) tab on the nf-core website pipeline page. For more details about the output files and reports, please refer to the [output documentation](https://nf-co.re/rnavar/output). diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml index f2bc183b..46025e30 100644 --- a/assets/methods_description_template.yml +++ b/assets/methods_description_template.yml @@ -3,17 +3,21 @@ description: "Suggested text and references to use when describing pipeline usag section_name: "nf-core/rnavar Methods Description" section_href: "https://github.com/nf-core/rnavar" plot_type: "html" -## TODO nf-core: Update the HTML below to your prefered methods description, e.g. add publication citation for this pipeline +## TODO nf-core: Update the HTML below to your preferred methods description, e.g. add publication citation for this pipeline ## You inject any metadata in the Nextflow '${workflow}' object data: |

    Methods

    -

    Data was processed using nf-core/rnavar v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (Ewels et al., 2020).

    +

    Data was processed using nf-core/rnavar v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (Ewels et al., 2020), utilising reproducible software environments from the Bioconda (Grüning et al., 2018) and Biocontainers (da Veiga Leprevost et al., 2017) projects.

    The pipeline was executed with Nextflow v${workflow.nextflow.version} (Di Tommaso et al., 2017) with the following command:

    ${workflow.commandLine}
    +

    ${tool_citations}

    References

      -
    • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. https://doi.org/10.1038/nbt.3820
      • -
    • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. https://doi.org/10.1038/s41587-020-0439-x
      • +
    • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. doi: 10.1038/nbt.3820
      • +
    • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. doi: 10.1038/s41587-020-0439-x
      • +
    • Grüning, B., Dale, R., Sjödin, A., Chapman, B. A., Rowe, J., Tomkins-Tinch, C. H., Valieris, R., Köster, J., & Bioconda Team. (2018). Bioconda: sustainable and comprehensive software distribution for the life sciences. Nature Methods, 15(7), 475–476. doi: 10.1038/s41592-018-0046-7
      • +
    • da Veiga Leprevost, F., Grüning, B. A., Alves Aflitos, S., Röst, H. L., Uszkoreit, J., Barsnes, H., Vaudel, M., Moreno, P., Gatto, L., Weber, J., Bai, M., Jimenez, R. C., Sachsenberg, T., Pfeuffer, J., Vera Alvarez, R., Griss, J., Nesvizhskii, A. I., & Perez-Riverol, Y. (2017). BioContainers: an open-source and community-driven framework for software standardization. Bioinformatics (Oxford, England), 33(16), 2580–2582. doi: 10.1093/bioinformatics/btx192
      • + ${tool_bibliography}
    Notes:
    diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 2689c2b9..76550af1 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > - This report has been generated by the nf-core/rnavar + This report has been generated by the nf-core/rnavar analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: "nf-core-rnavar-methods-description": order: -1000 diff --git a/assets/nf-core-rnavar_logo_light.png b/assets/nf-core-rnavar_logo_light.png index 9cca37e801c2dad85523ea3419b9aa15f0468640..509f923f092c056161198ce9745d0fe4fc83e867 100644 GIT binary patch literal 65097 zcmeEt`9IX_`~RRQS?ZKSWhn*~p=96c5GGq9OZHNfecuPCQz(&w!1jG0qM))u18{N;szxKLnntC7*Z0~7*=;B1!jv^4p5Gb_^hQ29NgTYTSd@O|5 zS3HI44fR<@BwC_WweNAg^K`t?ay|Ua^`zuS;o*5X;p5j0nLR_3TdTw-*C$<<{Vk$; z9`%au>-b1%=CCl=x~!Jp!Br{RFpzjKp!3X+Tb;*QRKss@Kb){h^c+@seV?p-3zMBT zv9)Zlu({<`v3Pc z_~QTk@G~L)&kz6ShyTBGp!b^mFYH1%8g&}PE+NMRdy{Rgwkaa9QvrRQY2HJz)6`6H z9;J$!8p?T$p0J;N*Ye!J#ykH8M)iUCxVX5E!@pK|Rzc1t45Gxe-2E^GvsRWhY(8G+ zqQw!LH!;zIl^)J$8$X^IcCItbD!;xEnF(K*M&+X@JSfW~(%%?AjAD}I{FvT)!b;+< zT`3RVvHyDV#tr{F?pFSzX|tN{P8k1QHN6RI-9sVD@-lUEm%l0Eg`Uqb{CpIznVgoC zqUmmd=@Irb{U+;BnnF@S4JpEd=f8=bxA|}L4A?vsm9JMY?xEj%PSrz{(B9T6zCrD{ z5aNCa{cB^cli-wq*o{Dpv7Lu_ua|VKlQa68K&C3~Q72#9XybNMzba}b4=Acza~8q2n+%iDoFDn0jDk39X?^7A)!^mJ;E z5ekGVYdquWg)k>J@LX5^<&$Ub>jptvS20#izP!}h(}bdq;~{4o<`Z~-?Z6?eBvmOx zsE#!^me;!Al9p_BB9-oh+Bc@3zYqDCn3hx{MhJ+VI+>dJOaT*E;koA-_dUK}Uzf&# zH;{fF7_10)<{MQM8t=)+Bc#9Hzz?%a`@_R0){SISt$Kn@K8L}>h6mZ|Sq!BZKB@H20kftU}^PiE` z)c*Xdd@3S@t0+sw_uO~aLtzgUG2d;xQ1Q*1H#0qHdV%)wP1#8svyWz%C}A74L_x?B3pf9H&Y@2X=|G$}7iYO?E5Lr+QZ zunjfr@njOx!!AI9VRd9th^kl#?3g$t5Dxfn?H4g>K($Nt+fHaOY#hv@QlJIXl)td!4Cw33#odkl6Y zV>S|OhL=y33;S(CMLA9S@}2)++OhBFrXf0zRg_T_+T~HTPwd7xJV6cPBJX{fB~&hK zs$Fc?B(tfBkrDJu$X3Q1{1zTNRk(@T;z!+JtsYJ#VQFEI95Bp+1d)p+`Gk3TG-5Wg zkhB!>_0%li8!7wS)(5l@KDF!}dm%NoRf{a39g|I_D;7#><0*1`M%3kp01AB_Dq!Zg z8ht}kcgMfVhs)|`f(tl+ixNr3KYnoDKRVH}!H24qCWtT&%xd}zW+opB3MoDNJ0-8f zNvx7d#yy3T+j3B!o%L;!;b>EGDQXB~+h}0EX^k<%)ZBpGVwTz%Bc=Z{6LNVVmQ)Zs z#qHX&f?Rw4S8Pz4H6Vlw2CL`ph1rxV>T3%^&1h1dBkPo8>RjJw|7HE<#P4E!4_OE` zO$@0HI!7pPZx!b@3)8f7f(6Vl`(n8hAxh@*>=H@8QQ)g9oK9SqBFr%3t$}fQ3U0|& zMTUI5{BLzyt1e{`H?CqHGJTzP#T38;zV<;^=nNbG6N-_k!KrUQDx)Z|AC(bG|5a8Z zB*H@M#uON%NKm+sWqkHO`)aB@we3grs9;DMV?Q{%PqLj~`hASTUIF*q`ZO5WR)wVFI`G?Zxevi{$Td5LndKR;aC(U=|9wR~L8w;+zr-%IHsbY> zUgGTk{6DWrVb zYX7qj`>+ae$t5+}$|T_!B3=Erhn`P}k1ai*^PzUqmU{4eDXuat%oMLHRxej$e~5m@ z@ADVp?D3O)y6!#xyXd$s{yrf~zYM$Yrd~^{xM%^*VgG&MleV6Y&|SUNwG!INi~rl; z<-XXdqpn!99)UghSN}nCVm|NOx&~&TmiGceJ?{6R>laTmSZ>pxJbelcMsk4R0F=Ar(?q*%!}BhZw%+9K`8y{Yh!MT%%c;Bib&k(wxLRjmW=N{ro zoje;XgQ^~##P@&C)S#ViS*=Lu%Jg6vf7wA7B1zehn!53h9Ut=hiFVdZ2A1)BWO+Or zT}sR*gJqqhOx-8b1SCR0`&Ue?BhO8gDxoY*R=fY z+Cyn|_k)xr7Y`wB{C-T)JdQ-^IL_#4Kt|xti;{O2Uif`>)vlM+z~WAes&vp2#~e;> zaP#^zhn)Ghwj{nES?XIu)mFnEPiGi7&MHYgMRFdBqLYyRcM0|3NrSwRzt{zDC$Q16 z*lJ*$9KIG@s!K*lv(_p8gm-n5bjuuJKPNIbLluNw9-=Anc+g>>{ftA1)Liqyomg7G z0lZGlRAqUVOzOE5hF~nSdqkDH#ahTn%b<|fSG~?U$lf?xD}R^!j=>M6H8HyWF6y2} zPGPZ%iKNdTp7uW4JWgAQE8vm;X_WJc)Enn#$({*pabQ-s4krlc*`UTUP?m@IrR(4uk6XT&bDN%A5aA~}3fQZ}+Rd6c3 z*IAG-N{$P(j4Q>Srfr2tpV8=0h{!#~3-AoOv!u9tWom_0YBxR+7|^?x3!H1(U)HeMcJvM;GiZDK%TC8~?<`}ApK9*l&Oz?(AV;afU?!7R7^1E3 zn(zjAZ>L6+)k_BZ;z(Js8zvb4U#rVK@}KTN_B?4j^DOxi6XO26e;wx5>Meq@OeH16 zPKhP&D9lsS_dDnqJvA_TPayL?T-&Eo4MaN$Vsh~LOFAw$sP98vj^)e3erB(Ix)0Ed zcRcmT-^mAK97kIoOzJos^3BBIn=oowuyWRsVNp-Q8QI%4?47^vYmBj55kB(7-5G-Jw=*jed)*MV}zlKa?!7quxNI9Dqv5~0*qxF{ z-|ays&_rj1kTx$F^uK@^zBGGr$N8@D5U_4!fjHEh%d}?#HzMqS1VBYf&^KYut?s3z z#x(Dl-G0}fkFA#VYCT#)Cajcq(Xx9}P9Gs}$ynv!cB`zU=s>7GEmrr*<+Gsc;!_6q z1=Fl1&esa#1l?YLx5t#zFs9X%$7g7LW1T&4gw?plYc~G0M)WlGL4fi~%|d=l{ONR0 z(ExtJ#m(uPIko8AUgyCi5<6xC?H?P${GQ>p{S!2bzAysv+#gde=;uWi-SN!d&Z0cl z=Vxa<6L=w~xspnfYZmT}S`g$EU~=c)X2)i+nZgjfLi{{7BR9A9V@M?IiAzae66wR{ zbVBUFuw%J$iY49n2)JM4(tQT$^3x(BBAJp1iSJ3%-4{`4VM1nRNn{A0Wy;eaWAc95 zmX5rTQxA~AmcS{swE)2-o_n~AHzPLsJI(%{&@RtXp}uWD?G!-#W|yZ}HlXQ(*l93tqTy}~zd~*$CAgPi|Hx9G?WY5}M z02i&|#Gzt|tMhtL2iunNy9`lKjcFtdl5U(c0=}qQSucG4Onn{mfpPuC~ zUODq^;@FC~c)^rubE~#vvhN#etKRV16JtlmZIYdM@X)Bpn0CtGAJ@B}v82Whya624 zAWNK=gJR5mxMhoFA9d`R9<}|+y@96bmehO5?J{6J#mA%^uw=C3g0&=Yhgqk{lD6Pl zA2MNCrS_F=zGQJRW^*O@TbhT;+S9Ov8I?CaYg*B%^XJm?+K0UD#yYZ6KNnk=2?@=p zc=mdfEVeY#XB$fMFMFYgxxJ-=GENxkH(mxUP$i=}qjnpYz~jsE$`XWx{Ko z{su~~zYEKQH!jQXa{LphLJz|!xE7Bz&XW0HhkW@%MrHfMT?G}tx!TNXzI;CFJ5KS| z+d?rqica4@b;u}fj(?1w;vxQs=2i$^nPv}O^2q1a?fY1*LTE(|m4YKGJh`lI0QgB5 zLd7Q`gSl>EmtO3M%k!8F{Q_tbt)Q?GgUEKEQ{K}&yDmX?P&-6cwO7Pf5_I02N$U;D z^>}L)h~66K!L}xBeQR1XE4$^_To%#xacxYw<_$IFVFHr~HRaRStq6wUxxh^9K{nwv zGSbBg62eHHrLdO9f=R$peChd;#blkTAnf=uz@z{+E z09mH;dkVd2@B;WHFHWdCk-9TsY`B4HF0mG@Y0w_n%lfxep=Py_`>pF8HAic zI5>Dzt5K|fzC3L9WK7<5F*_$RAK>TKRTAWIyYol#>f`FxkO*AF7vCO4Eh?p$q_x59cLmsMlbT+}V zaI|PtAk*V&lNx5bTV?I&R}u~D-glvDnrJQ!d9;*d={1AV_H|(ab9o^1DGx zEg*8wH=cWZ&jMWl(Bb3=VVJ2CsbSv&R{t)jDfS@mUP+~{)vZwNT@_+ChG}txxpgN5 zoEUkoKQHx6+acPT(tX;P1!#WopOG#Ay=mGdgRh0xa7Yzn`F)du8^WH4JELXyeXy9XZNETOysflQOlCGBF*;iJnGrL6%1H`;Ol5>#tPMvU^qdFg6f+ zJ15{3Uw%mDwl9BEHY@WzC}z+7&<^JkfyR=ThRTwkPyL*}H=xoj`;$p= zzvcr(!zV$+TpgsJOE5~&Iu_a!B5G-Szdsm3JB-9Fv?8G!dg;0Im|<{;?oNIT>Mw_u zc)4N9LGY&l#N!Pr@+CYtT`7<%?rS-11^B9A3X|D zz`k>awRwQ!@Zpjy&@Rq`BKE}8fF_hR1+je_VFF#Pw4WYkP`_+9>`NqEb*gHg1zKK# z9$UEbB;f-%d{2K8i4zlOMLs6c2Alex9lj=y7xD?ln8j|GV)T%Ht{_O8$oT_~^dpxb zh6WP}2HLBBFTy$k4vuWXZp^LOJN}+>so%B{$y?m^&t!i3t`;ZptDkukl%4!I;I-4amD{4_C|db zZO)L6QpS)3z?ueRT_Op~KDooYukNekjPxi;Afr7!vZ@W`8FH7KQEehTFy}6Xhdg}Bj%BxLhz^5<=~ zrJ&XZ1!n?b)vw=MrncjT`pUz!c7_Mm_2vn-!H_(%@uWNm`l$j4BYD3>1G>f&!KDEh zuXthGF+96Nj(Oc46AUNoKh0wc3yq*^&k*k3OQ%^>h~DYB_{L#K11?8(IF=tl4VlX` zMOG$&kXWFZlMd!&o2S^Ck@w$&+a4-RQxde8 zhGZVKLiQTS?|R%5$A%c8!MMTUp3#~rR4ufb%a_T=gv~&9CX$k42Q1}xh5@QxJ5-Se zO<11i9!(6?i7+79&@ktMc#3qHQhSn3jY# zn()HALZ!onAgu|0NiBT3VTe(OOFYa_MqYyO+Igr4F>MH!VT0Sdb_l2_5AA)BkRplz zY67NS#Pi%uH)8<~6fiX}J=utEmR9nJ$b(Slx}(J%bj-eu-&-8ZJ$G2ML6xQA zAn$*S1b*Nrux5H7vK9w{fGcQ-XFC?hb{WqE`jYR|FDtK<7QdrH5269ZQVSZR5JsC% zYD*y4oDl33NA7(pbp}7Lf=ANz3oMdIKMMhB_~RphsVuLXpoz@ncSX`BrMlA2&3=Le zr=R#GVf5O_Xw@XE`ka;gE+ojMDkPy4EYh2}2^PujSTtg^Dwjxl`x8^S*#Bo-a)~MA z>X3;%V(y9P{#itTa%OHjdaY7hm6%u0FA6rueZa!(z z55fR4_!W(|Y)7QOjkW(ASX(RZ05^mIM!wMa#KRYB6NL2nLt0$|L~%@$H13UkWcF=r z`R6Sb*U{lvTj&`WWK&2m$Hbo+Hj_uVHq@qrle~7EG{CIF^po4H9ib5MAw#`nF)#2a zskzw?mkZ`ZT3m&w({4j*Y3f&}v`ym3{rX>ST8FkF4wX+EYy#6Da?BGl^l2ksF*uF_ zSf~FIiseqVB)Xk7I-U)Z3xPLz)#r(2_XdOp+Q|V>M&R-JqC5!o-U^;CyNQJ96Fkol z0ui+IH8F;9L=Cclw!91!P9v0{6Ux$3o=Kw61;|qUDTx1^F2F78u$?LlqwQc#!YOyj z3wao0qG>yrwC#IMe%(Q5{p2e7gCJtkB>*DP;%-TMG&e^bSEfYxsr6E4u8>&@`vA)k zxdcFVEn&Lu2qsQM&ZGW+Xv1=NzHkVxy8(U~=QJ_fFaS@1l%flfx{Z7aNx5?ikptdu z{Iz(pIxZe5Lz~Z)10m7UbOc0FEs_(8Gq;xm5{Y)7VO{DbvU5p+_xE>uE!9gj!Iaau z%TFIXWBQcl8QS$m&d-|+{G1^WoC~bS1nb3WC$J$>;x_+XN(!O`AFjVa!rEXG5`K;b zLkucjdLoFq=2sw)uk#>uh1rhcpfy5-0i{s0rF|25=m!O-h2=Vit8$brH`j`EeQw`? zL6`I+b)0m}!FGYHzOt7qDQX zIS6n~695KoovaVSl!6c;GgU4mm$Y?s0f=D8&_)T~62QOo>)(U|a=<8| zmh<}3Vo5buv9oOvSK7;t4{f@qTbfzW%O{eaBbhLPRl$D5)gGw(des^iu6^*W01VD= zV`SCyCXV!F^g(CP^s5eD;YpQ(DVV+nE2t1WsC?LjMo#~>30v%zN7F=bEEDaTetXht zD1o#E_J1y^GsUSdbxb#c*pR9T1iLgE)cIhl2K;)5od|btFs`W=y+@_Ni2Go$G z@Q{h=CgX5+t#?(wO8mjy&(d?s1W;^(en=qu=JwRZH31Ya4A+#T-}62FOj(4Ize6K}@W6YZr^?Dem#2jOqCXeRmww! zGoXHbb(q>X%pi-d^xzQ?UExb;e0Y9E7+$IvUKF2wG*%JQ^{QuCsPZgsEN-9sivbU` z^o-vqspl3owq}(i0*$Rkr}*|_c^%3<0OR+;sp0(+>IjV)o+Gz$AOr8Yi18q}9&GBb zhCVk~4W$D)%R_z?rKpk>Y~a!^-}tp}xLZErW@WFlQsU52v7F)kHR6QLkLPa`e7PWu zP*($;n`-Gse6jdZF{fFHdOy&oao;`%FPORU1nYRZVCpQF<}Y*}i+P1BV@o7}St8x_r>2-9wNP;M8 zcD9UX^E6p$%+jaBD+&%Za`9O#c7)A0(g;|qKb}NcWL6&jTBlfN|LX0O_N>=8LS}~s zEG>-LxD6U{;Q6zLS7gq*oU)Xj)4UHIuOt8#v3%G9OgVIN1CN5DR`a*hn4WcMhgXDB zET3mhL~RFhA}g0OW>3rX=Z(1R8A>B*u+jHze?P<-rw@NK&kIl&y4o0 z%LA25?zFbbb0q!k(@9RF=!8@GnzM3FN?D7!<#~RA`YxsQ0HN@LgA74Kd!kPf;JS7( z{bOMTc9-*QcbLo2OA#@Kh`ezN@SyqA0S*o(*?$tUfu^W(7FFBZ2>=wKiV0x*H62-`5Fclu*L zA~Ipi-Mq2=6WV6m{YiUEZ;SypCJhiu0!L}LK>g?tkyI=$n*VCQQ_2pQKnKvZ`dcf( zW!^7Wh9_W1bPC5%$)`mLLn%YIqI6mGFsa$VK&*8n>!rELxi1ZUF(i)7X}Hj`zyj*c{HII61u=Y<{rl8{jrhqkAEU5q=%DQdXOIh0xDvYHV8Foh+13dBI$3Yd4~3b%RKPN&QF6obt$IcIBy*HauFFq|vp$<%f`KJ5a8XFyi<8}qXRuV}*ahZQ{g zB#I4Eenr^N1*2yg6?F<4vjkE^Y?n-RvKCWFXJJauev8uSfw0=yUMsh4+Z)tnp0TtN zhyM5PYvE0}LBHz<(y1Rt%#K}6GXFh~JA5SnU z(4kC|If7CaB`fZtoKX}kjSw>H4J{xGWQ8v&vsvc129b3({jj$U9dAK)8^_krX6J!# zIxW_rTP7Mp)wT=zd62oUF0=NxDXnf+`wUUv71&SpDi__ySdKB&|8%(&Ba<$!0N(do?Y0_U~$B}&=QlWP~%Hr~FH$qctY?fm)58_koMPp*h( zJn3j+J$KN@k#?RE6iF6U1l#d{Cx%pb1cTHP~un?rQDjRQ5zSi@)HkbH|YsJFE} z%IdEucy<51w_zb#xgMV1E)d6-W~&UlNK=dTyp9)j12D5bqpWdPHZl%RmduPR=4A;e0bB0cAG9A(?*V0)a!t%S*Pumi8vLLfTp)urZ-phYc`kn znQgB;!M50G<(_T&5zyFZTCoXVP2ukAo;;Y=wPf?8DSysHM5M?H_ zM?Wme+|<<6)Qt}@hB3?{hFEjUbOat=K2*|1U#4c`%Hy{-#+zE$7d#W!Jx0&BJ4!lA zfa!-QG4}*ZK9e$>O|?5TBlv}c?B5%;0m^F+?`B+!rxzE*;;)*`YcRhV4_Pc=nV4M|q$8`7S9o({=o;ipR}!KWvPa>3ogeEH1k6m9Ibd z*&c6fMz6k4v9uNlNMFG7E4_Rd&GH2dKT9!=t9!6PxVA|wDCi6ghLEN0zV&88OHD1q zXW-+DVY*u(O|nr_*!s|ws&Z<�ev`Q}H7y#R1zKkC5n?0_OP7^FqWWeXhX0t0pNK z(bt$TL*ehNPtM(;VA@5R9zN!e8~K<~cX3NnUF1p*`5e(DU1F8lRX-)8KbL`E|L`3V zNx2$Zf1S7Do%}yd%DH81m#>ET4sG1bNkca-B!p$@$27Ju`3?2uL@BKov2V<7mu!_y zZ{zyp_2QITSG-eP=P-{N#gu#(3@bdT4+KZJNda3|h8Nf=HS=!63yn&_8xd=3Jkhf$ z!}BGTsS9Rf-o-Z?Q?|cG3CC|q^rGJn>M0i8LCYqr+E3?cMnhr-$;c_-;y3nImk_jg z*SB>)9>F^Z*<}?lDtFvDC)3w(;J|^ymifdvBjSktDB*-0?<&&u_8~@@7`@G>U0<++ z9+SbA7tkuQpQRryewLjRBRYX|j#Qk}?Z|6*YO7K~og$D#s)y)BWmu8L?D||OjOHli z(rd40>4_~TSlT+@@R3Vwl4m533X}aO_w!RFZu2~QpnL7?*4I%LpD*2+wLVo|@%I8{ zzZ*2>_N_CqtE}T$qqCAa_KGgmtQr5qR1iS0X_i)@emeG`q0wmFbyr~nZu(wbqnm8n zm>_weO@nuHR=8~I#88`0`PS5U9d(wcUZTt7AX?2|`@=qRC83w>Mlt@JqGP!z*B~9k zLWkYhn<%5xrfan)FuTkCh{hk_05N^8n#jP+e{_`}<+~B3W?CiNuAua}a_MTdYyUEu zusJz*oM-`=N*{Piw?l43yLb=$GNYte%b+5I@-V7dC>B1^m zR*$`EP?Yr|V3rCL9eeM`ru`w7D!cmZMv3U8-`dIMVpnov@J7;{b@x9^3m-Z3Y{Z&* zD_zX0=I>)SdOkw+&z36W$kA!;9RD64IRcJ9N)qO^ytsAe+9S#M%>(p0L@&TU7Z<6d zXj3LQe0J3d7TseiYm0wOit-x`{PWm{J|RZs<&$+&Hgo2h z5yoyB+HQt44OJ{z%<^Nov&O3L_s`N7xT*-x6tM{ij1IE&RK^F;>C|9s3ZaVQ%s1ZD z&nS+C*X#c67*TD{>-$e&9F_U?(pP^n73=qY;t~6n@8+=ca8aLp%dr}3!iDJCk?<^K z&vypzO3_=}Gj~EnkD5>38d&H~S$*Q#8lks$jjwQi7#*)n;Y=>q4V;``tYFUD_J8e# zh|!nSX8$YmI;3~P|A88khWk?zH-)?If|Hk_xY3dxFKoZ2t zJhyn*p%TVmg-uCC^US3grB{BCe;gjJc~y-@ArHqhvcIIv>?>x{3Ka?IQMYkLr(_(> zW9Yhih|wXG9m5&4$o+&R?gWb^T_Edb8q`Plm^+Gd%I_1>MvGg_x>l(|hG zXL8v{RZZI(QAKaWHr5s{+1W7^G~V*hY!i97m?+bvfBkF?1U{OvO;CKD`v$kh#Mp6S zW}dnS&g=07uy2cfao?kBg`l52EM{x5^{qZ9WVy(?lQ9ObhGymV&M6W5@vZoDNTGn5;{NXx zX<|J~8H=}B&gYFdI$k|n(j)EUEB-F--tzpx?lX!kjav~2haKue-^}@3(<2`l9v*%V zpct`r=&rGCgdyq>V-|xIQ&eFazpBmQxvNAkeJ+~rNaF6(0Q}arT=aY7^=HiHH|9($ z2FqKi7a4zW5&2$7`1++}teA$yJok{Vzq)`Pmy%Nml3Kg-F zXgU?f+Q^T}S6DR=!9a6CFTM63I1qE;!8>bUFzl|a`*)PGkDYY|aNoPCe2S{MV#&TC z!F=~d-rdNg6D;BHXbe@$z9Ddm+VuDVjk-}hr>I}r58#I@|Hf&`?C6on@5rDQ;BtN* zCm#GK9DZNG)n!xr>vw+e68-Re^a17vyB)GrmOgb32YfBAX7Z}B^qsjdl3ZJRYm~<- zu>14DocgGES;E)15;iXQOAcTgE-RVS%WN{_ViKsrj|B?;TuuS3;|dS!u*jwlru ztBk1E6!us{JY>%V92A6y^0s)NzF5~my5ZE6)b0sJz-@?W8pFoHx$16HHPOny-p6#g{Jl;f&|&AJU;;%xQ`;X{=fW1tN4U72f4 zG2cMw-+5+3LoqX^{p5EUUI>9<26SbY{c>rF%o(YY8`tmLVq6s@K1cKBOl@2}*jRT~ zwnF^kOUr9N0z8a!ueni;qm=x6K}x5od!>a{9A3?Y6I!_mV$%j)A(Y*B&e?@v8S-a( zSs!W+gCwB|RuzEbEPOpaAT+ZfMs4{P_i7&;wmSDNBc#h04lydP z5hC|$bEW#=|eu-u>CWszC&qFp66I!fh(Y*Z8a;X4HJEb(E8rIV;uNI`YuH-0LG z_x|L@M;I=omg$aE(ovAcYk2X;oS)P(zTYR)WiNgO zyKe)d4l{1;mgU^sK2|@v0DmngV>`~z-{GLowF<(4%{)|B5!HIprtr|JB(XfNq)F41 zdBg7zqyK>m2|zW_rj-*ODz_K43Ai6K?;X2D^odN@Trxj!?`>nAs;1XPoBi~&g)}9R z%Mk9FZFTg7bZi1w?Ot=Hz}>6#t^$S6^%~71Rd%7%yXx;S_t zt$ev7PH)oT_RV1JM{E6CffG#%%Bw8`QG6>kQr&(jVIfv&iAif$%O5ydUwiap6W<&v z6Fcmpmhs~C*}t_NH&TIG85T<+5v{-jE2d1K8R0F3_wzj=JtlSsiU1_P;jIu^rVt_$ z12*~{@dWX^EGlooFiB*1lh^f3mtR~?6WXJ5B!8FTMy%2r1aV71x1-&JDdv*D$fk(E zVm%|}?A;~_a#xV!!8snvf{hP7d)bjzB}+edZ+|(zqRkJa54CYhAB$vW9i)=5Jb1Td zsKHz4h5CdIc?r6d&$A<`fhL|44`p0}NYs9xL{5hW#nr+3gyFT9ae7LB7N1huo;yjb z&wqUL-Jo$kkm45a9E#{1v?(hCYS$&-Bp%v6bD5a*gN`dT>3kVm>-w&YhaNy*!&?ij985sS&kCNa*JE8-5_j zl*)Ynf_EvK>~Nl0&OdOB-Lk>%-s?G}==9cy*Z4c0bLjG)or+@Iy6*0Mt>7%jftcqU z_udxaRbCWFgPc{vTfq-3ZDye=9>R0)Bi@CaU_mpj1{f~K9QZafW~F|U&y<^Q)&CHq zFo4D-zr(JPUg2U$d;*Q;!ZuHD4D6}d<7)|w^W(gcEkIi(h^Cp!=CPKa!I7uay&pJ8vY}rHdBkJ~S=vi+eT$}~wv;e%L7}&a*03xDe z641-lqNOI{=)U4uT~qf@4QM{Q=j=M%-eZ{#(dJS=iu^w{4uPI2(A91YbOkq5dnMu^ z15m)6Dz4IgZaQj_0FM0W-{F6{QB$+Ehc;Vmu4mC%2G{h-{o+HBkP?7|AROl^&*XlN zc{98Ncz*GL$dj#;uK8Yn9=-%52mw7idF*<#&aI$(UQuEe&OGOBRZcJaVH|)#IH90w zbu(d01*q~5_r>ReULX$yb~x$fg?8DnBhL)Ur!y5BcXn#3)B#SIPF@jTO#X+%}kW$rp4 z3HUieI@rAoBzq4wsev^5inv}1Sydf6MvtALXt@YrrxxtnRhJqC@h{PQq)%?!|2&PT zpP5>5)3pHS*KMqIO&W(WVY_EfVp{Cxd02)`XoJK9h!XVb@0(q4F2# zJ}mNy&+|Bnmlqv1P4hM{I*^EWBi?`d-6?cN$lB^``8zBA%$r;9tA!NF3I$fVIxVhD(!OdjKfxSyz0@J8@s*BK_WI$@|uGw$m!mVLT+5xsx z{KGk7{QTE}Jx58gK}JV44rH?!|6Sc8AJ)Wgapd0HBQ)FW>n>WJ;vmc9Ex!(h$pqqc z8QU$FAE6>prrggQ0J;1iHDkRVI|CX7z+Xi`kvVmn`a8x4e!nt|yE*#)L1tRH72FwP zy}zc8@yNOTAu%*!f}4v0+e|0--z5ooD6v-%V({(K1kI(3Hm*lpE4|pVS;4rleR&L?aN7Kv{&uC*`91Y|dCsl=N?)>V1R&soy^VyDmb4<38D)!4InyyH&6 z0f16w;%OKKXPivp?+|A&o!mWFCBUZO|8%zX^pC0=yn*wtvWC$=-ao&Z+91td6AYAd z!l-jeHRp2*41eHtPKGkGu>*&tXe0PnR3d5W%~sw)$Ql@8vJhADJi-kl%mUo*d9lT8 zdO|NQ3VcSJDtZcmSOat* zd%gvZvK$-FccrVC9p44n&2AF*>TduE);a!3ZvJ$2;kOrUzvKx9m&SqQ!UN^W&SlX+ z_Hcl^&Kr0c z2vJj0bsAlsEv3mQa4tNe+GnM*KG3D{Q6u-#U4aBKIj{YuYvU4kcx;N)(KzJ_={MjAFuLS?R3PHnijg*CMuZ5>*2TkknWmFH2nAKDBSVjNthgj z441SWzajgc%#wb9c|*XjDC@+^q1o~Vlsx-%@yuDGtMxmaxH4MIRjAOva6YW< zFzABA!sNW}3mFRe+N-*g+!j?W@*&}0ItKAZ)+U!^?=F6e$Ue;R>Y}Z+=M``$sRg*X z9$@rO*o*(H{6N!|M=q5ABL$mP{Yh>C$9-$4KFZ$y)1!4et}IvZ0*zuhK_@)7;<(0tx5Cm_Jqrzhea(H>C6xM|;cjg@1w zuhx7IF^WgVevuFJ96L?gU2apvTk)CZr*?qQ0T>mo@y@AFigJ|DC6+=ZF1>);wJ#Cu zDa?V5@}Slt@1I~fKZ#UZR_hF6Yx$E1Q;krj-qL{*Dcz1rXXlpGW8$14M)cyxf&+86 zb*Tj>$~LRK_QxFY6Hb~b5oSkV5zY@{Jq_yE{tzZJQm%6JAS#yb&kA8{GXB0jbBM@+ zZ-sfD+rX?hr|H;u2ge6bu>%Jfg6}b_?6b%wEAyYV2h7wQtU*A5!NroL-j;1`xMFXl zSIF@ao{GJz(ymN%m&LQ_-=mTq*Y&xolD`)q0IyOuhKmz0DmK-x?U?ez%3%;&B#Y{S zcKR?(;6!&T+oz`g-5p!NRnzvJ6bzS72tE*=SBRT1B(eV_cWQj_)tsbu+pee*w$Jyt zRxwb!*;1R4{axORv&G?Db8yEHS>c3Nrx=?IqPE^|29fmMJMR9n$Ws#wzY1@%hl{Me zuGwB}y&sGyjixIdegma38z|1h&!9G$bc@^0?E2B9rCdj+sHEFr^(c06LKYQpZMio= z76r-X?~#%*%On(P#i*>Itgrc}#_nA)Z+(Sb|M3cE_KU1Bq~yw?3QE%!Ve8I z9KS)gws75Rc>?g|TG-=@N6W~{#?UmcP!q$slAzUy+*sozSkNX+A83(}7TO4(!uk=9 z6Va5j?R6NedEbwrGJ0r_1||=l28w=M_x-k9VG9n6&^?A#^Z4V4!Jvb%UYl;`opV4| z;Z1V^!i5d;YOIR%0~g^wrmm@n+sVsiG`f6x8kvy1M}m&KHhD$QV>bF&@P?OfaBbW* zxC}sWl=Du-BRX~mTduC%3r-Ub)*q5Be2=qg>HmW=_D4LO-pQbvta6x_UG5C>KBJ-hc}&vz zZ?nwzsH)wou7?;C7=js7Y?7NI*=tx=u?=#zFkCg+SJMYG01Dn zo%MX{qLuA=X@pPb$z?@^;@3Ope7MJ1t2@9nbhOCgCt?bRQ_wPD-e}3QosK=x7I`@6u*Y&)f*YmpW*O8rQDj_T- z@}h93a%r@n4-iJLCjaHc3#jMD1SXhc+xbu3*;h{e`x*=6qom#zvWJ(#VRL)Mwh5FD zA0d`5DcpW``T@6y6l!V5ZR^l;J}ey_*!gm4(E^kZCR_v6K-n{-9Et|1+Lt*&ziqBQ$XXl>)uE;ekq^JE{zl2xhx>V^#t*KS+K zP0(&@ExRQ?$zXr$n%Dj#=U@Uz?nRyL=HXx`y4PR$SGem;yYr-~-?)EOog~+FoJ9S! z^}+KTC^n_Om%rQps2kVDz7Uj}>*sq300^hGGECx5S4OgZFRLSaA!}pE*q3yI3#(9Rwg zftY|o_2f243lz7s_IJkF&Y(}!ocZ|lN`{4U@K+-xfF@Axau+YY$CebSMlT85x3iTz6X+C|GlUiRiaRrN50`ZGJoy6g(1VHJP#d@Y%C0_2v zeYdcGU4|6zDE%cm!D{w4ai~PwHdO55>o4ybp>NxXRH^@{QnUNOWCB8!qO7Z$VqlOW zNasf1dlf(7u?<}0-|N+PPrsxK%R}dMt#wXIJ?7yJFwIe&*6ct5cq>Lx?JcV_@!1{5 zxQbJ)?BL5ZN@}2fTBX#POz(p`#V@-&1#e4weCz*<|E{ISg{KUPtp!_k}9@K1@mB7?>dG`_Z5$0R*ozIiaia!mt8GUhq z$~EQA9U*yf>BGuLPvX+Nw}Pz%q-T)V;^sF5ss~VD zy(CckI%aWcUnxOK?KOdRL_cF%NM6DF>OnbFKnx7&sH1Oa-U2g%&U+c!W{%+fc|@ZG zC4(%NFXpT@8&G^Sczd)3|3bNxP89@WTy0DehHRe*kQdMvQ_?#%_3v1zbOlB&+#4n^Bg7TZuyFk@ec%HdtcvOyuuyy_98 z1PLHr`$^>|ztey~!)%SAfT}ZiL3!FB2_vRVRpq1)N5sK|07RG#oIm)D_~ze2iXy3G=N#aGe$H}bppmCMKC15urD zBYDNQzvwY8e425y&2uCm)}6k=6p`>XSWXF~5a^BTO{bq#+6H+A{qeP@6X&}5nAUNN zu#wG1-AjyIyfBOrU-5N3DVgPM z3?=KCa-{Ojnx35U%-EKTxru8&E)k9df36s%fJ!BD+8tlXH;z1b(E6P8j_&lu1UG#3 ziZ8MVA<1mE}kilZE7d-S>a7_8p1orxsQgIJ+HwbBgyuar`a415jpG?foKE=+Qi zH>gOEyM)rngbbfAs~q2F`i1cmdLq)-MqBZ%tTP;?n==}492R#!+*R%jtSj!lOF9w2 zc4kh5HvcqN0Stt3%=2$3O1;sIOWl7K7v-z*1_DR`k4D~9+SBRYjmHZK)JkY*{l&gF zghnKz|6Y#^4qHzZl5Zzv@i{V&%lH{rgsg{nRRMju4Jq}g9vostXa33?lm!U5zCHOo z&cJS+b>H$hWH@>g>YV=g7?GF@ogKeFu0s`Zt~pibL;h%{eQl?}S8J#7HJix_NC^gz zh6GiYtN(!a`*wesFswSDd9&X1Gru=7&HAXRgqd>P$-TWrd_{zh>c>jmOHMD@DY0cY z)O0(8iAw+`u6?|trmC#XT)~0 zqwlp9+cAU$BJC2qb>>T1FQflL6m)rc9u{Mli6NR{^ap(cWgKTpfFc=!WSsg2v~0L8 zi^j_z1#;p=lss3d2tl(sOU;h=K|{vWk=Iycyv^Bs8&VrTM_;t*QGVc2#r)#}RwssE zi!PocnX4lDe;U56iSUWna@tQaj<$co+iO2N=*daUEbNQX=wYq4ga)f>ETQ1O10w} z8$$isCm3D;Kx~$^!0e{l=ZMk*FmFOi^}rucr?(R@7PLJvx@5!maM};SWbp2*(G{UC zxGvTTSP%>q%k~L)+uldo*MzpAy3^^vVl|1Zi~eh``Z_$W1~2#!7afz|c9p3!wdVwr z0HncX!lya*7wIA4Y0j!j#hZ9`wQu)ZQ8BpmH|Raw{9>unZ`((JOkwc;xrNo(Y^r)v z5EMJob?M@XiSsYrw;ZMW8@Lt3JjFhwmDzcIi2bSl;P4WM(i;0@%aEfe72l|3l*g3t zXaWcGr22~jgPPJ1yVEw%Nik-GWC}egHFHN{c5)tBPc^j*)935%%%7D(Jpu1M87GB` z&I$uYmhLO;gA6yCiOeHf^O*7o#%OK! z&qg`>1%9l^TZA1Ee2OBqU7ZSj!5J_01=AJy>agDL+(OK9-}Qd zDy*aLP4MgZ-Rz3YweCfbCSeql3lES(5cYCWckWFWzhGVoqYwS~BK~bQqs!eW5CM8(&Zj zxg=~lFlwE+$wJi8MzmJb=NYb@P4jInnsIGy<4OJ2*xusTj*}|em|{l)$zXzM%O3BA zZ%w^~0q(8Hy0g1X8!kBKPwI(0zIdSh5T#3Y@pGOYS$ed!9@)kB6}eKyI2NO?NGUo7 z!WtM#kV?j@{c8b-;aIZc?g>7~@PhOlPO5q783-N(xeNAs!OdcE;tu}e=tLDg-UBk{ zI5@Qg(P}d12!m$+8oiyKcmk=tJ2>)v_lPLHwby+gCc03JQ;WM-dF*e*x0zrQ6S{Ze zo9p8-bi!*mfVdfN_=c3IAG%+IwC|3idF|u)M%Tux{a75CME{NOZTx&`<7+!`Ea>j2!4}ZP zlt%a*35=!pk0h@>r?=2<*^r{@8OsMv=?PcwSEyA1gy`*fIf>DBB*V{-iX9 zPg!-H-RnV30eQQ97F^viW#E}A)xyx0F7ELxiybA;iq$`UXD+sF>kZW6FYOnG_ zfWim=M^6?Xp_ca8Q)x`&+m&l?e|VP7b~P}*5QtMhss3|lhRPsV_uX5-mG&q<_ak5V zOzV=Jy~O0GH@#s77@x`2m9A1i`S4gY<;dM;Vd4vrsa{DsCC;RF7nXUl+qpUTkb)*7 zKTdq-Qt(#6!uV-!jLr{d62?4(m8O|+E4B#p3qudh6;#Z6G*`>rz2C<+jyK<5^b@NY ztzr1ZzUcyx?Bly>%HWB*Z806YB~q2&HZ9t2Nf#ipwV~trE!Uyw>ZmUa>$BUWI#Mz- z`h^t*u}-8Y!iY(CZ;uPk|ZX(5ZB^t`IQfO-e)uXQ+0C|ztXd8hYu=Z z{bXBWYX|#Z#$E`Z;`a)tSqM!Z-aMoUdxLu!fZuQv}SUI!Pyc%^@K!ES@c~@-~fT&+GK3MR#{`ZMxJe za0)Iq6gxFz+gB9M+au=-MMfLA-)y+lTTM5xv+Pb_+pW8tIja1(7X8F?Rl8CBk8}?v z!^+z$$zE`o+3LuM$v;aoY}R)7l8(fK*Wql_sLA9+;mP zGgs;m|9DZLqWXh9Xtpx(;Z$xE24y~}WmeH%6-5{16sZ|x>M2Igwl?%lrZz0k;69Gd zgr1_kl+wuPHh!e^(oILs{h?AvpGME6Crkyyk z?O7B0&V4b;FxRE3a_M(lhFBP#@RtB1MVA-1#r=$okm)#NX=8I^iBR(n&uj zIhw_cxr9?@#db`v?h#shxK8?lC#~9*Lj1@%p+D1rN2Pji-+#hAhivOqtI4_k(@+QK zRw>iV#zU7}Sab~WQZc2f?G`>IfGiupBzSlBK0cvwDyu|3gKUfGE#k^Amr4!)5#VuR}%HzxIn)&=tSj*{!GC77J9w%G1?x9}J`2UhRs3 z0{zJ|?BbM9JAMP|rF(vMJ$|ezguidRfa>$S3D$1aG^$fYHGOp;%#*G8PT9Gj>5!fJ zD3`@8ok*3LOO{dQ$jNxzOTp36l>D{iClB{p{G0CApGahSTFE~#j$sfU>^Br{uZ$_qsv*vtZZJxC+_{ zsS34kSPtmFKEyNJ6b5k)N#^CL4*_QO(lcl>HwNLUjTR2!qXh{%THEjLc z^?^I+M5_8}#rZEoeLL}Q$xL#Kx=_m`F2mu+u%@sds72m;mknKDg>nk@o6LpH39nUHP!sCv1Tu_@k z%dD)njLcUtIgNdvve}Tt~%S~&z2ldUoj2ACMql5qgn#V{O zKXdZ_lYJ4mzhZhrxX-;zy+3AGw4s@o{8bshtC*ESA$&x5zyG5vDsbj_?$-Ldd}hN3 zCO!oj+nl~*uX4jTfoMvOBRT^1Ahen@@2a=C>SU1fD0{KF*%YyLul(?Dxq!AYikI5A zQ!2rLJC>W)p0BouFKcF<#`0_PeBn@d0&gDwVjA08xW9<><3lzvE4PWqDg|_<{TkZ2+u8gD!dVu7akbNQ+2itVA%5pH;ocR5OtTz5bYBo# zRuEoLTbZS?ch?$Wr=Xn6Ubka3tJLqyp|dX)p8BHfd`16My1}L`WDgPJ-}tEpkp`e~ z2hdTtq~OQ_m9*A!&#H;@@RA_YaC+Bxp4<5K;m3$4;7?zv(pS0^m#<=D_&JxLl1JmE z5YapS=RFUH@u(D!M0ZaQ(dV=UPAu=M zS+a5Wmt}}dl>RAwC+X>iR54RfNn7YbjZb1KFK?V^rwxcV5%UCm;qi|lcQHV5`eIIdyWcuEX|NxMzk5b@IgYakiJr5bGBPu%dt zm6r}GPa1#|BDe&k*mvZosws42DrK! zM*BJzH!Z3klBOQL+SFK8C3jo%LECDTyT8hw$LhvNSfo(|>n;r$yMp9cuiNAwWY{aP zg1zOJtJtOS@zcUfn|y-#W@c`~T8Dl=hf!06=s+#a2VA-jahL30C)zbq$1D+p98~8$ zOFIQ=q9g{0|L!=v{0NRqqjWE@@d-uOsa=#%Q?(zB#`bLByKESn@fVVxhAPQ-{R^9N zTkpF`spJBg`E~qFg>GelrqYop4+ZI{O{d%^5mB}C-x>X9MNp_W=6Tb0uj7BVv+mKP zT(PNV5UgO>Gm_~^!*QH@yo;v zYfIyaWv?o8cuUW5a(H+d=bq))%*NqlEF!f2u)&#Zs`L_?Jc9#C_^RU7ZIz=H#}e)9 zAh|`6Q7NE$QQPdI1$5R4K0b|0A|Le0I$nMg+Xc^}Ym!noE!UMhVD)lV>sbq3C2t?0 z7F+i1F0mPUJbJKct}?VL9EfON&Yrm0YZe$X`qa%|#XN?Jp)wbTTO)5!n6Cxw^kjd# z95jO&3!cPYv?och%QqXD&!(Dxu(`S>V7zp(#xVQ?&e+VsUy)gRlMn<*oopnn=N-^H zdXV3JceP;snrVB1a)Qt?sUY{E#Z%YMN?YZ4zryE(T@xB|abb|$d>5LY#izmucSwlf zmf=C{!Z;?5PlfkSD%)O}>1Vz0`SX1J-h;8baggmI1D zq`*{VlbB})JHOqW#`Xs?;6T^Dv7UZ;qs|Vm1J8;b6t;l}<#eAQ3mJw2@&w!}xu^-l zfdnHa|6NR=o@K^&+ezhM`U7NO?A>N3_U+H}lPOISlUs33QkYdTe?D~v7LHWv z@=%qjy%giJ+V^Vx=2GBfuvQ&9)(n|*Er;oY;h_}~YNQ!xj_UhH_+h%!$WElU90_nx zp6?^|HgWnjHyd0$<7XMaUGvLfkdeM}`;Jre_ z@RwC~HT%CYEP|^IEq(U1eP3F%FsAWXx;Oi6G*=s2#Okfg;v2M8krrMe1z{fk!2NIX zrGLM=m!-UQ-kT8$vd6(h_+npscuAb;-6tp?Z|*P9Z3z!m=GZ&T^5F@O2i&LiZ6v@C z?LqHk+|M)0!#|On;lp%k<*oYbaoI)9S)!^9O0DKzqV?Jl6>1}N3F_0sr=3?{r%OUU9P-p z(lgc*X?xv^CS5WB@I`Z)+Acqlb?N?LG;>?ls>7bWzMOBC=$Lo_)#a)~{xAR^(5SU^UdBP%kEhDthlQ&|rJ$UP)WyN|L zhBc?|7@4Nz%?^c^jyVZaEI1v#Y12T6P*LT1=uL{fU#7LJ_fJ)|bKx)w(P8b5AUOc`~cnUA*?OAp5iI=;!P&v|g~g3Vf(dNKn@=jdpn%yZ@47a9djS?dEsJp~c;$T?w~}V8bCa=8ww>T@D-g zm;8zoo`&^b#)qU-a%cSSnD?Gu2%Q1!Xijrhng6O7CjSk|c`sbX-JO-oTHjZZ_4Iif zq%qv+sJ8EMo84ED^OXwMaA#_kSq>doD2w~7X&dYeLn9RL*DHMHKr46D?YT|hFo{9GSbOCU$c_3fl#;h6Wu{k)LaQ(;qusA>QMOvLn zKhdRc*#?wz;l?6cV)nviBFOV@`@FRV-K!pX>bO-!suumoC;q|9pdrM+U3N|-r#1Mv zxjN9Wn2r02k3v+&!nl~=a!sinq502tOKDHuMsgZSNyWWv5dl5Hi z6{pspRvk(Hqv|!ub*F>fCkNUY3+h+g%*;2m#PZn;#|4&~#U}H(p-g8mHbzbVu*K%} zCDm8N*$lvppuzf~2y{Ma#2F3>Kei z<}Yg!u9u4MG+}VpB5f|HS{RS0NsT7zMv-a8-=8REJwqGzmQSIcvG%rf`oXhyZlx19 zQ_s+Ld9bnUO^jN4KENvf8qj_U3oXG%;-k{9_lHljgQ06jD`=;rHdBt5En``I0q!)P zbxHgGJx2+klL=IKN~mxduQxF1Dbrky6GeSqw2Z_* z_aM~>A3V7cz1$mIJ~%pQ$ye9F$n9~op`Lc`+a_F=y4|>vIaqNDq@=tGTF<%lLKzd@ z`}oo#@oW3vk1aMzk`+{C!+4p@`&mj9{QeJ}BY0t{CK8q)5Pg^~p1<{hj3G`<852Pl zep*mk{YT&~d$Z7vBfHY1e=vXJh%j$fcTza-=3lH+so$$y*wUPvzqz=8>?cFs z<*U2QLFbF3a;}KIEcqJi;daXABYrZU^q=QS{KE&R`C&eN$q$>F?7_9?GMT7k z-V>?Cb>OX6EbTV=sGJ}?qSs>5unV(Ry-z-Xb?#%o^J-_wDPcW-Prp3iCE1#EE~ll+ zH5_}C<50trknp<#wUCyr56<)Tz>PdJw#OsZqEh!wP}I34Q2UwK&Nv4(6>fxSz3Sn;E80Tt;Hm>z|-y9W`7JoXh5Si9Q<>3-Fj0SGl-0GQq6&CLhNvxW- z=ih95pjG-+B@Ry=s38Spyie05ONXv@FOiwf^vu^QE62I*B|f(iXlhT-yj0zfmoj

    )bNtXB<>| z?zw$VG?;}cA_WMLuWxkpU`bqq^-gI`l!vzyJIgmqm5DEFjm;@^zl*oW_s|8wm8e*b zz0XFbT9w}8+|d^`xK_6-vkAYgt=Keh)4pg{f8qatTnp1$c}kL8Q8Mn_uNQo(tIlKi zpX6ZQc^`-|an(4vp*vd)^SNh=Ro#iKRpvBh@*kGgjw6S?q%KHqoeH6(_1wIA`lV^z zAiRs`A3r0$<3C?@`aE7#*py0h!ZV&RT$9)V_a4o83@+F_%Eo_IXpu`p#0RmnkYKV6>PRTk%i$*vH0e2KA$-EIE^&JXaojXAE*53ZKr9x)`Qum z7UB9BUT@5(waVq@friz=*QwcTSIWnOG4BIs|6G-zA;m{oOAc}4!>le3X(;(rUNgef z(7*5!tt5aZn8P0!173!kFHC$!crh8;jTxMQSIE;}csC5F6Vx;H$&(nH3E%(&HAh^MAf}e0nfSMQPOniL_ z7j57+Bi!(wmiNfn2t9a|2C1x>?Ls7;Mf~#%uyxQ4XbR0iiZG~93)7HJPQ|COV0;>D z#;*;}%i>vM=bScHgBHF=!NCGns4A2;tr8_sKh_4a@ zt{B5ZWXgYDXOdJtuC%DBe?Lald9&;{9%iclNek+#CCvfe_-`5NJW@!FZA`&&O&=p9 zUwlVLYHm&ldOFGYwv^64tn!6!H32EqrT>2?b9bz=kKq{R5PdaZBW0#`LK1sQ18{uJjq4Q*}wb*uTa%(>{4%;VK01*KSq zh^qcE(^@tu>pk>REghc5E4ZPCWk%EaO%C z&%%0tbPv5YmqdT&R)}mL3i4XV6jvmR@TXK!7qX{ZJj;Gln!(~06Vc5%7Z>XGw*|CW z{3(&T7JDu_+<_&!Qbi0h)Zwm?Xj;_}Cbifn__LJbIWH-7#rR}P@spEbTfxO^XYW%M zhJEnJEAHE}H`p5>4E?|@|MY1)YOBU;fR@a2X-nTo)!{n3Xe8yyJAvAW=7UAr+^*hFU0;)||N9fTIy zB@~>=9fZueR+b%uo2$%=%7YAE@|9h4K3Gnr3xsLX&S#8Hmt95P4}F2SFI?k!cZE44 z^2&Ay?B%9a<(R{>NER!X`!cultn!S|gQPK!EeGM-a%y_zD!WSZ*gKbs4pw(8pY<-^ zZBJZw0{4iaQ9^ zT8kD}ql$!cJZi)g!$|5ll7vYeP!8VLd+Mk=2qkg8GX(MjA-$f&*W^R5TcrikeH_3g z2RzjTDrfB$SYPI)M3L--)_uH^7i!obxP{DPi zM5t48>!<|&hzBc#kyj=3dbup07F$XBsm!&;-|?ih7;FeG61KWhHgd-0#CxaI2<~64 zohOXU9U8pb+TZb2+zY+0l&eo_^T46u{q~Ue|CxIAMORWHakreaG}#%Q%Wu`*Og7GV zU(<`Cn@pWKnelXBd)xB7O*ED&nM^4DsVG+&`L>C}E7;)|eoNuO5us;xlLaK?UPnWL z9oIsOax`n6NWdBgeD0uZkVvFNYZ%?+(*c2XdpL?3?WayfRx`iGtCGnq$3sx;Vx(au zeMO66%Z|@fLcKSiZ}rdp!ka9fSR9_AmJ&!TPG)LeAcVXh*qv(ZH>Fx_p?Z7S7nWz) z)ey*k3!|#s(e?>@K9M-NqOo)0su5>}F+r^NmaMFtnvw_?(x_3SS5a+IXoVT<|7f5n z-$buLmMlGF3C@o%cq8VqPK?AJsprrN^WyKE4no3s8pPF}Mx72q;$0I|xYfakYG_Gc z357U>Rwm+~cQ?0o5ZVLAvyHORs^qFRX=&JXjNyp<-C>)ib3q~29*v;gHnL2YMhrPvbt=vSuYW4(cr@f z8=UnNlqNf&edfv)#HSxS=HRS5$s<37`H)w=WnJZkdw)=f6Q~4HzGpHu=cCi6ALdP1 zOCr9WAv56gk*@9&ED&R5pq8^O508?s7~M)Fejy@&lnCqs11Ju?5*TNoMVw8rVifFj zD0Up1el31t94lNCfFJZE_M$Bg$??f}Y%#sOy>j30VgauF7cy3Jc`~NLc@mm zb8?LBF*sBh>XCT{wRV0tuIBgEOClz^!hqnpS-}56WzSQ*Z%VqH3wb{?>5ydo4tnPU zxyUu-egF3R#hbM+cj|mFzLvWi^Qho&TOYdh=><&`I1208d#|_`Ht* zfRdAjL*2={gxY5jye5M9Fzx%{!{{ykj`IBreyhrM>4S#a(B$UT4niMF_`CmYdt<}! zv8TF&?0Y&h^K-)qPt6Bqvdv`30^U!{lAW*_lN~5#lp;HEsikw`{me=8=mP$JDi?Wt zpa#P;VlYn}B(4JBW&+~lL7B{A@a#9uw?wkCvgxV=oB4M7kt}3Vvit@|LV5W!K?I|L z;3>H|#C-&2vSf0SPNeU_A;)l4Y=bTzbFMEopMuqayJ>Lz%MeuS)id4_(^6#Vsx^#o zqJb}O-d?j;t$TRbuU`6g@^K<|lER|I)?xgC5t-FXN4tI4sFc_8?ck z_s6pNjh^u1IPD}Zwz6z0QHJgOnmH*Tb6H$7o)*DF6c6r@K!6SodT)WI{mhGGYJ}Iv z!G7g_coQcvliHBmNaKOzCs7eL*ZUIhBH6^Vh1?Ut9Hgq~`^Uy{HQT9hx&FUXSiT-x%ApC;r_aezH z5*`hvJZYm4$ztvx)wS-`9#1_?{hdO*b6x)e;_Sl70nEZD-K&s5e7azHJS6&nIr0Jy z?hX=4@T`nG|L}!jp#>f|MKlg4`HoU`vDo%oI}t>JFDa7b*?2-Xjg7j)tL_sR)!fA4 z23JD&1o4a40%LCb>_Aj+KL-dDo6-q&IyRM3Vtl zU6Y4%0zY5B3a3h_CFR^*rw14cAhz554#zc6UOiEcHj1tR-a)J!uynF>Gtjm(L5vac zkXVJ}Py~5D=3bgQMWH~wV;yehqYQ&q*5boqKlP*5;s z`X$CJ`Am|30f|^+vYK=ms{$_?=mVJC$3(L1Ny~P_IR~dzTaL2&%qKA?v&>rSREbn1 zkzOFc&M>~dF3>-o5p){uFYMDUgU?T*?8t2ujbV>sTsYHiSGuKX-cIu3QDPS6oVyA4EfZW2Xu4$^yXXbD|MOyt_HljBV9W z6`249m?4$_7Z3xlgJsFO8%4&}bYl3;ZyYtwQ0-PxX`kA^+oQ_p*x74by-6~1385-` za4&r=N%(~UHR7s(Dk}VPdPzeDZiiDz89;xt4p`a7Tg6>H)D3wmCj|!yibe7T{AVh; z*4=`{Lh%R{UP?R~u#_Hh;B9SUj(aupz6921>-B58q3%Q7{#bHcIb^a=%!{q|0`7%`CQcJU~7Riz({dUF&@K;~-%)}AK|MpP z6Vq)quNDoPAyEd~Zbr-yWc;Z)i+Ff@&0EFP-0rD^+#qCOLB+7J0{)#VaJAHF?AKT} z(v`Yr>SbyflDqkG5@ggM7A>wpIw7u#q*V7aSJ^-QJIP#+3%@TSRBw}~2Sq{JXiSHN zCvYnL$RPDV$sdq;5H!BCyKVExK{i3sTToWE`yQkVVmeuft0<@iSmwbkZ&W0`8Hq}1 z8pY?Q4kVmBAl-6C3703W%N+{L$2-ptYO!Xr_!s~_mYIKk#TD0f#l(r)50*1O zT~}6fshz-2@bN`%=&ax6Q3Rtco!>Xw+yDk&7V_`#v@)#s*R1XPkO;Kw|0ka~6a zdfJPaG8moV6TDf9k{=LetjpsNUZc}^*~h?omwZo}fmCQuOonx^b(n-}IZ3?t4W_#PZ236ID--qTq5GeclbvmU%r!C#T|19f7bM={LI z<$K@Ay!9H!DU!u7g?@d<%}CWobKJz-j;*zV=OZy49x4J6K894zlL`2^25M^|_z#AL zXRIxR;0&gwh`h+Me|Am;a4OM@*YSZ%LB0eoh2dUNAF~gb%BmMX2lz)ubQF>z&k;|v zXuXMHT#4$qC6F(|-5iTQ5?njvOXssIn6VZBhjT-nLXa_9J10)*#OMc(E~FW4_y!tr zpyow~JQ9{b<=G(42t7}_U*5Jis{Ng*(?eYKObubVVF;gk1;H1)`_hAs*i5FhyV1qL zn_mH!s86VWez=1m?V;$Vt0F!bK8UlrJ+X$$yoR+V$RpVdzGVrSVUrMb0r)I=BJkO% z_;ZL~1d55oZ&JGEJ7*n_=(lfD$}1Lk%(0H%06I0>{Em<8P@p2|9wmtwi94%en3joo zs5BV`Jf6IO|8BL{_3tX)rCp({-nhh}lkUihBo@j<`rW%CNRvD3+-zQN=HxCtvKuP| zNIYrR(!Tx^zCmRB+hK=BhiGvJBknGgf?KLqy8EO(XPvTw#;&~3B2aSu>7@gR1*ApI z0LrjP!rn1=%VhYywzo8Vfkez_K2wE(bANl+7!(j-Sw4~|2#VgPke%2TlsM#>2O zLM}42U(mDn^%}D32eRO)0Fs^#4_|RAO#u$wk7Qv?pvUbXdt{J;J3n6>YPP3zAc%2| zPvr-S$1_O%i!FnFDWk38P|nv@7)5NtM)P?EpeFjkip85!G?Z>Kt`3TKiU>k@Ntcr2 z#P?Bns)Ks){v6ddC*TseBo`@*_fg`m*AQz7*N~vkU=p*%bz-r|l&0E^;EHG2hogJ7 zCu*dN>lLXcfPHZSc%61JbC4yDBXEzmnAxoc&$#U`**7>xwezv8^?kb+LEiUk*vCQ< z7L||Hhfe6z;xo~-EvoBw=Vec1^%8ZRv&%|J+Be~9bP{&_y^J(7RzC_{lIY+z4=tj@ z<}I-`VGYH;h+>$^M(_cWr_3@9AZT<{dA$!Xh+&&#MKY6opZk-mKsA(SpLEx<$y^Cn z4gkx||C00p3n8eH*|2aioZK-IBa-L-fWcVn}SELDwx)Jllb2CHe3m@i&x>cGr9Ixs~!M zOG^|wxxkH`PTJTw$Vx6q7Ax79yy+6I=BgXb-)k6Y82cgezic&j=wqQLOON1tK{+=X zpWj+L2-Kss&cf)H4VjJEQG?~4_z1!Cfu8!z!_~*+8S%dTn}^P&d(*_}T)uaQKEDMB z0M~w`LHBpvNQK~#Louu+Jzk=+1pSQ(JmX9iy~{1i%Eh*0F-nab-tJ2*b{NC1GBZkm z<5WTuPy?R>lK%5c)Rw5S8C1f%69VqqvsTC+|9xOtHLX(Gm(+n1R|+kgDIR!cZe^SRw}7d z;1&em1-gDV6g*@e4JNquZCras|!I3mmu2_8wnNe^b(RX!YgJmR@kpN_+ke zN`AvRg&|j zlt6_`N3vKGh+P?G>H$^=Hk26yRz|@`CzS8?a?UqmvhMU)n#Q*q&hVAJM7=7`g@9pe z89^<=G(sm_Xlz7mRswoTyYz60oQcfIC5`WJn*c#XDC%LR1XncX@lk5zthKr8aWR6g z*hz(MArpKerN|aCl=H|}N;ULiw!VkJdB6UT&f3!vDrVG_N30uZJ*3FGavst7@RE(% zQ3-P_&_?8bq2tAqnG~n{@01>-qa3GMUVkVib@76t>i+aY#M?422j6bHc9ILyvS*B> zQQ;hTorEx+5%Ejntqj?MpK@L-A>*grn3}Xmf~eL9A<3fu@V^M${v%Mb`npo{-kWab zY$g4;waJ-CY5_)}&t6?C)$H8ON*&Z{gA*WkD2AnI$WqGr+dDx4Jha4IECI7ORlX%xLkM2S>PMcfQAoTHXiHgre$Ng``C+UO#Tf z%h)nwFM(vfd1`y)$+e<9#vF(0WB#2seWeOrC8+#Sznrt;aTFq+VHge(W zrLULV-9kwxSkZvb=A>{4q$?@Los{c>y!(<4Z}}x7H_1eA)Vm2%hAVvAq&Gr=X3qss z%ZI$*`HOR832P|h_`UCt@YeCB?vDk`1ijIFpj0~S;5t0+y?on^xUzWvD01NIzw-6X zg!GOMi0ue9#H92NEiey6Cu+B^icR#ZYNp@eiUFO?Nfr7Ruph>k>z8L==o+C44y|SzJlM0I*>xbKB8ipr}PC$Vq1>q1lcQUVmYSy6QkL>A*e-!H* zE^(h_rDTROBbAFN7eq_a_1wd0CwYNzI#a@`n-!AuwhhFxQXr+>8N&+;k^;lb@8IM0MP++-^ot&?qrdT% z@mt^g{?3Z;HrZm^T9}sx)ecIrLxK@CD-D*|m9|IDBSIvWPqVHyJ{kM@xVB3677f>}YM!uoen+4Oz@ixxU4lLhmdnA5_Cq zn!eQCP6VBdu#5-q++!n15F&4}luzs{UuR55zOLgFrsna*>NC!J?Cp@C$r2nxuAoQ6_@4>i!6BY@q3nq~DerN>eBtm6*u#Q`uY>m(|fJDWc zpd*|pqn5K+7*%^nTL*KYS_V1t6%vq`ecJ&{84B}oF zCzG?le%RKJAo5Za*j|fNy}S>y9=!0XA^r$uwZD_MT)i18>}k80A($6~-0{+6T>DhH z))3w`G*u{EYE@%Bnl`c);H`-I_l(mxT>~H9CT$R>H^+UeV*&En!Rqu z{b+UcK~w&8PUYTj?1*4Qo4e_xVehcV!aJ`ri#6`$VfW$Z)xp#{#z~hsQAf`=ZCNL{JQMT4Pss0(=nZcMfFg6F79R(b&tT1 zA~R(|O243sb%AyG9^}`bKkgKq*>=nPf)x~SUzz6ij(RZ7+V`Tx0@d|mcE1L^^tM(30<+-Ybq|(J5AS4>HfrK@Y`q@59{K__?e~yDbZ00uR4!EC zK}u!5t72Q@REmf9ef}1&kj+`|1rPau?0t7!lV{g|kOEdquvGz>S_5cB5!nPms|XQ+ zDuf|Z5Rg3r1Z3ALDoca1M->GL8&qTm(O@KifXECpfUE#P_WGR!pQnA_|9}6z{q)m1 z?)$pVb*?kM=X*{9gm(E7=2H|D`j;(ksHnfb;1n&bi{t?^S&)R9orBo^#NM{9d2y7p z`D_1uhzwi3;#YUN@_PEzs+`zskIg^Bsk9v1;vG|bfwr)QhS~Dkm}PNBI-sK`tK9)e zkJdds_Q1|w*|W{(9GbN9yLxfq`sbHlj_SBR5Ez+&lW+y5-b}+3v@BrY$2A;AHAaI^ zF4R=7D>eGverd>~lflXTZIk7vTH3o5BE!;MpOeNndRCzXINEOA*uK|xI61tx_}qr= zv+AAC3<~Xa%N!T(FVqZu#1eq z=SIa2&=f*fNcF6%M9tp0T@%9C2kbypP5m}aGDJR56*bd zW8jZp?Yn}^25g0ceLmN_ZSI%xyK!?kN343waPdKzuF#_uvw>NCivWgc2yTpy%Z~o! zvYM`Y{7&yP#ci0s3(S2K)UB`3>2cFL#!$fja`4NLC*zf0`J*Zu!nT^D5LeKE^PmG& z&ymg}CQdut-g$uZCp2UOfvttiLwGy#dD!F$7s+#@#|RHDAPn?n%-WKfJ6<<3N`LJ= z&xURK;uFS&9)@q+{jbhogEqw^@eA$YcQ9G>89R40=c74{c8Co$c0FpGH^cWam zriE!zE{}SoxImI<{2#81bC2&{iP)|dq&UTE!<@&@0_(w(vY#e-EZNZl^@73t-mh~Z=D@JX24~;>Z1LSw^_U#=hQ;gjzh>#HdW<`i;1C z7pI0l@p(NS)QT^zh=EyuKGUUl;hCiyE$TI8a}WB>3*DIy%5*|k0_N30Wd@Iybn}wd z7pf-RWrC|#!fUz{f(v^6NToAXOQh`Md8)sE;r(xb5CQ-OB8FUCc(;()a{7uWhz!*- zlv+Pc?WJXqlXkEIb@gC*ClQ$GFtnTqaRZ%4C_8#spq$!M%&6KQ|AsST_KD9I#sza6aIa`qXxBmSS* z&vB}p?uqaAn=icjyb==Y)P?aLtI&Z1%7^${hxIwTW`I#JV*-NAF;34mkRdv z^K4C6d+pIH$3a`1v-PCBmg7#3+DUsw7azVn=6|_wryBCjp>j>ukN^j19MD4N&Nr|0 zTVKOe$omz~e{!QjoWRHe$u=zKso67Ca4;ic%)N*|4L<*r&b@gOA{~ssOn}s{Na+aZ zt7&ss>XzvJuzbVvk=xz}FOLJ*SI&9L`Rse!m{`VY_Qi%NU8y2qonArKsyt$aR1;7C z+#bE3)Q}xLcU4^UaJCBbDK#w1AP6}(AAS``{S{Io3(jZ&oS9y(da};}`;UCQkTFIp zk8J(y_d`2*SM~ywH>lZg(*3Xe?H_#%&4Nw;YPa_BK@bPhj#|Ot+!Z1 zB@YEgpv%afBstnN0o|&+Z9+x}1%pxkm(^n?-rSqa2zYh_6oy=vg|dmi{jCN^NQ%IS zD;zut4qm~08mK-iZ>{ScBQIPm&!LQl9-7ObW>NP{7XHAa6udB~f6+lyo>u=KRtP zqo7AlpT=NZVb#eJa}JceYoLWB?9qAkFBl0=0s~9%2cDLXl+N=g=mIVu)m*tWpbrgakS3p73@A69 z`KRW4!FGaHwJ_C>opO5HYgO(S-E;c1sba`!t?R?zSbkylPl*-OI@GA!Ew?^k%G{ts{}8o-rQ zm=7=-4K6`}sP93dbte!vzd z95Hr9l;_DPe8noa-}SBetKKoF4J-f4kj9nfp9ZRW2zne6TTpN^Lg-|37pH@nLZZo! ziB95>;aH4{7CZw6%R;nH&BrxbJuJ|&it4*!BA$`A(SVTWMn<(IpIW3-1BE#NQ%t)C zC?xSwO@E-`Ef1!^Qf|};rmLa&?t-@&GZPFaV#XC!|I3@sFjI0X0Oc4D1)qe>M-5I@ zQV&8VSiU2VBpjvzF~bv7i3SomBf}mhJ8;=|5|ZdFwkFC$2ORm76-@GUH?cXGYze5l z!AI)npRU^uTqdoBsVV0~<@59*L?8~a>}{M(7zCGwtauB4W>v^ffxmbpJQLeRY}Snd z;qGl@Tv;!9`1uGpEEr@NsV=Mp!XPFo?G>p85yK@0#;5@&GV^G2QTdtOZ50l628U93 zyTx+Xc~;AMdK6~WAdU)ELshuL66OF@ND-mNoLXd~sNADOU5PC{pMAASog%L7;xDfv zrJGyf`^UN4>)(bwv^ef_%!6b0a?HzE`QKPk3~%Si6dRBSdwS9No2!GU$I)4Lk^+o; za4r4R0K_||YBPsFrj7p4vUN6>GasCxaRp4n~`g6)!SVmaZ4t!F>xF9My&b&-60O z4-SxHX=XfuVNM^$?n5l&Ub?~eP3B`T$1T37G#-I$0Wg<7~&5U;#+r!A7GQXds$8R2f*4;-c5+{zl*m~AIFr2Ar!s=lx zt@)j8MYv^j7fIOy%X;9c)R_dGK83> zed7ErJ149XPA54nj~S{F?DlkA=3O@D?j4X6$J{K0Dv3MotODZO50!tENhulxHFlNP zI+U65i@(tL9K!S9Lg}S8`+-3SZ%$3XA{QI<}jj$*L)^42Gy7Xi*Us7q-ighOzs@`qg&R6 zx4lR@C$wj+RD5B^ozu;?V(8mK>Hhp5kh}B_ytAEuTB)x3`;`(lwYznn@!&!)1?m}N zU;?&-O{?@c-&v6l8+95ny`%L!?;dlszfP-oapEA2dtHBPQ_&hmU4(~Q(EQLt=dm39UHn}=)R0r{L?%115|?&N5=1RZIgZ9crVr1b}{wbGpPCA|efm$fHp zJz?$01uC>+$uFXxxZZaMRY^hYOUi1SY~l(=P6^O{je21gS_eTn1C845Ks-a3CS5nw zueKwpZ>+h7RPxP!g3^GCbiumKw{&`oraIt2qAQkGghC(7P3Y}b=C^|z#vS5k6n-jQ zSe%{STsdRi_n=KOThT0UynpL@MTq4jmobJ|x>=N?^Fop1^!>H)(8a;c`pb1fJG;*- zd@yjq)r2{9-s}kjaWUaq4psma|5lAFJ(>msug_WHX17fS?@&Gs#O&pQBthXVYwy3F zeJo5r^9T$a1tBK;#HMy$s|LTD>(*-wOXcuywDW`3`*4S*E-t@me@pR(C zfbqtfP4b#eMM$Zb$*|Ee41AVsj48ZY$}{e6b;*AK@kzahc50w>Rl}0BVTEQ4hSi~0 z6t`U0LgE8{a;jpr*c*iwO?d;Mjdvv@=tqEdU4=;2d|>OwcH)Lv*i`|{%tSv5ll1e( zaaFT$Z$@NZ%Y)-lNWo8BZv2m4##jF$dNAsRl73{A0Gy4{@lGxkI_j*GU|QXvUAV{Z ztam+P*T6Z}$z5$+0tu-uhL2y?yK!zRz($ujvj#;9516RRDvHq#Ir4lzE6CvZ*LBAo9=@yrKLFqIs$fW2p?sQn z?ZX2LVCzcI^y!w}c0!iKLdWY_!>m6I@DJRFeGiXuk_@4>WNpRK2hDuPv41W6k~L+! z;(z&qlD3d6Eae=glYxGAr+lxUV~APYv!rboVjb<-&w!`GNI3Se=JE}&jfV?;ki(u> zDnld87=CZD4G`H6x6BD8E%a_dn+@88sHohWFc`OZcE^A6UFR|j7x)%n_*VaWA$qL- zt)CrSNM`76*7N!IiabL@?lK+-(T|aYp#iHK!=^x3Yons>2ynnS;7Aen!#@SMMQsfy!p==vj+hzZ;xZc_)1juW?VWP;#ty~JyXCeqJWHIzzY|53N+g1Fd-vUT?oAq)Uj4-laLt&iq<6Ba$ ziHF9Zrw$)&g$I4QRca zSOQ!O%sGBPhdj*ORdc5ZLh^AK(d+`i@+=Sfg!f3so|r!`wCSF~+qkTEgN94$WGuhG zgHpH@CmG(Bdo3MH5&%PTG-F-g710@qC}?VgPwbQl4|?{qBGA?Ud4YFoiq*Y<+74U< zHBCZ?Y9SoXhil6Xcnv7P%Q;Z(H>v{}`dv%eKt!uQ66HmY2XPpUoIYp8U6n#M(9_aSbtJX7-xnV_bd*7-2vFT!`cvP^*EqE}>Jc(r$O9J!c3$OOm-85W z*Xj4O=;5Px{_!h?^DwW;kq-0dq!B1dkHg4DW)7q62CG zxDSUIU!{n$sJ6o?m-DorDn*xWhRG!t&|yf1NN>46C%gJj~(}HoPAXD zL@4)Dv@3!gIl3-Y7O~p&&}cwNC*3Pmu`xopbDT&P6i&Y%{m_u_U#v^C+RznJ1f=`t*SMsz1Ae0*MWlEK2q*y4YfW5>$>XK{XN(aIp*i4O8D^+< zn3wy{T`HV`@hndEn1?Ymerq9Cb_)Y7dSgJ2bhh@WkLT&xXAmJ4N>=pP4eH6znjOVm zfmRAHxtCG1MetX`8MdhQ_lvzOsgYUXLVqm^);N8&rf33-nwQW zrK?cH?_w&CA;|mtfDg$zOj_9?M}hR%kW<|l;_{lp<<{KoN$>CXyZ8sRp#=Om0kf&s zs>s=?wA-h{jG^jk2zLla3hxw}j@S6n{G7r7PQs%H?B>B<)T@X8+$GFv#6$agR|*nJ z;7cA&eaky$*Wu28RlWWPv?Dn@ex1EEU^ozZAXvaTY;vu1=G#n%^1$Jv+w}9J2)nhqRKs(LLxbbXH+Sdk*NZNZ zM;|{M-)@k77sUQ@RtJ}InA5DoPYD7--_G>^P!58WLhSk<*$==20qAKy&>4kv)Ue0? zR~JMRhr0YOEkDeao_}IGtje*P9D?M~+mZIW!p=uc%nA!GbA&lyf$HG6{&~$pt8@7USLrJx!LJk(YhL7|T-vk}6UBG3sk?tTbNV zD5r=I^NqZ@-Hr-Lq*tSsqGiP@Y7Ex7$^4`?NDDW(Q zdW3-eg47Q#_hY11Hc&P3bWV)=6mCx__9U*+Jd?sc{Z#Z@z7iv>0LuRwLB>kmDq-5p zr2AfmskGXL69G*YTv8evL}A-iOe=RWyD-%!^(%j6<~ErN+oM*r(@d^cIY-WUxT9Wc zVoys^=3@gfgs~IiJ?2F8ey!NRBN0&_`l&V%60$U#!7sI`{6< zpkBnDK1PX%4TOOtyQKOz1>G5@mCPvz!ungm;XiO<(FWMHVv~JUD2jkgx=gf9b9T8B zrJ|&BXIWd0bk{wi>ahsk`!WqSb#p5iJ$GIO-Ecys6s16FB}$fjtI;3PdD5PdCW~{t z#%r&YTML_deB2k@>D0hA)KFq#e$$g*>s$#=i%@x-5)&oMm`2JeHF!0b#F{6)Kb9av z8Ty=7X0BW4rWM7&3C_l!^CRpK;P#>EDtBX+^eG%zSN z`7ma)N0xk;c2@CrV#Uu?bG<}`gvUkPoCpLd`pEi8f2_|z(9Ot#?lZ_!`>{S((~c~e z$Gyj>=Z_|(1;nnOCzM9YM*D_e!xJaUF!>jmuS_e25*vM$DKe7&!-B&M#z_MMY{-KZ z%O{Ue8mi>wKE@stb|DYr24c}8#%@-nB%)R9Pw3n5W~M3#g4LwOz2%F&j2)9GQ>1i& zZ|>R%4vd_m(GDrL6c5b5MOi(jGo{t-$CfWHx-L*qTk0%xQtuMIlbS=^9c!LWe~dYC zBf2g-K%YN7WM+aTHCTy8PijWXIaBlJ-nr-xz1a4(l4UVRH}#X`WhP_KWEUO9Yb zZmkQ-`6S4ocJD2pu&Y0PtKC$3U(G?rpZ)UUBDR5!_`AA=l}{&IQC6v%wPUSEmZ`Oa zD2+rq?|qHr9AeWYCK1*DfbQSu*=e>U>2FKuNiFxW!D2C+l9XD8bX2ulq^B1Cu8LXO zo;l;hv88VvF|vJML~Yusy?OSdO`!@2Q4YM^5Tv9oFJBv$p1D=?8w5uIEk~Q!#+yai z#1q*#1~LFZ_Ya#*YT}jNF7WGqMo%8TBT6fwIN2d zB@GwKm0WeXnif6#6t#t-Gz>ZINSheOREI>)MbOTUI{9~hd4;J~qTrH;3xoNj{DTPT z-m4@7Y#gpL2rX3o6K|FDZDFEjc(P8@cd^;VM z*pPJ+Cj}NycimtIWhsaJ0>*PwY}MbyU`0y5P2%G^n{(72S`dX&tqAX5@%ic|n?La5R6!D9QnM6c_toARfZdYoF6SotEI-xF{d7w6`#=k*MI zKI2%t)LM~xhY!z&h!Hsx#)Nf$LI-?mRr>)^HG^24 zN8H2_`gNGPn989?KHc&3D6Ee&_GrR%Qy2qrsk;mlTtN)>%TJH6jnpEn|BjEo@R;6C zJ7hoF%|P%*qx^O7cTZc777Lf8mrYu;yaO?+GZ=mvPRug0l~|PV`+5Q4zlkS^#N^m@ zT|)AZGH!Jv2D534HN6|IzD8I-Ls&9y@1k8xa;c*nt{A*pppZ=zurC_dP;wWx6`4SC z2$QH5KeH7U`X1ryYWiY%1Qn7(|Hx2#fWrE8dCko?7c1*+ejC;HsEn6bX(X!Vp?6Yy zpJO&(5Yu`@uhYna;uo3WVBHpPq@Nbv?7;Kf@G`(dDlSPa3ut7U_pIP;aXnLPss&c# zF6A!|L?1Us$c z<%5sr78kS)e1`tPIQMUyi z@3f{rgM~`oh;Q;@Tpko+EQK}2GslUjQr~byu|~v>zE`xfU+f}f36*8w>TQJHPog#r zjdQ9607BkQw~#EErO$Rm2dkS(`0 zHaM4SFaCiKDCt*b%p z78>v)m7frAUM6Vhd3oaYg}Rn%P-L)RDrZAQw3*^L6Mr%eI>SYV+Uu>DMopG;$}hi? zC2zngHlvC-{SWL_JWlNk_LHTDbvs zU0`mh@m?&O*cMxv3jrdzf|GQ<(SDQ>%1*e}^+1qVmzZO*O5WiJ0!qM6U7}LO>YVol zf^+#fmqg?1o=){Cy-H+1ef-XZLX}2yaD~v*Pi8q0$zzhn0kz~ePl%RnqBgvJDRgc8 z-3tYi$U?jP8RRF@xO!m#F0l-rM+E#n$b(R9PQ+ODi|+92)y4W);_qUGY}E*K0YGp! z&tfe;i`obqw%4NK@na?>b(p?Z(H*bfeQ9QISM`*&-8dGzPze`DpueOJCp}CC#Z(lb zNpWu^0o3eqUVJS}e!|#OC^32p2Ve%_tIOK6t-3QXT$f28-n3)y7^qB{VSzOwG0E(* z-TwSZ5Y)%Au=I8C=ni;kitDyGqSHLys9mT093m#b7Mltw`u(W6xpD%X$k<~5#fGH+ zS$woY1*Fu6u-)y^liS;AvbO7~nL;Ae7*nWzP|16+_;X!}8MD*H>}v zkT4>1u&06(5raNWQNo9t6oDPdt18{g*{2Ww$nj+=)i!di;vDHG>NU0^Q+@$OCfpGw z5?Oj&TO@Y(vyTU9+zN~`e?j#}7FOL|fy?bm(eW&F`+hh*C=n5t>JDHm`&cqMNR$!L z@9xuDMgMMjjP>!w-~BPM$Hx*%zrG&P9roA1pX?dyWZr@2@Q49SHQ2=cDhUc)j_#tA zzYn}pmWd`MsnB@7xc4}m0Ns-?2Fo0w<9--yY71)@o!2Y-5&)`W`alg2ovVByuIqTY z!7Zz%&=(ndPM-UZbhla3X-W8-;^8}v)?zD*LAb^hRP1izgeL8fA5(SE1ZF(i0E&RHQ;>`-y1DwVjAI zFC)U3PdGxXRHG-Py{{)$?X&1pN*D_KyVDwSgh`1?e82VH@Aov7z&cU09gQPYYDJ31 zVdMybwRc+$X&11f+Dsodb8tS% zz(<}^9~&qgkz*!yk%?~@J7*d=CDyoCygnxAl32|g#KmTF_4HW!(-FCV$=fuJ&R2C9 zhM77;pV`qcwiXN3O)LNay~QY__CyR)QYBy|(2~%s?{${2JHAFqYOxP+V6H5=zT#Hy zM*y?L)hO!Z`S@X4!ui253IvbB3-!fEs$_NT8NOYsi4D>0I?8%+{f}KzPz6a9A33X* zKo3KOBx`aE4WiP`)uGfBar?N8!oJebmsU%COs9?v7g=rHQoBs{oAUJK-)&5-Ye6lc zR{kJpm~O6SfL9sMH4!HwQOy|U6v`@%u8B!|b$?0Wu!B*%rfp0oAO}w8Nax_1gTcnEGjV$#U?uWZI(@{$NW}Y`+3o zxfKwlw8L6MIWD&aZM@)@zZVy^4J=k^#+X?u z%ytgVNADH^$K)^gJ?IN_1y|zdGojhgKhraro7>2VVO(UQ8^!{@_)EM8nt{M4EnX@C zUYGpda~*Qm{RFM7p&`^RjpVV1((n=Ln!wFZg2Ol5pGiq;ENywLEK^X~#QEYQo`JfA z2Alao!yruppa7Nw1rR5yrS60VBsZeRom)4(tqe>~gKKCjub!{M%`VLDZ&Rk-W86vvDP(j&2L3`{4`|Qpk#kl);U=#}xEJ~*Rr1g}u`(ho4&IG#N{a@#Fk*#n8<~@zHUbr_4l?5%O3p6oLf-aP1Xy}q&Rg+nVJj8k(aeh zMcOpRja>_Bapjy;YL%(E%B+NKWHXpghu|=+r+h8}1tB3?)-svC8!$LQEBUZ%c^jZC z#-5zlm6>y{0BvcVqSl<|u`7>)kWwsAe&7_YvDe{LMB?uBZ8(*FagfJ43GA#@mMjn> zySd@T>FXRR3xB{D8;Ivmc2#S4z_y#~cbden#lly+$_Uv8)$E;?k34g^^aOSq8MC<`X z_UUUkb9t2onKF6*~sK`pZ2mORB3R*u{o?CXH?HzUZ< zzD&*;*_HjcX@?1}K{wn{DFA(J*btzGm-oMnJ)HxPL&aiDVeCrJZwX~tAE+BIN9EZV zg~j6_Vr9v1WLa>=XCea5xrI{Tl^cQR4ET%S9dd zWxHw#Y7oY43R&ca^|?lf0C+&Tg<*U}7D-Thp&(Pw>-v5JT7x{Hm;<(ziWGD?&gE|N z1oEUTpX3Sy#Zq{!ESZxQmI;iDJf%az-yI*vD_h!FZ$TyG%xjPa}&sn#PeQf96B2L`&-lEYHc3A2QDLTrWGi z`3l2eG!=6+y&wWKhkC>_gkUuZv{nfLaQ3?-iyl#H-DrujSLk5H;tbNKxI zUx^jpLqp;fxLx5&3CdyGp(LtL#&8{`+AnfY{tmA^*-sf9OeJXws-tVHn-c#D_Ch48 z0)un^JETj*IZh`glnGPWw~!#ix>e{&pdyY*j^2nC76p#j4q}!M&){FduhC~>%i?AU z^h}Tpi7I=-TBaY&u1|^l21uYgr*K)d+v^;HbzZ+O@aDO3s^V?4;P5Nk$PAi_auExO zQaw3?nv!GO^_hSvCC%_)XWwDpsgB}qxqlXsVlPrb5E)0dkS_2fPmMtd=JyhPCSI0Y zL|wX?IA+-XoOWoj^D(9gJeHde<*x;G3+MKgQICLr_s12e*W{qJI7?D=7a0shE-eoO zwf2o-*Q|B(4iMeVEUw*pV%x_Pi!eJ$iYs*t1SuBkwm~TJ6erubp;IEY@(OM=ABx(F zb5pm8s)s2o=u%rGebK3q_xsB%FH2rO3njsi0P$i2|3>pTvtqksmQ>km^gA(sO#&x4 z?trLmBv1J{lI3<8H_D1Du0wx`cdFHZ2RGV>bdv;!t1yh|rk3kw`3p%6K9;!SSw&Ub zyBuKc0_jHVqSp&F&nzoq6xu6_3$R!~`U-X-g|VmNG8W={U#cxH1Hx4S`iCjm?E>%u zVEj(Sbaq)#U(WeyXb}Z%4Ee;>qZ%EkE-06HN3UMAt=O)h<)8#6y)LUkBW6*!onJv!K6w~C&P7;ioy2Ww`AVrA_i2b_C`}iE z)VXWGCfQ%2>xFvN$qAD`8oGR!%91Trj5iM=%|=gnQcu_fnj@+7e9(91&Ag6RiS^BQ z^XF}nzl$Xs{@xrhI>{Vge;t#nGwq@G821O)E ztb|faCKB5SDRJ~~($#$E&AA(djQZ2jC>2$*@|Rd&eHB;h>b^936b8a?vHe=H^2iuZ z#jPnrti>Z1mF#@7~BBEHM&0!|=F{RVRFEN$sH%elnW!0-2YCisUbg1#O;IIjn7DrFVJ2=ix5Lchb zk_{D~jd_$e1B(5Mtj(6g;oogL(~m4aX#9#K`CfH9Wmy>*c$pAEFi)UU_t#7YnWpG@ z)H3JBnWDO0?t}os`U-OenxU?wdt3%b)uwK%Y*^p{f@(Zi*_eXzYW-3&9V(*c4!*&VNM@L_G6`|ZGkhKQeH|OC5 z7mXkownO7ao7swrckq<>ps4)dz2wAh@iiXoLW^3_ffWXCP11Fz6vf$ZZcS>tHH;10 z9in8#z9N}FvC=0uBm=Q2WbL1c4PvsaydP=`g*C!HhnV)5K7j%}F(1&D+iq+8xbim) zaiR}3bCEfTf#~F+(3f4ZvMf&)oUsT_vG|;7*9fJ$E@7(*{Xy;xRO>>$y`t6(s<|AV zi|uIEUqISWQ*ij=&fZ|x=m4%ws>RE|KnDU~-#%2$0V$ipQa3$d&F*p+G;#|B6DNvY zHWx4zK1+hbLC^{KKn=Rejzfq58!J3W>hprQI|nWzB%bbIGd$O(a-8zm^pV_VCJ~rn z;0T*sniTfy2`_)^BhWoy)n+1503WU#27Fv@SXKSl%QM2t~nw$IeFpa^Pc2P9glp(yk@v5@+gZ-HBxt zHf($LxWYPao*@>9sE~`yO=uC$1o-9@E8iZ_K&|u;%b^|Q)b#RRilT=N?zH0w2@$~FCq6Y&~az&XTV70Qf67v0%Ab1D+mA4hM_ znb5$tW3eM^7t;WcF;FuQQvN1X0`1b~ktzTJ<=n`c3s(`H69_Vm1?(zL%LiKDPmtRb`<{X+;d5i^d8EnS#kX-WvjD;USG*u0&eJanVw;=2x~7OfiRmW%Jas{e)eN16-etsM+K)K ze}tiC0FMm3L={XSAJX@CF>(?v5PE=L-7SZ|@=qo~NIoZ8#?nhq74RO0^Np-5*ddI9v;|lc;ueY#%1&vSq)F-8x0lm&#^HlNeC=T=LQf4!b#rIObb5 zPgG5yGxL{fByq=Mr;oDp4ni>*u=z=7QA3IbpRls^!S-0|3UF6ZF%j=YtPyyVGBy@!bn=l`WV;VfcWGJR|~V1GhH8Nb38 z-;et24C7N(!#yKv+uI{*bDw=bhy=;qHxZ-@EFOK`i_lh4LOSZvP0B7iOaT-jR9Q}3 zEEo?`p@l<#F;SVV=mZ7G6V`3LN!Mu|y@)8XU_F~ytL?&AR%b5xOegi}hXaL}fqu6w z5#r=!_PJjWg$Y0`@=X2N#;x}q9~Cm4tB4ao&*!gh$d^M!ikm?tmbq}GFF}sjwgcfO z#3nuFUeK2qAevLn23PZ65F5|;Jx3r!voPn+8sd!C)$^_9dS>JamD%WbpSK<9WDsl9 z%r`0&Q|K_nrzjSpS+-CpLvc2O=S|^xZ!kPy*7sf?E#3xK+tJt;>*G(zslbuM3Rm9j zwv9x`_O_*F)H)k4+{u|JM&+oVebFYOV*?SSSd*>8?bnR3uVg5~6*z>jZ8TT(JaOXw z{;Fx$>ZZxVwviR(=Puo2M;k=XIk2~{ixa5kX(UGgM3)MD0HKbhYPW)ZYwWZ>^L|xu zh4A(0kwV%{AFBE5Z4!~11jhZzzU;t8jqeyj5hVBEf1X}eR!l@P=6WiBrC8Q9>Utib zK)d9>JF)oLuis!NrzS)I#gQ|>Hd?CGkl$+WQmaqE`l2EQCLo|PxkLqmrr6?NN73u= z7e#cc0Siay?P48$X@VE{B$V-Vdr!|9ob>JKxb4H3orf)MT*k`ic%8)~J8kuZJJC34 zZ*xPeOm9Z1j~$_H>8q=2642U=Nu;cV{Ry*=|2W-OrYNwpQNCIPz^U?YJY)Kr76QKc zDz@m-I3z!?p9?*n0UeTD3zkQ{r>R(T!8JlbA0r9nr`D~UoQX(UW!Ol_7UA^A5=S6E zo1l{c*Yf^fXF)2xTRKFN3oszq0xF(*DP4VmK|V$h>AWSK{58(*KxC>MyRx_5ZTgK9 zp&Dq5N0|Jq`PXTvYEDMvRE^q~cHFcml{%`}tnFn6ock$Um(h1`WC~9#(mg`228p=8JTa9*1K>36Ai65Z2P8k^;XWv{DE|)vct+Ub!2{} zp+3=hwZfG%IVNCc6xHD!Zc+kGUx2{jdW#yl0QpW>oz2ws*fvx+8Y3yOVw)-0HaedQ zBAWCU)P?z|8GSE20V-6!lnG%d4jXLFF~2g!gS@1+Kz$GCf1;XonVJn_5wRj_?b3w? zwvkKNb{aJ50i(EGVjj<$?YD@X-cXqVEDavPNZC$~&s-*dKZz(Ts|N7G4TLxkVf_jL z544PPQ!EN{{zWEG$q6`I_q^IXjo#oRPE?}e^QAA#WUZy3GNbk4b|sS^;vRcKaa>cXly_u6-ElNg!D^ZZ zWyxP$O#o0rq*Wm=FQS%w$F3ct&ixfH0OYpR+*s*?YC%su+xZ?OR%~Pa!pAIZufoy(O~llXPol9p%J{~BUJ%cn!1(s-(cx{ z8SpeT$uhcPB4syWiCV_Nlb3O7BV$(doqsmF4D- z=r0+2qBeoihK^UE7nDeO09pP9(ns`?S37dE_f-zE_1(C778$l3BmLvaCh@LSPWDo) zk){GDIX@pSptBuxujM2b+pObNn^c_JCPjdX;6Ev$aD;tW80|O{i;fL+$M5_?cI3=i z%0xlhNUC~6Y(m%8)VR4~zD$0xgYiCx(NjFt>a42kedLa+Z z?t*M$*7~{;HeN!d^iVUgDkyU94{-JGfuZ3jr2NU^DB)HqCeT^a$l7i&4_?6dI$n6_ zrJ^U3(v>KLoZM3UH1d7Z%0R+MF__&zu+z4zVzZ#@<9x0HIyl_HTq7 zbU+Zfbq82<9F4|t16+3@X&7@-NfHBWi5C_ZhxKbHroM+VF>&|54kg+&1&qMV3=)Am zNq(b531Brmxmc{$vDlP8QTv$FLdJ5_T-(UZ-Z}(I6Knq0t_G`#fIUJL!M18(FsDz{ zrqPY+8*CA@ZIfo!!WxQt@2IIV|K18hoRFvvvjrNKhiA0CLV!5CmD|z|bk+4}@M_MU zF4)NrQt!vr1DdyU=z%|=cKveAO?bAMfjlFe#VZfJI*eBU9_c&v5SSKZ(e3|jLt0P_ zc0qT86bU_k)xJ|?*vE-j zw7_EHd=M*dS&BHj+g;d%4`Lo@u=!(8(Bi!&yR+Uba=Y*bH{2~zdU|#v1H5E#)|1z; z-Ip*~j`OgT0+1wXQ#WoNcV>(=YY}13a;cB4Nb!C}vY#e?r%7S6WHw`uT81-T%XCUS zmDLGH2saJzAWRvam1;3ofo=$?!yoj2v1dI0E;R0=S+0=?ls;top z6Zu;Jys?{k=rgx~p$L8sb=aF?G$O+11sb7`bH=7Ze*vM2FJp^L`dtv>Rt^(((9IXO zt$cw#Uy=j&`k<6I_aU)diUQ2p?7j;oIZtr$I&8tE|ElFf6#fRy>>FM_gb+bnY35qK zEctQQid-msOaJ7sHkJm%8;86ZJJ>1=wKJvj)gnUMbBJ05=>?&9t4m%%K(!WNyfmS& z&wL%WFu#JzsyO%H-LjLNcYc$_PWIWQlUVy)T!Ik(4LTZ7`|$?9LL(ZCunvU4d^C?r zDt(l(XLklJUTa&fyX@z~fxTY)AUHfj7K~W#U#c*U0M@boH|%3Lkd%AeI39*tps{gM z)p*QJetV--_KQ|5N~bTLe#|yfz?a7~te-XhwetmW231ypuK&}O&6~oJ*zb_I<-lEx z#?w&xfW|o8G0a9;a!>iKDIkNJC0DLPVfl;u%J}>Uk!4F^-VEgSYZp$jUdtqk>3fAh zQt;Fz$iN2$qGT;UiQ2=j`V#2kv5^evV@5O%KO;O!rA5@{No1bT*WRp8Wy#y;jBsEX zPLi`v_1GCgKZYr88`(dB#Fdc+fdUi9Fl8`t9zh?=#AcAB73{Y4Nq;iJ(!ly^y%|ei zdmEp!$*TA46HMzsO#j4(DhrOBn+D8PFKFz|Q!Z#5c?>ewMP>m8Y47iTU|9nx-qFuH zXLJ`^@fuBfJKmesR+fBr+mPAv7-DJh^g*B|C+cI@T?tzvw(AtZ;peolCWlDR5y4^L zna^Q1k5L4GTTwA8jfM01Vi#@z4Sl0~59rX+QX+`xG8?Bvd`~~3KSVhYQu2rw z0b7SnRW{C@@vR;c#^eXKj?EJ)4^rmw8U-Jl6r?oXtN~vq-WL>i)8fN>)fUmvF(WM0 zFVT|zy7)LQWsm};x@7aU*Z(UkidfJ61fcD44zU0Lz7%jMaTGNTivEmx^NpXz{)*bH zLL&sA2Q~>A;LTyIPY8aeH^c5z*`)5ag7JT&A-0$xd%EQNfp_Jq0Dg_JN0?!aN2L8e zR_yrwY!G|S#3vKfV5n#DQwsxOoRpWUx*j0}WSlqYOc?P0Ts~TA52m@^W z(2T2wi-W_Z_p2>sF1xHMSTkp=$K;7b7#mHmrZ#Z1 zw@1i=7p!i0;t_8yIS_zp`BTU1|Fw7hZ%tNFoDh>IbDE@ND5AZMikcWAu!Z`Fsjtwm zN3LH0b%r<#GGxGHNScPz62H8&AYl-d8ka9J$Z#+S!f%V3d}#s_qPd6Zw3i$K4sCYs z`|?Zw1Lvnb+jH;E{oKzv=e_5gyU7{&b=!r9pEIl4^}kBFh2JN}5{=9q%x;}zLhONg z(5Avpi8bKs(&UX$&8tMHzToV9n6-s()FLRS{{4XJ?wNCQ>c5#v7g4iZ6Qw*jY3GOV z`k*O(8|NWqIbiTD<=A^=a+p$Z-Yup}Km9WrnT-UDP8a#@dk>#a5qWffW+KJshkJ-q zuX(tObV@e7E&d_njpr2-QkefE8l9;LOKMCJ1+;MrL=pr_p(3NTKz-@v+Mb$;0>V?Jg}$K`sN(Y86*jaqy8|;{05}hO;35=i7c_f&5ipr_J0wsJdfFaLF2_% zeZ+DOi~J@M2!dv>v?nRucL`!>CziNn*am2+Qk{E3_qzd<=;RgN^&*+KEXrI=^}kcu zqa9yxQC$L!XccfzA(6?eQAFH{Vq!pXAdZGd?rH;>U4jP9)K9(Iai-gSxTZwn_UO;m z&(0QI%dU-kRd`CVLBXMS^DP4NxhPb)3fklZ!I#%Ela*@Oo+z`9QYx*vSJIe-+OQ-? ziYVa&KYoOP+n6CIsqi$Eud*kg!S%{1L*^nlMCwcM5g2~;PVBWq9NxPG1n=wD>x6xO z(cbq>uJocPBxnli)zZ=wb?&D-e+Lqk&fp;YdnTvLT6Bgn1WG!@OLE{ioCiyNf`17_ zo{);K@0_)-4s`e6HE|`;F5xVOLzn_eZ}Kh2ot4-Lf!r@OEUjE~>#?Cn4w3&PAGs!O zh5d|x|A(j26_9?Ys(7Wa7#~ej%g$ljR_HlZl=2q6HG&@g|&kA(gz(&7To8pi1i*Du? zU543r>^7=eDc=%(luDPQx-%F;O4A5jp@Suxbjzjj)?)=(ZTkC9aa&Qbf^5ZIWWI}4 zaz^P39hsBcJEit6YTnt`)E)U(&viMfmnO#{q;yx^jTIf|HoG&9Z1$z^ag4_BmeC3j z53y1^6rffkedVD6jN`f-!<8xBv>;V2%e)gS+Q$!@K!!m&at#`EQXj2?u1fD~raJn@ zCiGDmo>0@|pS>UATEj+k4xG#2H;Qxthn{RULC>ogvn4p_Wk)u3xptt1WvQQ{(Bd^{ zGb?C{#WoWa17+$Np*uH;a}=Ea-aC!#(!yxelW&3Bgg$~fIeig~7xVZ6+dUpU;=$sL z9nhlrfHRtVkyg=3PRsT4i6P#GAdGrWXv3}y3q#uS4rZoN#(trr2#@@QRTYvcUbhZV z;BFT{S=MEsMgKYPa!(SXey+=KcQ~*rlL_+GhZ;S8({*}5c-_FPTWh{oVmn)6)3K`) zk6`gvmLK7QiAku}N!$in^1Uip z*RKY)G*!4y&^+c^EKgx6z`R4Oe2HInU>xxO^=R_6*-kHl&Jyq_QhPZyZ^xud0U;Vo z0$qQ?V#4AZ`RXiSE2JtIL%F2+nOQbLLtl%24Q~<&?4$ZeJ^g8Ncs7;99S#CEF`K1! z+!q|4Dm31rp5$BnXtwONjz*Wn7qqq0dBfNWbaE+xYalkNf2DnhT3l$cU0ag(zQm}b zRxrCmjdk=9aiZJ$>Dt<<{N9cuki~vC|57*u4G4J}!+c{iGjsc-z?{nr*N4~}k-Wfo z9IFh0(_9i(ST`-?iCaXzN!0$r?Q|N-52-XXWKHV|IcC5C>#C~gADNkvb}iLwGIhJv zBQ!a%&sFxGcz6bHEaORt>hz5}oKUH7ceGK9Pg(#23UU#KR}}R{GKsO6f=hP6a0jCB ziPmy~twPARd#Y)j5!4WVazE22>*8qPbS{-87I`OTA_lETTvA< literal 9910 zcmbulXIN8B)HR$CAaq0!q)JyHpfpi>??ocL2uMJBuhIgDii(C#=qg2e?+6hQLRVTq zNM(;MAHauTF$(FngEo$8uCTu_7 zm_M5hQxPgmf$RQnZVe^{6J(EcOLpzoV?sf{wD~2Zxjq;TuI;yc4LkCJ?v>eD;Qx%8 zYC&GZb)|oflI;sZZb1yeX&^?eb6NU!igJ?5Cl_6(grU0*2iayU%0h)bp9-m0HKL`6 zf%b0k-#P3PPeLh=k5*GE6{V1O3QclEZSReM%|L4zR|mXA`A4Ank!bJ=8S9pu%Ca7FRMUp6 z0j1tt2WRZjUiMTz!EFg)?+Rl7>I#|O)?*pmN8%wu3oj<_mPa-J&~W?WQbwJh0(Zo^ zVqge;M|~*$Ptd0n_`yHQ{hc9#Y!e>^EjQ2-3bf23foZG;mAiSmW~xDdrNTwi^+P~A z5LsQ0(Xp94>NA5Aisg^tj7bsFtFFLs;=b`N+)mlP(ej?CG;z!8a!!h?wMx0IIt9My z!&A7U1)&+qqNr0F>YTB%dLuZ>tgT$4FH6NiGYEAiNk%>2B5&lDG_0E*u2dbMDluW# z5o}+_usYF&-kh2$_%r^r=tKrT3YSVGzeZi6&sGq%&xn;EEup$=nb%){PaoxAF30uN*{1J0l3Vs%YZ^h3Ca(-N=b z-wu|zK-Hp`7EA=^qGzFSZF@9L_s~vTbdGkJo4=friyKJjTfOYC%ITBdA;vhkL6F3# zfK7k!^D}?Z9lHBRv2+FHlF_V~fc>i^#->XF*^rqlgZO_d-` zDi7HCYv?rk0YVQBekRuzBtsneB4^uQ@WMnef{qwb>aG(2fjY^x{r)spVIq<-K8NQH zLon6RhNyW*RSMugn+Os>J0RKU&&be}b+@-CajJg`&B@z2+n}@IMp}sKYlO(9L7Net zG2KGJ4~^`{Vm^V5b!AnPdor=)C|)@FU-$y6?plHQC$M@6RTAY6#e%sLT}x?`(ow2& z>a*86DGwpd92OQs4?jgJ=FPT>D2cZzoTfJoI@`mw+V{#1Euq=KH2;1y)R3ifw*K}^ zb^=OYXhK$W;A@OlJ4I$?KF)L~(SJ@(e7x|=gjF6yRk^ekdJ*7#mZbKXG3bai0%b#S z-Y<`RnMsZ^28QFYaBP;YRwC}j>;p&R@xYdH-w1zWBB(%pEKQt$p5wnmiknW(R4jRu zZb-(Z>ol6YX_}__6XchlOp$9-&2HJ6Uwrw)*iHWw_T5tF^9|HsZ;=jA3l+U(In?e& zR)=&S0!7|&1wH&(1AcD{X{UZb5t4Iq#D{&MT4=)1_d+IdvEs>F%LI?B-%P9I7qkdJ zpQ-R{w$@QLgRv0$`l-F|E7Ga4Zqq#h_5to_AhD30udVxt+$r<=6gXVgi% zKg}dHQ>o(pO4{Zyo(96bsLx0tgAQlu`Cl6(OpKZ0*CbIZ;_dnv6NH#12dfd5$Dl=Y z^%?ifwCT=MQuNucy_Fl8-h67wWTWU`?Hz|3A5ljmPK}BJ0W|zJoly#AnMFp`?h`c>~^YZ-0u-z~yOMSZgulMbQ%dWP+H>3vKN z#z|#9%T0iB-ef^Q7dm;CDC-hne3;x{aHWmR+80kwYH_N{V$0d;KFs>Qw^w$F`4KhC z9IwglDE3%2aTyb^1qLRLeycclS}}r#VPdMsYNe4(hdnnRF2S zgvB5DFT6)|wKX>>nNQp_lUOv+jF}-D_a3!mdb`_%=#snXYA~^+;DC+n9tBB8I(lmo_S0$$jZ+URzY>VJ)T>*A7uM(eN)}#IrHt>sExAMMn$?R_-aAC8{!4bF&}%tC8H4D%6%gD zR&rv&%PNpJHYg}3m$23PU}icHCehr%s-t?W4RBY{uch*_RoKkycOwM$P5JWg`@%1OTMAFUsH5G^jV34J1rR%_m3G zH@05$TnJw-ni?HZkysoe-Tw6Nw|2rk=&_Mxk^N@@TMFkb?y(n}!5}4l!FD(1OOZbL z0vnrER0T%ipxqhNhWuWeV{4wUeK;Hxq!15#{4dwznR#DLiK~B_!pN7sJf)?EVW!(@ z&(yb1T)8hcqHq4Gt!ab?)xK^%#x! z?h6X4sCh7Pq;;vKCAE<)>FIabjq4B$4&2B;e(@J< zzyen7yGaMzw|_c#egC{;y@wpG;^YmS>rbcR&pF2u%`|>S*i>o@w#$`rns^rg0HIo^ zYLo`azl7;@Fs{?tAPZEDn&h=)(ZosuVZXkb{KH31)IS_3KT>TP67y`VCPi@&rHK5*u;tZz$Z+r~$&Lyf8Ji+Q#{r%-|8vy& z2a`4A(Qg#Mw(g}nqB+8_{ewwKCr~8Z3&aJzgraG13FC`Q3y+p{{3*Znh>P)C=p8d4 zzE>4%8g)ss!i#y=TN~Fq#}ZGQ#gtk5*U2ME+uw!1_os>C;@BUHM(l9BK!ArNd$$~0 zCo8FZ=59wD-N>48FzdXK2y)%z#3GMYz?HTs>zVQmgFL}9d~waciiRxH@HM_C!pr`o z%2Mz{$*+{G54^)1D7Zh6JOiXte*~%{wZCAEL@EE@W#122fjS6Sl{uNyV1RC=wjkK_ zx*Xe0NQwZ{V1kg#(R?zNb|qdnHmC0eF8ZBP&ila+chCw?-_wqsTF}`T3(t+!^R@zl zC~~1Q5lX?H2du-+q$_vuf5yP3C$Zv0TM(5{9PJo(=C5(|Ynw1~z|S~eh*e89NT0I= z%!a9V)bJ@~%L{6XU%9tv5A`%jQB+<)=M;7KNKpwlVxS;E)Y1Y&d$1s*UDn;hBb{2V z0zb-={KsTvGg4+8b&+@YRi?%wMH{9&9#?|dL|KM(jP{Wy*Hp@ zMe+5bg#b*Uv;>8%4ijbhM+$Di8qEE-#7Km4R>65 zKNBu(AV4K=XnJ`$>N%h~ZeJmbUUisoW!?^Ki$14j^lB=#OH55dM&5^ zN6fe>Ja&{uVx$IUtY)q$EyFsKwd^?Y&gkFaT95<2;n7IpW4DHDXYGUx2div6Z^ECDkf9J=S#JmBgjO1vw@!Zf$lVu%wu$Z6_R$fF2(8sUDNn!gv4OQ> z$X9Q>k!TI3ra^rvx#F7Dq{Kv-^cqDz9NObJsx(k7ybUQo8XK$>B`aIl8dgmosT|s| z`hg2?>F;9>CRlEdb8ICAbrF$p88&6BLTw_q1cyBDD&9rD@ooqW+1oCx?}rc=woG?}@^%i%-32RJew( zu3^A@c=}wU+#|vVkYxn-&o9=Zg<*QZP%3V0cAlTnqND4d1(&g!w!FrtLrpJ#fksKD zlf9-50$ocQYCRRhR*Mra8dUs@+}EvYbEs2p%g<3e2)zn?JGk{TLyz2VvABw}@gp?hgyD-lT zwz&1As_H?o$4}+*IW$%^Op?1;3t*5bg_X(L?30gay#zsdRtc5SaTadLOZIQV`I%CG z-u83U-f+Qsz2eym8d&mJb7&x=qXVAqy9)~!DR9zoNS{nMnyMkywTFx zEsK{AFnVyz-DFB{ouAv~h%P=faaE9UUIwXjEnnKPQB$lbbaG?#P{jWo-h>;AhMQLO)u7)k{Z-dgrM*3#6Zu zWJo7&H^H=+n#Sh7Vu7N}2 z$FE$|1G8z6@Y(Z#+l=#8{Bc1pPx|-^8YJ@AWUM8@r8XO0)`Sy4W9I;hQd)#7z-%|O z1(=X$zq6eR`z07ovh(6RPB^XDlzWPANAHlNo18qSKPykx%>|h`bpE=5I0&2({+YrYpQwnd*rQuiOzhYO^_sN zz{j;&5PP{9^h7xMy|Y|f*Q6?WJ+-@yM8uXk@0_KmQ@-W zySycEaJa?=xDC{(0%Wy3aY!>wOlXBJSz_K$kZ&mh2U;tXSa(NF@3#R;>&N>jyQq&o zeUb_-v!&+GL&5M zGeaSVU48O#R!swfeK0wbHu-O#wm5jBUc680If`Fm)tgjDY85IbWUOXOt zqP)Ia88o-=xQv%%Iab^YmIt&T4jiRf%fg5$Prf`_Ir8Qte*LNu^BjD>v1LP>=ji~Xn<(K*=V_j6_;n7OhZXl6y!c%MSa>3qH${9QQ)UobZEiT zvz`zv|F^==QM-FcyWJO6baa|Ka}$KQwla2^>q7d0Pa_QZySC5gLySHy(Dpmr7ZG?- zdjMW^y1`!E@P2$v@gpB{g57Vm>VWMcHw389 z`qE@y!)3gM8a%8}l)*kOj4=JRgvv5$fXrAU*`qt5?IbMig84uKm(X^x!LJ zpR?7^nY_04ml_vrC!vl1EfPs~%sAo0mk7#JFaEhhj|CJk$_+R)b9|0^AiFH*=$yj6 zY~58gOn1Ii6LTd%w~Dr zpLg_Gwp#9NBaaC4$RuY+F((?tVx7M1lI;GRg}j zZ*rxG_P=^JIuPVRHtz!eFk>j3ocn=v;@&hBMtw`A;;VY_r~T zUTiipZ&7K6LL$GVLvC&+n&l8WxDwK!!_@vn5U0Ai?lkLISY!rzen^|ADG@4GM4J}# z3y8-Hi=c@`k0Zkua{uG%sr@CD{E0TXZ4yP2^RSFw#g~6?zJ`~=>fCm4dgra6_P}!3 zj2E*+`0G9Ef6QaC^*8sl|LFX@u=E2WMK=KFLzCcZ7tE%d0v8Zq%!QucDolSKMWZv} z_2eTPELA6%yG%WbqwPuN5C#gk+nYn1oC_uk!oqjwI>6HnzGT&foK~EC|AqKd5%3m0 z!X3)$nkJ>D7M&linI?j?Nxnf3N7;l@ed@uKT+l&2@wXDWRM~F_9|WTYOV<*=T6003 z&}{0q-T$BT2BjcsP>uORxXkx`k4C=zl#WEgR%tUQl2VZFC(f&a=6{>f`M76w=(DQu9GvmY_tQN9|~ZsV(kuJU`dndfD!XoLgkPjOl^ga?*!$2m-kVl#g!EQ(6+UT z>d^nH^s1iRvkjU`)u5?#F$iV8f6V6K?}=j9OCADk``I>ajkU1eyL^J&&vkG8L?T}G zaA=Qc+OgwJ@jQWC7pz1<48Ge_nPphg_;G0MG>g>c_l9ap4e7dA8A3Ygi@hU*#NS)v ziBnD2V@d0sA5uLinTUH-YAS`4>S1-xmns+gB4y?X#D@Y{bF4KUD!PTvYAz%KVwfhc z!f#fuq!ITzy2p(`+L#d)hU7?6eJG`cQOn&fCK*f2K6x&b;a(+kwTkMvu*A+r=dag~ zC5pW*-gwk#i+PJR2#}L#Ch*J|Qx6&aJlgeKJ(v;Z2)<3E-MYMdY>WFkesnQ+pX?s^ z5Rv5=eIqb)tvY$JbvAz73Hm+oVctSuDF2>i{b`%1r`9S5L4Ea@27tuJ*u|YLlZ`Xm zM}S2k!CHso@oxV4I+VGp1ko>C{44v4y87*sjHkoOUIk+iz{vsCay3IkKo-`*P66H7 znbgR;_IW>6GWJp)v&=6^r});(Ob^eAzGyOYeP+f8I=%TmOGN$|J!W0`$Z*4Zm z)v;5hMn3M@PU||I;*`exaDFda1v(-F?gii4i@4eq-rQ5kUnZU4jcLc}Ca|(8$FxA# z;WZ41uS3DCAP~9De`f*U3+Rp=C7V|VXw&7see#z0yl1|H&>CucBff#xq`M6G zBlC%mVzKq@UE?>(O~zb6lmIDnj23%+Z}oDwWOf`6eaW0%Un7GlxkckMFR4%cnPaBV zsFI$IV#72Yi_j%7!%Jx*{HQ8(r|)Z^*@;j2MgAUO$DVVnr{l@a8PK0=DzSee+rI4A zigxVKH&=#9N{n?R326tJ{)+al+v+=m4cF1?czubtB!qquqjzW)^XITgKeI~j*kQ53 z5j2^b%hjRP6U_x+{$@>4JQvygTZ9Q`!rVqpMCGO;83MK4Kmhl%{M-3UQi<_unA*3& zef_Myxu`ziZ}PRPreuJgmpYg$Va{zt0jM88zVcO@s)C%nu>Yr6`L%m`dFSKj*{?4~ zmpHMsd(W43PMz7L^4)90J*-K!9zCaI&)O~nBUE}ZHn<$fIf+iRBG*|3i6F1&XLeL; zJNWEK1(Lv)E`_Ym*lc|pjpwfTU=@OSh!s@7U)r)hO;@B9ZIp1CSGhT?Zsu6I?6B2q zqnR5;2~XD|iy++L#M+>$D_x6B?zn(lc>r4DGwLVQ%5ElOS2gYr3)f~3xK2f^cCZ8x zXY2oPZ+~QXksR= zo-af$M4`}|5m*R@zo6<;R2zRaQyq5i%SO@KS>%V!)?iS&1xIB4W@}vp9{=L}Yij=l zWY$ch#zUr%t(LB2U=5`~{c6Kc;taR4n6@?n`qU#p%U!X*4eEVD8D7_we3DZnvc|fV z`_H{)y#l0Yv@&AEq_h5&q;0!Q&Xa|K<44fXc2GFO^jN!xNbmRxHTIkV$PE9En;=O%z0;~{9~r~^NyVzlI_Y}sfYa6CjNvDEUj_}y4H1V^KNvG z<-;A{6zw(+8znZBd2oQwhqsI_A0w*XkknpI@FVh1D=@v)*6YlOVGV^@P@0n;zXvu&gm(d)M4$;+8SNvu3(fp$y-KxM|vCB>lIfZF4sM2YRrmhxs2)Xg-O$e zdqYXv4;w9v#?8%qkd1Q^Zwgy504r3>ZnojRS2!8M{99(z8_vG^q8PdtI7d(wL=q0c)wEI_mB7tZ=eJhq5iWb<2Rm$8Ae-r|=0 z;uuYZ+dSL7^)JOV&BnjRojPeBa7W>_kYpV@GnHgT;~6(xm-}pSfz?JLgferx+Rawq zE%-g^)t8LRvz$|8^WhE2OM}wyN#^5VLWLQ>{{NijDsj7r>Y?Z00~good<% } else { %>danger<%} %>", - "author_name": "sanger-tol/readmapping v${version} - ${runName}", + "author_name": "nf-core/rnavar v${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ diff --git a/conf/test_full.config b/conf/test_full.config index 718622c4..979f49f7 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -10,8 +10,6 @@ ---------------------------------------------------------------------------------------- */ -cleanup = true - params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' diff --git a/docs/usage.md b/docs/usage.md index 08da8b39..3a706bbf 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -57,7 +57,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: ```bash -nextflow run nf-core/rnavar --input samplesheet.csv --outdir --genome GRCh37 -profile docker +nextflow run nf-core/rnavar --input ./samplesheet.csv --outdir ./results --genome GRCh37 -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -76,7 +76,8 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. > ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). -> The above pipeline run specified with a params file in yaml format: + +The above pipeline run specified with a params file in yaml format: ```bash nextflow run nf-core/rnavar -profile docker -params-file params.yaml @@ -88,7 +89,6 @@ with `params.yaml` containing: input: './samplesheet.csv' outdir: './results/' genome: 'GRCh37' -input: 'data' <...> ``` diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy deleted file mode 100755 index 9b34804d..00000000 --- a/lib/NfcoreSchema.groovy +++ /dev/null @@ -1,530 +0,0 @@ -// -// This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. -// - -import nextflow.Nextflow -import org.everit.json.schema.Schema -import org.everit.json.schema.loader.SchemaLoader -import org.everit.json.schema.ValidationException -import org.json.JSONObject -import org.json.JSONTokener -import org.json.JSONArray -import groovy.json.JsonSlurper -import groovy.json.JsonBuilder - -class NfcoreSchema { - - // - // Resolve Schema path relative to main workflow directory - // - public static String getSchemaPath(workflow, schema_filename='nextflow_schema.json') { - return "${workflow.projectDir}/${schema_filename}" - } - - // - // Function to loop over all parameters defined in schema and check - // whether the given parameters adhere to the specifications - // - /* groovylint-disable-next-line UnusedPrivateMethodParameter */ - public static void validateParameters(workflow, params, log, schema_filename='nextflow_schema.json') { - def has_error = false - //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// - // Check for nextflow core params and unexpected params - def json = new File(getSchemaPath(workflow, schema_filename=schema_filename)).text - def Map schemaParams = (Map) new JsonSlurper().parseText(json).get('definitions') - def nf_params = [ - // Options for base `nextflow` command - 'bg', - 'c', - 'C', - 'config', - 'd', - 'D', - 'dockerize', - 'h', - 'log', - 'q', - 'quiet', - 'syslog', - 'v', - - // Options for `nextflow run` command - 'ansi', - 'ansi-log', - 'bg', - 'bucket-dir', - 'c', - 'cache', - 'config', - 'dsl2', - 'dump-channels', - 'dump-hashes', - 'E', - 'entry', - 'latest', - 'lib', - 'main-script', - 'N', - 'name', - 'offline', - 'params-file', - 'pi', - 'plugins', - 'poll-interval', - 'pool-size', - 'profile', - 'ps', - 'qs', - 'queue-size', - 'r', - 'resume', - 'revision', - 'stdin', - 'stub', - 'stub-run', - 'test', - 'w', - 'with-apptainer', - 'with-charliecloud', - 'with-conda', - 'with-dag', - 'with-docker', - 'with-mpi', - 'with-notification', - 'with-podman', - 'with-report', - 'with-singularity', - 'with-timeline', - 'with-tower', - 'with-trace', - 'with-weblog', - 'without-docker', - 'without-podman', - 'work-dir' - ] - def unexpectedParams = [] - - // Collect expected parameters from the schema - def expectedParams = [] - def enums = [:] - for (group in schemaParams) { - for (p in group.value['properties']) { - expectedParams.push(p.key) - if (group.value['properties'][p.key].containsKey('enum')) { - enums[p.key] = group.value['properties'][p.key]['enum'] - } - } - } - - for (specifiedParam in params.keySet()) { - // nextflow params - if (nf_params.contains(specifiedParam)) { - log.error "ERROR: You used a core Nextflow option with two hyphens: '--${specifiedParam}'. Please resubmit with '-${specifiedParam}'" - has_error = true - } - // unexpected params - def params_ignore = params.schema_ignore_params.split(',') + 'schema_ignore_params' - def expectedParamsLowerCase = expectedParams.collect{ it.replace("-", "").toLowerCase() } - def specifiedParamLowerCase = specifiedParam.replace("-", "").toLowerCase() - def isCamelCaseBug = (specifiedParam.contains("-") && !expectedParams.contains(specifiedParam) && expectedParamsLowerCase.contains(specifiedParamLowerCase)) - if (!expectedParams.contains(specifiedParam) && !params_ignore.contains(specifiedParam) && !isCamelCaseBug) { - // Temporarily remove camelCase/camel-case params #1035 - def unexpectedParamsLowerCase = unexpectedParams.collect{ it.replace("-", "").toLowerCase()} - if (!unexpectedParamsLowerCase.contains(specifiedParamLowerCase)){ - unexpectedParams.push(specifiedParam) - } - } - } - - //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// - // Validate parameters against the schema - InputStream input_stream = new File(getSchemaPath(workflow, schema_filename=schema_filename)).newInputStream() - JSONObject raw_schema = new JSONObject(new JSONTokener(input_stream)) - - // Remove anything that's in params.schema_ignore_params - raw_schema = removeIgnoredParams(raw_schema, params) - - Schema schema = SchemaLoader.load(raw_schema) - - // Clean the parameters - def cleanedParams = cleanParameters(params) - - // Convert to JSONObject - def jsonParams = new JsonBuilder(cleanedParams) - JSONObject params_json = new JSONObject(jsonParams.toString()) - - // Validate - try { - schema.validate(params_json) - } catch (ValidationException e) { - println '' - log.error 'ERROR: Validation of pipeline parameters failed!' - JSONObject exceptionJSON = e.toJSON() - printExceptions(exceptionJSON, params_json, log, enums) - println '' - has_error = true - } - - // Check for unexpected parameters - if (unexpectedParams.size() > 0) { - Map colors = NfcoreTemplate.logColours(params.monochrome_logs) - println '' - def warn_msg = 'Found unexpected parameters:' - for (unexpectedParam in unexpectedParams) { - warn_msg = warn_msg + "\n* --${unexpectedParam}: ${params[unexpectedParam].toString()}" - } - log.warn warn_msg - log.info "- ${colors.dim}Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}" - println '' - } - - if (has_error) { - Nextflow.error('Exiting!') - } - } - - // - // Beautify parameters for --help - // - public static String paramsHelp(workflow, params, command, schema_filename='nextflow_schema.json') { - Map colors = NfcoreTemplate.logColours(params.monochrome_logs) - Integer num_hidden = 0 - String output = '' - output += 'Typical pipeline command:\n\n' - output += " ${colors.cyan}${command}${colors.reset}\n\n" - Map params_map = paramsLoad(getSchemaPath(workflow, schema_filename=schema_filename)) - Integer max_chars = paramsMaxChars(params_map) + 1 - Integer desc_indent = max_chars + 14 - Integer dec_linewidth = 160 - desc_indent - for (group in params_map.keySet()) { - Integer num_params = 0 - String group_output = colors.underlined + colors.bold + group + colors.reset + '\n' - def group_params = params_map.get(group) // This gets the parameters of that particular group - for (param in group_params.keySet()) { - if (group_params.get(param).hidden && !params.show_hidden_params) { - num_hidden += 1 - continue; - } - def type = '[' + group_params.get(param).type + ']' - def description = group_params.get(param).description - def defaultValue = group_params.get(param).default != null ? " [default: " + group_params.get(param).default.toString() + "]" : '' - def description_default = description + colors.dim + defaultValue + colors.reset - // Wrap long description texts - // Loosely based on https://dzone.com/articles/groovy-plain-text-word-wrap - if (description_default.length() > dec_linewidth){ - List olines = [] - String oline = "" // " " * indent - description_default.split(" ").each() { wrd -> - if ((oline.size() + wrd.size()) <= dec_linewidth) { - oline += wrd + " " - } else { - olines += oline - oline = wrd + " " - } - } - olines += oline - description_default = olines.join("\n" + " " * desc_indent) - } - group_output += " --" + param.padRight(max_chars) + colors.dim + type.padRight(10) + colors.reset + description_default + '\n' - num_params += 1 - } - group_output += '\n' - if (num_params > 0){ - output += group_output - } - } - if (num_hidden > 0){ - output += colors.dim + "!! Hiding $num_hidden params, use --show_hidden_params to show them !!\n" + colors.reset - } - output += NfcoreTemplate.dashedLine(params.monochrome_logs) - return output - } - - // - // Groovy Map summarising parameters/workflow options used by the pipeline - // - public static LinkedHashMap paramsSummaryMap(workflow, params, schema_filename='nextflow_schema.json') { - // Get a selection of core Nextflow workflow options - def Map workflow_summary = [:] - if (workflow.revision) { - workflow_summary['revision'] = workflow.revision - } - workflow_summary['runName'] = workflow.runName - if (workflow.containerEngine) { - workflow_summary['containerEngine'] = workflow.containerEngine - } - if (workflow.container) { - workflow_summary['container'] = workflow.container - } - workflow_summary['launchDir'] = workflow.launchDir - workflow_summary['workDir'] = workflow.workDir - workflow_summary['projectDir'] = workflow.projectDir - workflow_summary['userName'] = workflow.userName - workflow_summary['profile'] = workflow.profile - workflow_summary['configFiles'] = workflow.configFiles.join(', ') - - // Get pipeline parameters defined in JSON Schema - def Map params_summary = [:] - def params_map = paramsLoad(getSchemaPath(workflow, schema_filename=schema_filename)) - for (group in params_map.keySet()) { - def sub_params = new LinkedHashMap() - def group_params = params_map.get(group) // This gets the parameters of that particular group - for (param in group_params.keySet()) { - if (params.containsKey(param)) { - def params_value = params.get(param) - def schema_value = group_params.get(param).default - def param_type = group_params.get(param).type - if (schema_value != null) { - if (param_type == 'string') { - if (schema_value.contains('$projectDir') || schema_value.contains('${projectDir}')) { - def sub_string = schema_value.replace('\$projectDir', '') - sub_string = sub_string.replace('\${projectDir}', '') - if (params_value.contains(sub_string)) { - schema_value = params_value - } - } - if (schema_value.contains('$params.outdir') || schema_value.contains('${params.outdir}')) { - def sub_string = schema_value.replace('\$params.outdir', '') - sub_string = sub_string.replace('\${params.outdir}', '') - if ("${params.outdir}${sub_string}" == params_value) { - schema_value = params_value - } - } - } - } - - // We have a default in the schema, and this isn't it - if (schema_value != null && params_value != schema_value) { - sub_params.put(param, params_value) - } - // No default in the schema, and this isn't empty - else if (schema_value == null && params_value != "" && params_value != null && params_value != false) { - sub_params.put(param, params_value) - } - } - } - params_summary.put(group, sub_params) - } - return [ 'Core Nextflow options' : workflow_summary ] << params_summary - } - - // - // Beautify parameters for summary and return as string - // - public static String paramsSummaryLog(workflow, params) { - Map colors = NfcoreTemplate.logColours(params.monochrome_logs) - String output = '' - def params_map = paramsSummaryMap(workflow, params) - def max_chars = paramsMaxChars(params_map) - for (group in params_map.keySet()) { - def group_params = params_map.get(group) // This gets the parameters of that particular group - if (group_params) { - output += colors.bold + group + colors.reset + '\n' - for (param in group_params.keySet()) { - output += " " + colors.blue + param.padRight(max_chars) + ": " + colors.green + group_params.get(param) + colors.reset + '\n' - } - output += '\n' - } - } - output += "!! Only displaying parameters that differ from the pipeline defaults !!\n" - output += NfcoreTemplate.dashedLine(params.monochrome_logs) - return output - } - - // - // Loop over nested exceptions and print the causingException - // - private static void printExceptions(ex_json, params_json, log, enums, limit=5) { - def causingExceptions = ex_json['causingExceptions'] - if (causingExceptions.length() == 0) { - def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/ - // Missing required param - if (m.matches()) { - log.error "* Missing required parameter: --${m[0][1]}" - } - // Other base-level error - else if (ex_json['pointerToViolation'] == '#') { - log.error "* ${ex_json['message']}" - } - // Error with specific param - else { - def param = ex_json['pointerToViolation'] - ~/^#\// - def param_val = params_json[param].toString() - if (enums.containsKey(param)) { - def error_msg = "* --${param}: '${param_val}' is not a valid choice (Available choices" - if (enums[param].size() > limit) { - log.error "${error_msg} (${limit} of ${enums[param].size()}): ${enums[param][0..limit-1].join(', ')}, ... )" - } else { - log.error "${error_msg}: ${enums[param].join(', ')})" - } - } else { - log.error "* --${param}: ${ex_json['message']} (${param_val})" - } - } - } - for (ex in causingExceptions) { - printExceptions(ex, params_json, log, enums) - } - } - - // - // Remove an element from a JSONArray - // - private static JSONArray removeElement(json_array, element) { - def list = [] - int len = json_array.length() - for (int i=0;i - if(raw_schema.keySet().contains('definitions')){ - raw_schema.definitions.each { definition -> - for (key in definition.keySet()){ - if (definition[key].get("properties").keySet().contains(ignore_param)){ - // Remove the param to ignore - definition[key].get("properties").remove(ignore_param) - // If the param was required, change this - if (definition[key].has("required")) { - def cleaned_required = removeElement(definition[key].required, ignore_param) - definition[key].put("required", cleaned_required) - } - } - } - } - } - if(raw_schema.keySet().contains('properties') && raw_schema.get('properties').keySet().contains(ignore_param)) { - raw_schema.get("properties").remove(ignore_param) - } - if(raw_schema.keySet().contains('required') && raw_schema.required.contains(ignore_param)) { - def cleaned_required = removeElement(raw_schema.required, ignore_param) - raw_schema.put("required", cleaned_required) - } - } - return raw_schema - } - - // - // Clean and check parameters relative to Nextflow native classes - // - private static Map cleanParameters(params) { - def new_params = params.getClass().newInstance(params) - for (p in params) { - // remove anything evaluating to false - if (!p['value']) { - new_params.remove(p.key) - } - // Cast MemoryUnit to String - if (p['value'].getClass() == nextflow.util.MemoryUnit) { - new_params.replace(p.key, p['value'].toString()) - } - // Cast Duration to String - if (p['value'].getClass() == nextflow.util.Duration) { - new_params.replace(p.key, p['value'].toString().replaceFirst(/d(?!\S)/, "day")) - } - // Cast LinkedHashMap to String - if (p['value'].getClass() == LinkedHashMap) { - new_params.replace(p.key, p['value'].toString()) - } - } - return new_params - } - - // - // This function tries to read a JSON params file - // - private static LinkedHashMap paramsLoad(String json_schema) { - def params_map = new LinkedHashMap() - try { - params_map = paramsRead(json_schema) - } catch (Exception e) { - println "Could not read parameters settings from JSON. $e" - params_map = new LinkedHashMap() - } - return params_map - } - - // - // Method to actually read in JSON file using Groovy. - // Group (as Key), values are all parameters - // - Parameter1 as Key, Description as Value - // - Parameter2 as Key, Description as Value - // .... - // Group - // - - private static LinkedHashMap paramsRead(String json_schema) throws Exception { - def json = new File(json_schema).text - def Map schema_definitions = (Map) new JsonSlurper().parseText(json).get('definitions') - def Map schema_properties = (Map) new JsonSlurper().parseText(json).get('properties') - /* Tree looks like this in nf-core schema - * definitions <- this is what the first get('definitions') gets us - group 1 - title - description - properties - parameter 1 - type - description - parameter 2 - type - description - group 2 - title - description - properties - parameter 1 - type - description - * properties <- parameters can also be ungrouped, outside of definitions - parameter 1 - type - description - */ - - // Grouped params - def params_map = new LinkedHashMap() - schema_definitions.each { key, val -> - def Map group = schema_definitions."$key".properties // Gets the property object of the group - def title = schema_definitions."$key".title - def sub_params = new LinkedHashMap() - group.each { innerkey, value -> - sub_params.put(innerkey, value) - } - params_map.put(title, sub_params) - } - - // Ungrouped params - def ungrouped_params = new LinkedHashMap() - schema_properties.each { innerkey, value -> - ungrouped_params.put(innerkey, value) - } - params_map.put("Other parameters", ungrouped_params) - - return params_map - } - - // - // Get maximum number of characters across all parameter names - // - private static Integer paramsMaxChars(params_map) { - Integer max_chars = 0 - for (group in params_map.keySet()) { - def group_params = params_map.get(group) // This gets the parameters of that particular group - for (param in group_params.keySet()) { - if (param.size() > max_chars) { - max_chars = param.size() - } - } - } - return max_chars - } -} diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy index 25a0a74a..408951ae 100755 --- a/lib/NfcoreTemplate.groovy +++ b/lib/NfcoreTemplate.groovy @@ -128,7 +128,7 @@ class NfcoreTemplate { def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] def sf = new File("$projectDir/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index f5bdc826..3124b731 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -20,40 +20,11 @@ class WorkflowMain { " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" } - // - // Generate help string - // - public static String help(workflow, params) { - def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" - def help_string = '' - help_string += NfcoreTemplate.logo(workflow, params.monochrome_logs) - help_string += NfcoreSchema.paramsHelp(workflow, params, command) - help_string += '\n' + citation(workflow) + '\n' - help_string += NfcoreTemplate.dashedLine(params.monochrome_logs) - return help_string - } - - // - // Generate parameter summary log string - // - public static String paramsSummaryLog(workflow, params) { - def summary_log = '' - summary_log += NfcoreTemplate.logo(workflow, params.monochrome_logs) - summary_log += NfcoreSchema.paramsSummaryLog(workflow, params) - summary_log += '\n' + citation(workflow) + '\n' - summary_log += NfcoreTemplate.dashedLine(params.monochrome_logs) - return summary_log - } // // Validate parameters and print summary to screen // public static void initialise(workflow, params, log) { - // Print help to screen if required - if (params.help) { - log.info help(workflow, params) - System.exit(0) - } // Print workflow version and exit on --version if (params.version) { @@ -62,14 +33,6 @@ class WorkflowMain { System.exit(0) } - // Print parameter summary log to screen - log.info paramsSummaryLog(workflow, params) - - // Validate workflow parameters via the JSON schema - if (params.validate_params) { - NfcoreSchema.validateParameters(workflow, params, log) - } - // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) diff --git a/lib/WorkflowRnavar.groovy b/lib/WorkflowRnavar.groovy index 52176347..72f686fd 100755 --- a/lib/WorkflowRnavar.groovy +++ b/lib/WorkflowRnavar.groovy @@ -11,6 +11,7 @@ class WorkflowRnavar { // Check and validate parameters // public static void initialise(params, log) { + genomeExistsError(params, log) @@ -46,15 +47,57 @@ class WorkflowRnavar { return yaml_file_text } - public static String methodsDescriptionText(run_workflow, mqc_methods_yaml) { + // + // Generate methods description for MultiQC + // + + public static String toolCitationText(params) { + + // TODO Optionally add in-text citation tools to this list. + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "Tool (Foo et al. 2023)" : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def citation_text = [ + "Tools used in the workflow included:", + "FastQC (Andrews 2010),", + "MultiQC (Ewels et al. 2016)", + "." + ].join(' ').trim() + + return citation_text + } + + public static String toolBibliographyText(params) { + + // TODO Optionally add bibliographic entries to this list. + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "

  • Author (2023) Pub name, Journal, DOI
    • " : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def reference_text = [ + "
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ", + "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • " + ].join(' ').trim() + + return reference_text + } + + public static String methodsDescriptionText(run_workflow, mqc_methods_yaml, params) { // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file def meta = [:] meta.workflow = run_workflow.toMap() meta["manifest_map"] = run_workflow.manifest.toMap() + // Pipeline DOI meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + // Tool references + meta["tool_citations"] = "" + meta["tool_bibliography"] = "" + + // TODO Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled! + //meta["tool_citations"] = toolCitationText(params).replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") + //meta["tool_bibliography"] = toolBibliographyText(params) + + def methods_text = mqc_methods_yaml.text def engine = new SimpleTemplateEngine() diff --git a/main.nf b/main.nf index 6fc4d58a..866aebfc 100644 --- a/main.nf +++ b/main.nf @@ -25,6 +25,22 @@ params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta') ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +include { validateParameters; paramsHelp } from 'plugin/nf-validation' + +// Print help message if needed +if (params.help) { + def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) + def citation = '\n' + WorkflowMain.citation(workflow) + '\n' + def String command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" + log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) + System.exit(0) +} + +// Validate input parameters +if (params.validate_params) { + validateParameters() +} + WorkflowMain.initialise(workflow, params, log) /* diff --git a/nextflow.config b/nextflow.config index 50f76a90..fe9cde98 100644 --- a/nextflow.config +++ b/nextflow.config @@ -12,12 +12,12 @@ params { // TODO nf-core: Specify your pipeline's command line flags // Input options input = null - - // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false + + // MultiQC options multiqc_config = null multiqc_title = null @@ -27,7 +27,6 @@ params { // Boilerplate options outdir = null - tracedir = "${params.outdir}/pipeline_info" publish_dir_mode = 'copy' email = null email_on_fail = null @@ -36,19 +35,15 @@ params { hook_url = null help = false version = false - validate_params = true - show_hidden_params = false - schema_ignore_params = 'genomes' - // Config options + config_profile_name = null + config_profile_description = null custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" - config_profile_description = null config_profile_contact = null config_profile_url = null - config_profile_name = null - + // Max resource options // Defaults only, expecting to be overwritten @@ -56,6 +51,13 @@ params { max_cpus = 16 max_time = '240.h' + // Schema validation default options + validationFailUnrecognisedParams = false + validationLenientMode = false + validationSchemaIgnoreParams = 'genomes' + validationShowHiddenParams = false + validate_params = true + } // Load base.config by default for all pipelines @@ -75,13 +77,11 @@ try { // } catch (Exception e) { // System.err.println("WARNING: Could not load nf-core/config/rnavar profiles: ${params.custom_config_base}/pipeline/rnavar.config") // } - - profiles { debug { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' - cleanup = false + cleanup = false } conda { conda.enabled = true @@ -104,7 +104,6 @@ profiles { } docker { docker.enabled = true - docker.registry = 'quay.io' docker.userEmulation = true conda.enabled = false singularity.enabled = false @@ -128,7 +127,6 @@ profiles { } podman { podman.enabled = true - podman.registry = 'quay.io' conda.enabled = false docker.enabled = false singularity.enabled = false @@ -172,6 +170,18 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } +// Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Singularity are enabled +// Set to your registry if you have a mirror of containers +apptainer.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' + +// Nextflow plugins +plugins { + id 'nf-validation' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} // Load igenomes.config if required if (!params.igenomes_ignore) { @@ -179,8 +189,6 @@ if (!params.igenomes_ignore) { } else { params.genomes = [:] } - - // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. // See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. @@ -198,19 +206,19 @@ process.shell = ['/bin/bash', '-euo', 'pipefail'] def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true - file = "${params.tracedir}/execution_timeline_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html" } report { enabled = true - file = "${params.tracedir}/execution_report_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" } trace { enabled = true - file = "${params.tracedir}/execution_trace_${trace_timestamp}.txt" + file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" } dag { enabled = true - file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html" } manifest { @@ -219,7 +227,7 @@ manifest { homePage = 'https://github.com/nf-core/rnavar' description = """GATK4 RNA variant calling pipeline""" mainScript = 'main.nf' - nextflowVersion = '!>=22.10.1' + nextflowVersion = '!>=23.04.0' version = '1.1.0dev' doi = '' } diff --git a/nextflow_schema.json b/nextflow_schema.json index 4395e63f..9d2c4bde 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -15,9 +15,9 @@ "input": { "type": "string", "format": "file-path", + "exists": true, "mimetype": "text/csv", "pattern": "^\\S+\\.csv$", - "schema": "assets/schema_input.json", "description": "Path to comma-separated file containing information about the samples in the experiment.", "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/rnavar/usage#samplesheet-input).", "fa_icon": "fas fa-file-csv" @@ -57,6 +57,7 @@ "fasta": { "type": "string", "format": "file-path", + "exists": true, "mimetype": "text/plain", "pattern": "^\\S+\\.fn?a(sta)?(\\.gz)?$", "description": "Path to FASTA genome file.", @@ -157,7 +158,7 @@ "description": "Maximum amount of time that can be requested for any single job.", "default": "240.h", "fa_icon": "far fa-clock", - "pattern": "^(\\d+\\.?\\s*(s|m|h|day)\\s*)+$", + "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$", "hidden": true, "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" } @@ -174,12 +175,14 @@ "type": "boolean", "description": "Display help text.", "fa_icon": "fas fa-question-circle", + "default": false, "hidden": true }, "version": { "type": "boolean", "description": "Display version and exit.", "fa_icon": "fas fa-question-circle", + "default": false, "hidden": true }, "publish_dir_mode": { @@ -203,6 +206,7 @@ "type": "boolean", "description": "Send plain-text email instead of HTML.", "fa_icon": "fas fa-remove-format", + "default": false, "hidden": true }, "max_multiqc_email_size": { @@ -217,6 +221,7 @@ "type": "boolean", "description": "Do not use coloured log outputs.", "fa_icon": "fas fa-palette", + "default": false, "hidden": true }, "hook_url": { @@ -228,6 +233,7 @@ }, "multiqc_config": { "type": "string", + "format": "file-path", "description": "Custom config file to supply to MultiQC.", "fa_icon": "fas fa-cog", "hidden": true @@ -243,13 +249,6 @@ "description": "Custom MultiQC yaml file containing HTML including a methods description.", "fa_icon": "fas fa-cog" }, - "tracedir": { - "type": "string", - "description": "Directory to keep pipeline Nextflow logs and reports.", - "default": "${params.outdir}/pipeline_info", - "fa_icon": "fas fa-cogs", - "hidden": true - }, "validate_params": { "type": "boolean", "description": "Boolean whether to validate parameters against the schema at runtime", @@ -257,12 +256,29 @@ "fa_icon": "fas fa-check-square", "hidden": true }, - "show_hidden_params": { + "validationShowHiddenParams": { "type": "boolean", "fa_icon": "far fa-eye-slash", "description": "Show all params when using `--help`", + "default": false, "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." + }, + "validationFailUnrecognisedParams": { + "type": "boolean", + "fa_icon": "far fa-check-circle", + "description": "Validation of parameters fails when an unrecognised parameter is found.", + "default": false, + "hidden": true, + "help_text": "By default, when an unrecognised parameter is found, it returns a warinig." + }, + "validationLenientMode": { + "type": "boolean", + "fa_icon": "far fa-check-circle", + "description": "Validation of parameters in lenient more.", + "default": false, + "hidden": true, + "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." } } } diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 6d42ac31..fd6c2b0c 100644 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -1,21 +1,19 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - VALIDATE INPUTS + PRINT PARAMS SUMMARY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -def summary_params = NfcoreSchema.paramsSummaryMap(workflow, params) +include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' -// Validate input parameters -WorkflowRnavar.initialise(params, log) +def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) +def citation = '\n' + WorkflowMain.citation(workflow) + '\n' +def summary_params = paramsSummaryMap(workflow) -// TODO nf-core: Add all file path parameters for the pipeline to the list below -// Check input path parameters to see if they exist -def checkPathParamList = [ params.input, params.multiqc_config, params.fasta ] -for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } } +// Print parameter summary log to screen +log.info logo + paramsSummaryLog(workflow) + citation -// Check mandatory parameters -if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } +WorkflowRnavar.initialise(params, log) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -69,9 +67,12 @@ workflow RNAVAR { // SUBWORKFLOW: Read in samplesheet, validate and stage input files // INPUT_CHECK ( - ch_input + file(params.input) ) ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) + // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") + // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ + // ! There is currently no tooling to help you write a sample sheet schema // // MODULE: Run FastQC @@ -91,7 +92,7 @@ workflow RNAVAR { workflow_summary = WorkflowRnavar.paramsSummaryMultiqc(workflow, summary_params) ch_workflow_summary = Channel.value(workflow_summary) - methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) + methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) ch_methods_description = Channel.value(methods_description) ch_multiqc_files = Channel.empty() From dd90d55300efa261888e990a1ab103925320d4e2 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 12:23:48 +0200 Subject: [PATCH 08/34] typo --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 643ec742..21484ba3 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,9 @@ Each row represents a fastq file (single-end) or a pair of fastq files (paired e Now, you can run the pipeline using: - ```console - nextflow run nf-core/rnavar -profile --input samplesheet.csv --outdir --genome GRCh38 - ``` +```console +nextflow run nf-core/rnavar -profile --input samplesheet.csv --outdir --genome GRCh38 +``` > **Warning:** > Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those From 6331c1424da3116c9d3495c2fdec30c44ed04d75 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 12:34:11 +0200 Subject: [PATCH 09/34] update all modules --- modules.json | 206 ++++++++++++------ .../local/custom/dumpsoftwareversions/main.nf | 6 +- modules/local/gtf2bed.nf | 2 +- modules/nf-core/bedtools/merge/main.nf | 17 +- modules/nf-core/bedtools/merge/meta.yml | 2 + modules/nf-core/bedtools/sort/main.nf | 18 +- modules/nf-core/bedtools/sort/meta.yml | 2 + modules/nf-core/cat/fastq/main.nf | 2 +- modules/nf-core/cat/fastq/meta.yml | 3 +- .../custom/dumpsoftwareversions/main.nf | 6 +- modules/nf-core/ensemblvep/vep/main.nf | 21 +- modules/nf-core/ensemblvep/vep/meta.yml | 22 +- modules/nf-core/fastqc/main.nf | 8 +- modules/nf-core/fastqc/tests/main.nf.test | 32 +++ modules/nf-core/gatk4/applybqsr/main.nf | 12 +- modules/nf-core/gatk4/applybqsr/meta.yml | 17 +- .../nf-core/gatk4/baserecalibrator/main.nf | 12 +- .../nf-core/gatk4/baserecalibrator/meta.yml | 16 +- .../nf-core/gatk4/bedtointervallist/main.nf | 14 +- .../nf-core/gatk4/bedtointervallist/meta.yml | 7 + modules/nf-core/gatk4/combinegvcfs/main.nf | 12 +- .../gatk4/createsequencedictionary/main.nf | 18 +- .../gatk4/createsequencedictionary/meta.yml | 41 ++-- modules/nf-core/gatk4/haplotypecaller/main.nf | 12 +- .../nf-core/gatk4/haplotypecaller/meta.yml | 16 +- .../nf-core/gatk4/indexfeaturefile/main.nf | 12 +- .../nf-core/gatk4/intervallisttools/main.nf | 12 +- modules/nf-core/gatk4/markduplicates/main.nf | 32 ++- modules/nf-core/gatk4/mergevcfs/main.nf | 12 +- .../nf-core/gatk4/splitncigarreads/main.nf | 12 +- .../nf-core/gatk4/variantfiltration/main.nf | 18 +- .../nf-core/gatk4/variantfiltration/meta.yml | 17 ++ modules/nf-core/gffread/main.nf | 2 +- modules/nf-core/gunzip/main.nf | 12 +- modules/nf-core/gunzip/meta.yml | 35 +-- modules/nf-core/multiqc/main.nf | 6 +- modules/nf-core/samtools/faidx/main.nf | 22 +- modules/nf-core/samtools/faidx/meta.yml | 14 +- modules/nf-core/samtools/flagstat/main.nf | 17 +- modules/nf-core/samtools/flagstat/meta.yml | 2 +- modules/nf-core/samtools/idxstats/main.nf | 18 +- modules/nf-core/samtools/idxstats/meta.yml | 2 +- modules/nf-core/samtools/index/main.nf | 6 +- modules/nf-core/samtools/index/meta.yml | 2 +- modules/nf-core/samtools/merge/main.nf | 10 +- modules/nf-core/samtools/merge/meta.yml | 21 +- modules/nf-core/samtools/sort/main.nf | 14 +- modules/nf-core/samtools/sort/meta.yml | 2 +- modules/nf-core/samtools/stats/main.nf | 8 +- modules/nf-core/samtools/stats/meta.yml | 24 +- modules/nf-core/snpeff/snpeff/main.nf | 10 +- modules/nf-core/snpeff/snpeff/meta.yml | 6 +- modules/nf-core/star/align/main.nf | 28 ++- modules/nf-core/star/align/meta.yml | 32 +++ modules/nf-core/star/genomegenerate/main.nf | 10 +- modules/nf-core/star/genomegenerate/meta.yml | 15 ++ modules/nf-core/tabix/bgziptabix/main.nf | 10 +- modules/nf-core/tabix/bgziptabix/meta.yml | 5 + modules/nf-core/tabix/tabix/main.nf | 2 +- modules/nf-core/untar/main.nf | 4 +- modules/nf-core/untar/meta.yml | 1 + 61 files changed, 652 insertions(+), 327 deletions(-) create mode 100644 modules/nf-core/fastqc/tests/main.nf.test diff --git a/modules.json b/modules.json index 0abe555e..3e8235c2 100755 --- a/modules.json +++ b/modules.json @@ -7,173 +7,241 @@ "nf-core": { "bedtools/merge": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "9e51255c4f8ec69fb6ccf68593392835f14fecb8", + "installed_by": [ + "modules" + ] }, "bedtools/sort": { "branch": "master", - "git_sha": "f1f473b21811b958d1317c4a97c56e16d3ee40f9", - "installed_by": ["modules"] + "git_sha": "9e51255c4f8ec69fb6ccf68593392835f14fecb8", + "installed_by": [ + "modules" + ] }, "cat/fastq": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e", + "installed_by": [ + "modules" + ] }, "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "76cc4938c1f6ea5c7d83fed1eeffc146787f9543", - "installed_by": ["modules"] + "git_sha": "05c280924b6c768d484c7c443dad5e605c4ff4b4", + "installed_by": [ + "modules" + ] }, "ensemblvep/vep": { "branch": "master", - "git_sha": "bea3ca998816a7f812b1bbbcb27c3a9ffbac0706", - "installed_by": ["modules"] + "git_sha": "9f9e1fc31cb35876922070c0e601ae05abae5cae", + "installed_by": [ + "modules" + ] }, "fastqc": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "9a4517e720bc812e95b56d23d15a1653b6db4f53", + "installed_by": [ + "modules" + ] }, "gatk4/applybqsr": { "branch": "master", - "git_sha": "f2d63bd5b68925f98f572eed70993d205cc694b7", - "installed_by": ["modules"] + "git_sha": "240937a2a9c30298110753292be041188891f2cb", + "installed_by": [ + "modules" + ] }, "gatk4/baserecalibrator": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gatk4/bedtointervallist": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "2df2a11d5b12f2a73bca74f103691bc35d83c5fd", + "installed_by": [ + "modules" + ] }, "gatk4/combinegvcfs": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gatk4/createsequencedictionary": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "541811d779026c5d395925895fa5ed35e7216cc0", + "installed_by": [ + "modules" + ] }, "gatk4/haplotypecaller": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gatk4/indexfeaturefile": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gatk4/intervallisttools": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gatk4/markduplicates": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "0a261469640941da2488e1a5aa023b64db837c70", + "installed_by": [ + "modules" + ] }, "gatk4/mergevcfs": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gatk4/splitncigarreads": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gatk4/variantfiltration": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "2df2a11d5b12f2a73bca74f103691bc35d83c5fd", + "installed_by": [ + "modules" + ] }, "gffread": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "gunzip": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "e06548bfa36ee31869b81041879dd6b3a83b1d57", + "installed_by": [ + "modules" + ] }, "multiqc": { "branch": "master", - "git_sha": "ee80d14721e76e2e079103b8dcd5d57129e584ba", - "installed_by": ["modules"] + "git_sha": "a6e11ac655e744f7ebc724be669dd568ffdc0e80", + "installed_by": [ + "modules" + ] }, "samtools/faidx": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "fd742419940e01ba1c5ecb172c3e32ec840662fe", + "installed_by": [ + "modules" + ] }, "samtools/flagstat": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "570ec5bcfe19c49e16c9ca35a7a116563af6cc1c", + "installed_by": [ + "modules" + ] }, "samtools/idxstats": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "e662ab16e0c11f1e62983e21de9871f59371a639", + "installed_by": [ + "modules" + ] }, "samtools/index": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "samtools/merge": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "0460d316170f75f323111b4a2c0a2989f0c32013", + "installed_by": [ + "modules" + ] }, "samtools/sort": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "a0f7be95788366c1923171e358da7d049eb440f9", + "installed_by": [ + "modules" + ] }, "samtools/stats": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "735e1e04e7e01751d2d6e97055bbdb6f70683cc1", + "installed_by": [ + "modules" + ] }, "snpeff/snpeff": { "branch": "master", - "git_sha": "03eae88f7c30fbad0152b941bbeba48ae8bc3d07", - "installed_by": ["modules"] + "git_sha": "4d584d5cf6ed5f7363a51cdb4b3eb25398e9e537", + "installed_by": [ + "modules" + ] }, "star/align": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3", + "installed_by": [ + "modules" + ] }, "star/genomegenerate": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3", + "installed_by": [ + "modules" + ] }, "tabix/bgziptabix": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "591b71642820933dcb3c954c934b397bd00d8e5e", + "installed_by": [ + "modules" + ] }, "tabix/tabix": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", - "installed_by": ["modules"] + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "installed_by": [ + "modules" + ] }, "untar": { "branch": "master", - "git_sha": "cc1f997fab6d8fde5dc0e6e2a310814df5b53ce7", - "installed_by": ["modules"] + "git_sha": "d0b4fc03af52a1cc8c6fb4493b921b57352b1dd8", + "installed_by": [ + "modules" + ] } } }, @@ -182,4 +250,4 @@ } } } -} +} \ No newline at end of file diff --git a/modules/local/custom/dumpsoftwareversions/main.nf b/modules/local/custom/dumpsoftwareversions/main.nf index 3df21765..c9d014b1 100644 --- a/modules/local/custom/dumpsoftwareversions/main.nf +++ b/modules/local/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.13" + conda "bioconda::multiqc=1.15" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.15--pyhdfd78af_0' : + 'biocontainers/multiqc:1.15--pyhdfd78af_0' }" input: path versions diff --git a/modules/local/gtf2bed.nf b/modules/local/gtf2bed.nf index 42589bbd..b0c7e42a 100755 --- a/modules/local/gtf2bed.nf +++ b/modules/local/gtf2bed.nf @@ -5,7 +5,7 @@ process GTF2BED { conda "conda-forge::r-base=3.5.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/r-base:3.5.0' : - 'quay.io/biocontainers/r-base:3.5.0'}" + 'biocontainers/r-base:3.5.0'}" input: path gtf diff --git a/modules/nf-core/bedtools/merge/main.nf b/modules/nf-core/bedtools/merge/main.nf index 21b2e645..6868d39f 100644 --- a/modules/nf-core/bedtools/merge/main.nf +++ b/modules/nf-core/bedtools/merge/main.nf @@ -2,10 +2,10 @@ process BEDTOOLS_MERGE { tag "$meta.id" label 'process_single' - conda "bioconda::bedtools=2.30.0" + conda "bioconda::bedtools=2.31.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bedtools:2.30.0--hc088bd4_0' : - 'quay.io/biocontainers/bedtools:2.30.0--hc088bd4_0' }" + 'https://depot.galaxyproject.org/singularity/bedtools:2.31.0--hf5e1c6e_2' : + 'biocontainers/bedtools:2.31.0--hf5e1c6e_2' }" input: tuple val(meta), path(bed) @@ -33,4 +33,15 @@ process BEDTOOLS_MERGE { bedtools: \$(bedtools --version | sed -e "s/bedtools v//g") END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.bed + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + bedtools: \$(bedtools --version | sed -e "s/bedtools v//g") + END_VERSIONS + """ } diff --git a/modules/nf-core/bedtools/merge/meta.yml b/modules/nf-core/bedtools/merge/meta.yml index 76743679..82248afe 100644 --- a/modules/nf-core/bedtools/merge/meta.yml +++ b/modules/nf-core/bedtools/merge/meta.yml @@ -3,6 +3,8 @@ description: combines overlapping or “book-ended” features in an interval fi keywords: - bed - merge + - bedtools + - overlapped bed tools: - bedtools: description: | diff --git a/modules/nf-core/bedtools/sort/main.nf b/modules/nf-core/bedtools/sort/main.nf index a0ddddd1..df372bc5 100644 --- a/modules/nf-core/bedtools/sort/main.nf +++ b/modules/nf-core/bedtools/sort/main.nf @@ -2,10 +2,10 @@ process BEDTOOLS_SORT { tag "$meta.id" label 'process_single' - conda "bioconda::bedtools=2.30.0" + conda "bioconda::bedtools=2.31.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bedtools:2.30.0--hc088bd4_0' : - 'quay.io/biocontainers/bedtools:2.30.0--hc088bd4_0' }" + 'https://depot.galaxyproject.org/singularity/bedtools:2.31.0--hf5e1c6e_2' : + 'biocontainers/bedtools:2.31.0--hf5e1c6e_2' }" input: tuple val(meta), path(intervals) @@ -39,4 +39,16 @@ process BEDTOOLS_SORT { bedtools: \$(bedtools --version | sed -e "s/bedtools v//g") END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + extension = task.ext.suffix ?: intervals.extension + """ + touch ${prefix}.${extension} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + bedtools: \$(bedtools --version | sed -e "s/bedtools v//g") + END_VERSIONS + """ } diff --git a/modules/nf-core/bedtools/sort/meta.yml b/modules/nf-core/bedtools/sort/meta.yml index 1b6ebbcb..3a3b4e4d 100644 --- a/modules/nf-core/bedtools/sort/meta.yml +++ b/modules/nf-core/bedtools/sort/meta.yml @@ -3,6 +3,8 @@ description: Sorts a feature file by chromosome and other criteria. keywords: - bed - sort + - bedtools + - chromosome tools: - bedtools: description: | diff --git a/modules/nf-core/cat/fastq/main.nf b/modules/nf-core/cat/fastq/main.nf index 8a0b5600..5021e6fc 100644 --- a/modules/nf-core/cat/fastq/main.nf +++ b/modules/nf-core/cat/fastq/main.nf @@ -5,7 +5,7 @@ process CAT_FASTQ { conda "conda-forge::sed=4.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'ubuntu:20.04' }" + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(reads, stageAs: "input*/*") diff --git a/modules/nf-core/cat/fastq/meta.yml b/modules/nf-core/cat/fastq/meta.yml index c836598e..8a39e309 100644 --- a/modules/nf-core/cat/fastq/meta.yml +++ b/modules/nf-core/cat/fastq/meta.yml @@ -1,6 +1,7 @@ name: cat_fastq description: Concatenates fastq files keywords: + - cat - fastq - concatenate tools: @@ -16,7 +17,7 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - reads: - type: list + type: file description: | List of input FastQ files to be concatenated. output: diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 800a6099..c9d014b1 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.14" + conda "bioconda::multiqc=1.15" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.15--pyhdfd78af_0' : + 'biocontainers/multiqc:1.15--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/ensemblvep/vep/main.nf b/modules/nf-core/ensemblvep/vep/main.nf index e9d25f3d..da0e3646 100644 --- a/modules/nf-core/ensemblvep/vep/main.nf +++ b/modules/nf-core/ensemblvep/vep/main.nf @@ -2,26 +2,26 @@ process ENSEMBLVEP_VEP { tag "$meta.id" label 'process_medium' - conda "bioconda::ensembl-vep=108.2" + conda "bioconda::ensembl-vep=110.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ensembl-vep:108.2--pl5321h4a94de4_0' : - 'quay.io/biocontainers/ensembl-vep:108.2--pl5321h4a94de4_0' }" + 'https://depot.galaxyproject.org/singularity/ensembl-vep:110.0--pl5321h2a3209d_0' : + 'biocontainers/ensembl-vep:110.0--pl5321h2a3209d_0' }" input: - tuple val(meta), path(vcf) + tuple val(meta), path(vcf), path(custom_extra_files) val genome val species val cache_version path cache - path fasta + tuple val(meta2), path(fasta) path extra_files output: tuple val(meta), path("*.vcf.gz") , optional:true, emit: vcf tuple val(meta), path("*.tab.gz") , optional:true, emit: tab tuple val(meta), path("*.json.gz") , optional:true, emit: json - path "*.summary.html" , emit: report - path "versions.yml" , emit: versions + path "*.summary.html" , emit: report + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -33,7 +33,6 @@ process ENSEMBLVEP_VEP { def prefix = task.ext.prefix ?: "${meta.id}" def dir_cache = cache ? "\${PWD}/${cache}" : "/.vep" def reference = fasta ? "--fasta $fasta" : "" - """ vep \\ -i $vcf \\ @@ -59,9 +58,9 @@ process ENSEMBLVEP_VEP { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.ann.vcf.gz - touch ${prefix}.ann.tab.gz - touch ${prefix}.ann.json.gz + touch ${prefix}.vcf.gz + touch ${prefix}.tab.gz + touch ${prefix}.json.gz touch ${prefix}.summary.html cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/ensemblvep/vep/meta.yml b/modules/nf-core/ensemblvep/vep/meta.yml index 3a4f8d1d..7783847d 100644 --- a/modules/nf-core/ensemblvep/vep/meta.yml +++ b/modules/nf-core/ensemblvep/vep/meta.yml @@ -2,6 +2,9 @@ name: ENSEMBLVEP_VEP description: Ensembl Variant Effect Predictor (VEP). The output-file-format is controlled through `task.ext.args`. keywords: - annotation + - vcf + - json + - tab tools: - ensemblvep: description: | @@ -20,29 +23,39 @@ input: type: file description: | vcf to annotate + - custom_extra_files: + type: file + description: | + extra sample-specific files to be used with the `--custom` flag to be configured with ext.args + (optional) - genome: - type: value + type: string description: | which genome to annotate with - species: - type: value + type: string description: | which species to annotate with - cache_version: - type: value + type: integer description: | which version of the cache to annotate with - cache: type: file description: | path to VEP cache (optional) + - meta2: + type: map + description: | + Groovy Map containing fasta reference information + e.g. [ id:'test' ] - fasta: type: file description: | reference FASTA file (optional) pattern: "*.{fasta,fa}" - extra_files: - type: tuple + type: file description: | path to file(s) needed for plugins (optional) output: @@ -72,3 +85,4 @@ output: authors: - "@maxulysse" - "@matthdsm" + - "@nvnieuwk" diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 9ae58381..249f9064 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -5,7 +5,7 @@ process FASTQC { conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - 'quay.io/biocontainers/fastqc:0.11.9--0' }" + 'biocontainers/fastqc:0.11.9--0' }" input: tuple val(meta), path(reads) @@ -29,7 +29,11 @@ process FASTQC { printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name done - fastqc $args --threads $task.cpus $renamed_files + + fastqc \\ + $args \\ + --threads $task.cpus \\ + $renamed_files cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/fastqc/tests/main.nf.test b/modules/nf-core/fastqc/tests/main.nf.test new file mode 100644 index 00000000..3961de60 --- /dev/null +++ b/modules/nf-core/fastqc/tests/main.nf.test @@ -0,0 +1,32 @@ +nextflow_process { + + name "Test Process FASTQC" + script "modules/nf-core/fastqc/main.nf" + process "FASTQC" + tag "fastqc" + + test("Single-Read") { + + when { + process { + """ + input[0] = [ + [ id: 'test', single_end:true ], + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assert process.success + assert process.out.html.get(0).get(1) ==~ ".*/test_fastqc.html" + assert path(process.out.html.get(0).get(1)).getText().contains("File typeConventional base calls") + assert process.out.zip.get(0).get(1) ==~ ".*/test_fastqc.zip" + } + + } + +} diff --git a/modules/nf-core/gatk4/applybqsr/main.nf b/modules/nf-core/gatk4/applybqsr/main.nf index 9dc6ac68..b515f1c5 100644 --- a/modules/nf-core/gatk4/applybqsr/main.nf +++ b/modules/nf-core/gatk4/applybqsr/main.nf @@ -2,10 +2,10 @@ process GATK4_APPLYBQSR { tag "$meta.id" label 'process_low' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(input), path(input_index), path(bqsr_table), path(intervals) @@ -26,14 +26,14 @@ process GATK4_APPLYBQSR { def prefix = task.ext.prefix ?: "${meta.id}" def interval_command = intervals ? "--intervals $intervals" : "" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK ApplyBQSR] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" ApplyBQSR \\ + gatk --java-options "-Xmx${avail_mem}M" ApplyBQSR \\ --input $input \\ --output ${prefix}.${input.getExtension()} \\ --reference $fasta \\ diff --git a/modules/nf-core/gatk4/applybqsr/meta.yml b/modules/nf-core/gatk4/applybqsr/meta.yml index 3fc93f10..3002ab60 100644 --- a/modules/nf-core/gatk4/applybqsr/meta.yml +++ b/modules/nf-core/gatk4/applybqsr/meta.yml @@ -3,16 +3,17 @@ description: Apply base quality score recalibration (BQSR) to a bam file keywords: - bqsr - bam + - base quality score recalibration tools: - gatk4: - description: | - Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools - with a primary focus on variant discovery and genotyping. Its powerful processing engine - and high-performance computing features make it capable of taking on projects of any size. - homepage: https://gatk.broadinstitute.org/hc/en-us - documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s - doi: 10.1158/1538-7445.AM2017-3590 - licence: ["Apache-2.0"] + description: | + Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools + with a primary focus on variant discovery and genotyping. Its powerful processing engine + and high-performance computing features make it capable of taking on projects of any size. + homepage: https://gatk.broadinstitute.org/hc/en-us + documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s + doi: 10.1158/1538-7445.AM2017-3590 + licence: ["Apache-2.0"] input: - meta: diff --git a/modules/nf-core/gatk4/baserecalibrator/main.nf b/modules/nf-core/gatk4/baserecalibrator/main.nf index b668a1fc..318703a4 100644 --- a/modules/nf-core/gatk4/baserecalibrator/main.nf +++ b/modules/nf-core/gatk4/baserecalibrator/main.nf @@ -2,10 +2,10 @@ process GATK4_BASERECALIBRATOR { tag "$meta.id" label 'process_low' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(input), path(input_index), path(intervals) @@ -28,14 +28,14 @@ process GATK4_BASERECALIBRATOR { def interval_command = intervals ? "--intervals $intervals" : "" def sites_command = known_sites.collect{"--known-sites $it"}.join(' ') - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK BaseRecalibrator] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" BaseRecalibrator \\ + gatk --java-options "-Xmx${avail_mem}M" BaseRecalibrator \\ --input $input \\ --output ${prefix}.table \\ --reference $fasta \\ diff --git a/modules/nf-core/gatk4/baserecalibrator/meta.yml b/modules/nf-core/gatk4/baserecalibrator/meta.yml index 08c1ebbf..a6b06c73 100644 --- a/modules/nf-core/gatk4/baserecalibrator/meta.yml +++ b/modules/nf-core/gatk4/baserecalibrator/meta.yml @@ -4,14 +4,14 @@ keywords: - sort tools: - gatk4: - description: | - Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools - with a primary focus on variant discovery and genotyping. Its powerful processing engine - and high-performance computing features make it capable of taking on projects of any size. - homepage: https://gatk.broadinstitute.org/hc/en-us - documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s - doi: 10.1158/1538-7445.AM2017-3590 - licence: ["Apache-2.0"] + description: | + Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools + with a primary focus on variant discovery and genotyping. Its powerful processing engine + and high-performance computing features make it capable of taking on projects of any size. + homepage: https://gatk.broadinstitute.org/hc/en-us + documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s + doi: 10.1158/1538-7445.AM2017-3590 + licence: ["Apache-2.0"] input: - meta: diff --git a/modules/nf-core/gatk4/bedtointervallist/main.nf b/modules/nf-core/gatk4/bedtointervallist/main.nf index 41fab003..a23abd06 100644 --- a/modules/nf-core/gatk4/bedtointervallist/main.nf +++ b/modules/nf-core/gatk4/bedtointervallist/main.nf @@ -2,14 +2,14 @@ process GATK4_BEDTOINTERVALLIST { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(bed) - path dict + tuple val(meta2), path(dict) output: tuple val(meta), path('*.interval_list'), emit: interval_list @@ -22,14 +22,14 @@ process GATK4_BEDTOINTERVALLIST { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK BedToIntervalList] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" BedToIntervalList \\ + gatk --java-options "-Xmx${avail_mem}M" BedToIntervalList \\ --INPUT $bed \\ --OUTPUT ${prefix}.interval_list \\ --SEQUENCE_DICTIONARY $dict \\ diff --git a/modules/nf-core/gatk4/bedtointervallist/meta.yml b/modules/nf-core/gatk4/bedtointervallist/meta.yml index 986f1592..40daf752 100644 --- a/modules/nf-core/gatk4/bedtointervallist/meta.yml +++ b/modules/nf-core/gatk4/bedtointervallist/meta.yml @@ -3,6 +3,7 @@ description: Creates an interval list from a bed file and a reference dict keywords: - bed - interval list + - bedtointervallist tools: - gatk4: description: | @@ -23,6 +24,11 @@ input: type: file description: Input bed file pattern: "*.bed" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - dict: type: file description: Sequence dictionary @@ -38,3 +44,4 @@ output: pattern: "versions.yml" authors: - "@kevinmenden" + - "@ramprasadn" diff --git a/modules/nf-core/gatk4/combinegvcfs/main.nf b/modules/nf-core/gatk4/combinegvcfs/main.nf index 035c171b..152fef42 100644 --- a/modules/nf-core/gatk4/combinegvcfs/main.nf +++ b/modules/nf-core/gatk4/combinegvcfs/main.nf @@ -2,10 +2,10 @@ process GATK4_COMBINEGVCFS { tag "$meta.id" label 'process_low' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(vcf), path(vcf_idx) @@ -25,14 +25,14 @@ process GATK4_COMBINEGVCFS { def prefix = task.ext.prefix ?: "${meta.id}" def input_list = vcf.collect{"--variant $it"}.join(' ') - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK COMBINEGVCFS] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" CombineGVCFs \\ + gatk --java-options "-Xmx${avail_mem}M" CombineGVCFs \\ $input_list \\ --output ${prefix}.combined.g.vcf.gz \\ --reference ${fasta} \\ diff --git a/modules/nf-core/gatk4/createsequencedictionary/main.nf b/modules/nf-core/gatk4/createsequencedictionary/main.nf index bc324ada..15a86bea 100644 --- a/modules/nf-core/gatk4/createsequencedictionary/main.nf +++ b/modules/nf-core/gatk4/createsequencedictionary/main.nf @@ -2,17 +2,17 @@ process GATK4_CREATESEQUENCEDICTIONARY { tag "$fasta" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: - path fasta + tuple val(meta), path(fasta) output: - path "*.dict" , emit: dict - path "versions.yml" , emit: versions + tuple val(meta), path('*.dict') , emit: dict + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -20,14 +20,14 @@ process GATK4_CREATESEQUENCEDICTIONARY { script: def args = task.ext.args ?: '' - def avail_mem = 6 + def avail_mem = 6144 if (!task.memory) { log.info '[GATK CreateSequenceDictionary] Available memory not known - defaulting to 6GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" CreateSequenceDictionary \\ + gatk --java-options "-Xmx${avail_mem}M" CreateSequenceDictionary \\ --REFERENCE $fasta \\ --URI $fasta \\ --TMP_DIR . \\ diff --git a/modules/nf-core/gatk4/createsequencedictionary/meta.yml b/modules/nf-core/gatk4/createsequencedictionary/meta.yml index bd247888..a421e681 100644 --- a/modules/nf-core/gatk4/createsequencedictionary/meta.yml +++ b/modules/nf-core/gatk4/createsequencedictionary/meta.yml @@ -3,30 +3,37 @@ description: Creates a sequence dictionary for a reference sequence keywords: - dictionary - fasta + - createsequencedictionary tools: - gatk: - description: | - Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools - with a primary focus on variant discovery and genotyping. Its powerful processing engine - and high-performance computing features make it capable of taking on projects of any size. - homepage: https://gatk.broadinstitute.org/hc/en-us - documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s - doi: 10.1158/1538-7445.AM2017-3590 - licence: ["Apache-2.0"] + description: | + Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools + with a primary focus on variant discovery and genotyping. Its powerful processing engine + and high-performance computing features make it capable of taking on projects of any size. + homepage: https://gatk.broadinstitute.org/hc/en-us + documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s + doi: 10.1158/1538-7445.AM2017-3590 + licence: ["Apache-2.0"] input: + - meta: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - fasta: - type: file - description: Input fasta file - pattern: "*.{fasta,fa}" + type: file + description: Input fasta file + pattern: "*.{fasta,fa}" output: - dict: - type: file - description: gatk dictionary file - pattern: "*.{dict}" + type: file + description: gatk dictionary file + pattern: "*.{dict}" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@maxulysse" + - "@ramprasadn" diff --git a/modules/nf-core/gatk4/haplotypecaller/main.nf b/modules/nf-core/gatk4/haplotypecaller/main.nf index c011376a..478681bd 100644 --- a/modules/nf-core/gatk4/haplotypecaller/main.nf +++ b/modules/nf-core/gatk4/haplotypecaller/main.nf @@ -2,10 +2,10 @@ process GATK4_HAPLOTYPECALLER { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(input), path(input_index), path(intervals), path(dragstr_model) @@ -32,14 +32,14 @@ process GATK4_HAPLOTYPECALLER { def dragstr_command = dragstr_model ? "--dragstr-params-path $dragstr_model" : "" def bamout_command = args.contains("--bam-writer-type") ? "--bam-output ${prefix.replaceAll('.g\\s*$', '')}.realigned.bam" : "" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK HaplotypeCaller] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" HaplotypeCaller \\ + gatk --java-options "-Xmx${avail_mem}M" HaplotypeCaller \\ --input $input \\ --output ${prefix}.vcf.gz \\ --reference $fasta \\ diff --git a/modules/nf-core/gatk4/haplotypecaller/meta.yml b/modules/nf-core/gatk4/haplotypecaller/meta.yml index cef608b4..27633cca 100644 --- a/modules/nf-core/gatk4/haplotypecaller/meta.yml +++ b/modules/nf-core/gatk4/haplotypecaller/meta.yml @@ -6,14 +6,14 @@ keywords: - haplotype tools: - gatk4: - description: | - Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools - with a primary focus on variant discovery and genotyping. Its powerful processing engine - and high-performance computing features make it capable of taking on projects of any size. - homepage: https://gatk.broadinstitute.org/hc/en-us - documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s - doi: 10.1158/1538-7445.AM2017-3590 - licence: ["Apache-2.0"] + description: | + Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools + with a primary focus on variant discovery and genotyping. Its powerful processing engine + and high-performance computing features make it capable of taking on projects of any size. + homepage: https://gatk.broadinstitute.org/hc/en-us + documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s + doi: 10.1158/1538-7445.AM2017-3590 + licence: ["Apache-2.0"] input: - meta: diff --git a/modules/nf-core/gatk4/indexfeaturefile/main.nf b/modules/nf-core/gatk4/indexfeaturefile/main.nf index 9793c90c..d3bb04a7 100644 --- a/modules/nf-core/gatk4/indexfeaturefile/main.nf +++ b/modules/nf-core/gatk4/indexfeaturefile/main.nf @@ -2,10 +2,10 @@ process GATK4_INDEXFEATUREFILE { tag "$meta.id" label 'process_low' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(feature_file) @@ -20,14 +20,14 @@ process GATK4_INDEXFEATUREFILE { script: def args = task.ext.args ?: '' - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK IndexFeatureFile] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" IndexFeatureFile \\ + gatk --java-options "-Xmx${avail_mem}M" IndexFeatureFile \\ --input $feature_file \\ --tmp-dir . \\ $args diff --git a/modules/nf-core/gatk4/intervallisttools/main.nf b/modules/nf-core/gatk4/intervallisttools/main.nf index 84da7bf7..0054659a 100644 --- a/modules/nf-core/gatk4/intervallisttools/main.nf +++ b/modules/nf-core/gatk4/intervallisttools/main.nf @@ -2,10 +2,10 @@ process GATK4_INTERVALLISTTOOLS { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(intervals) @@ -21,17 +21,17 @@ process GATK4_INTERVALLISTTOOLS { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK IntervalListTools] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ mkdir ${prefix}_split - gatk --java-options "-Xmx${avail_mem}g" IntervalListTools \\ + gatk --java-options "-Xmx${avail_mem}M" IntervalListTools \\ --INPUT $intervals \\ --OUTPUT ${prefix}_split \\ --TMP_DIR . \\ diff --git a/modules/nf-core/gatk4/markduplicates/main.nf b/modules/nf-core/gatk4/markduplicates/main.nf index af9a7cf2..f4b3f6dc 100644 --- a/modules/nf-core/gatk4/markduplicates/main.nf +++ b/modules/nf-core/gatk4/markduplicates/main.nf @@ -2,10 +2,10 @@ process GATK4_MARKDUPLICATES { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0 bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-d9e7bad0f7fbc8f4458d5c3ab7ffaaf0235b59fb:f857e2d6cc88d35580d01cf39e0959a68b83c1d9-0': + 'biocontainers/mulled-v2-d9e7bad0f7fbc8f4458d5c3ab7ffaaf0235b59fb:f857e2d6cc88d35580d01cf39e0959a68b83c1d9-0' }" input: tuple val(meta), path(bam) @@ -25,33 +25,43 @@ process GATK4_MARKDUPLICATES { script: def args = task.ext.args ?: '' - prefix = task.ext.prefix ?: "${meta.id}" + prefix = task.ext.prefix ?: "${meta.id}.bam" + + // If the extension is CRAM, then change it to BAM + prefix_bam = prefix.tokenize('.')[-1] == 'cram' ? "${prefix.substring(0, prefix.lastIndexOf('.'))}.bam" : prefix + def input_list = bam.collect{"--INPUT $it"}.join(' ') def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : "" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK MarkDuplicates] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } + + // Using samtools and not Markduplicates to compress to CRAM speeds up computation: + // https://medium.com/@acarroll.dna/looking-at-trade-offs-in-compression-levels-for-genomics-tools-eec2834e8b94 """ - gatk --java-options "-Xmx${avail_mem}g" MarkDuplicates \\ + gatk --java-options "-Xmx${avail_mem}M" MarkDuplicates \\ $input_list \\ - --OUTPUT ${prefix} \\ + --OUTPUT ${prefix_bam} \\ --METRICS_FILE ${prefix}.metrics \\ --TMP_DIR . \\ ${reference} \\ $args - - if [[ ${prefix} == *.cram ]]; then - mv ${prefix}.bai ${prefix}.crai + # If cram files are wished as output, the run samtools for conversion + if [[ ${prefix} == *.cram ]]; then + samtools view -Ch -T ${fasta} -o ${prefix} ${prefix_bam} + rm ${prefix_bam} + samtools index ${prefix} fi cat <<-END_VERSIONS > versions.yml "${task.process}": gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') END_VERSIONS """ } diff --git a/modules/nf-core/gatk4/mergevcfs/main.nf b/modules/nf-core/gatk4/mergevcfs/main.nf index d418468f..dfb5b33a 100644 --- a/modules/nf-core/gatk4/mergevcfs/main.nf +++ b/modules/nf-core/gatk4/mergevcfs/main.nf @@ -2,10 +2,10 @@ process GATK4_MERGEVCFS { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(vcf) @@ -25,14 +25,14 @@ process GATK4_MERGEVCFS { def input_list = vcf.collect{ "--INPUT $it"}.join(' ') def reference_command = dict ? "--SEQUENCE_DICTIONARY $dict" : "" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK MergeVcfs] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" MergeVcfs \\ + gatk --java-options "-Xmx${avail_mem}M" MergeVcfs \\ $input_list \\ --OUTPUT ${prefix}.vcf.gz \\ $reference_command \\ diff --git a/modules/nf-core/gatk4/splitncigarreads/main.nf b/modules/nf-core/gatk4/splitncigarreads/main.nf index 77092a29..0178976c 100644 --- a/modules/nf-core/gatk4/splitncigarreads/main.nf +++ b/modules/nf-core/gatk4/splitncigarreads/main.nf @@ -2,10 +2,10 @@ process GATK4_SPLITNCIGARREADS { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(bam), path(bai), path(intervals) @@ -25,14 +25,14 @@ process GATK4_SPLITNCIGARREADS { def prefix = task.ext.prefix ?: "${meta.id}" def interval_command = intervals ? "--intervals $intervals" : "" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK SplitNCigarReads] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}g" SplitNCigarReads \\ + gatk --java-options "-Xmx${avail_mem}M" SplitNCigarReads \\ --input $bam \\ --output ${prefix}.bam \\ --reference $fasta \\ diff --git a/modules/nf-core/gatk4/variantfiltration/main.nf b/modules/nf-core/gatk4/variantfiltration/main.nf index e2837a91..387ff8ca 100644 --- a/modules/nf-core/gatk4/variantfiltration/main.nf +++ b/modules/nf-core/gatk4/variantfiltration/main.nf @@ -2,16 +2,16 @@ process GATK4_VARIANTFILTRATION { tag "$meta.id" label 'process_medium' - conda "bioconda::gatk4=4.3.0.0" + conda "bioconda::gatk4=4.4.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gatk4:4.3.0.0--py36hdfd78af_0': - 'quay.io/biocontainers/gatk4:4.3.0.0--py36hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0': + 'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }" input: tuple val(meta), path(vcf), path(tbi) - path fasta - path fai - path dict + tuple val(meta2), path(fasta) + tuple val(meta3), path(fai) + tuple val(meta4), path(dict) output: tuple val(meta), path("*.vcf.gz"), emit: vcf @@ -25,14 +25,14 @@ process GATK4_VARIANTFILTRATION { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[GATK VariantFiltration] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.toGiga() + avail_mem = (task.memory.mega*0.8).intValue() } """ - gatk --java-options "-Xmx${avail_mem}G" VariantFiltration \\ + gatk --java-options "-Xmx${avail_mem}M" VariantFiltration \\ --variant $vcf \\ --output ${prefix}.vcf.gz \\ --reference $fasta \\ diff --git a/modules/nf-core/gatk4/variantfiltration/meta.yml b/modules/nf-core/gatk4/variantfiltration/meta.yml index 04b1c086..2260f37b 100644 --- a/modules/nf-core/gatk4/variantfiltration/meta.yml +++ b/modules/nf-core/gatk4/variantfiltration/meta.yml @@ -3,6 +3,7 @@ description: Filter variants keywords: - vcf - filter + - variantfiltration tools: - gatk4: description: | @@ -27,14 +28,29 @@ input: type: list description: List of VCF file indexes pattern: "*.{idx,tbi}" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - fasta: type: file description: Fasta file of reference genome pattern: "*.fasta" + - meta3: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - fai: type: file description: Index of fasta file pattern: "*.fasta.fai" + - meta4: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - dict: type: file description: Sequence dictionary of fastea file @@ -54,3 +70,4 @@ output: pattern: "versions.yml" authors: - "@kevinmenden" + - "@ramprasadn" diff --git a/modules/nf-core/gffread/main.nf b/modules/nf-core/gffread/main.nf index b1a8996f..f4472b0e 100644 --- a/modules/nf-core/gffread/main.nf +++ b/modules/nf-core/gffread/main.nf @@ -5,7 +5,7 @@ process GFFREAD { conda "bioconda::gffread=0.12.1" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/gffread:0.12.1--h8b12597_0' : - 'quay.io/biocontainers/gffread:0.12.1--h8b12597_0' }" + 'biocontainers/gffread:0.12.1--h8b12597_0' }" input: path gff diff --git a/modules/nf-core/gunzip/main.nf b/modules/nf-core/gunzip/main.nf index d906034c..73bf08cd 100644 --- a/modules/nf-core/gunzip/main.nf +++ b/modules/nf-core/gunzip/main.nf @@ -5,7 +5,7 @@ process GUNZIP { conda "conda-forge::sed=4.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'ubuntu:20.04' }" + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(archive) @@ -21,10 +21,14 @@ process GUNZIP { def args = task.ext.args ?: '' gunzip = archive.toString() - '.gz' """ - gunzip \\ - -f \\ + # Not calling gunzip itself because it creates files + # with the original group ownership rather than the + # default one for that user / the work directory + gzip \\ + -cd \\ $args \\ - $archive + $archive \\ + > $gunzip cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/gunzip/meta.yml b/modules/nf-core/gunzip/meta.yml index 4d2ebc84..4cdcdf4c 100644 --- a/modules/nf-core/gunzip/meta.yml +++ b/modules/nf-core/gunzip/meta.yml @@ -3,31 +3,32 @@ description: Compresses and decompresses files. keywords: - gunzip - compression + - decompression tools: - gunzip: - description: | - gzip is a file format and a software application used for file compression and decompression. - documentation: https://www.gnu.org/software/gzip/manual/gzip.html - licence: ["GPL-3.0-or-later"] + description: | + gzip is a file format and a software application used for file compression and decompression. + documentation: https://www.gnu.org/software/gzip/manual/gzip.html + licence: ["GPL-3.0-or-later"] input: - meta: - type: map - description: | - Optional groovy Map containing meta information - e.g. [ id:'test', single_end:false ] + type: map + description: | + Optional groovy Map containing meta information + e.g. [ id:'test', single_end:false ] - archive: - type: file - description: File to be compressed/uncompressed - pattern: "*.*" + type: file + description: File to be compressed/uncompressed + pattern: "*.*" output: - gunzip: - type: file - description: Compressed/uncompressed file - pattern: "*.*" + type: file + description: Compressed/uncompressed file + pattern: "*.*" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@joseespinosa" - "@drpatelh" diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 4b604749..65d7dd0d 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.14" + conda "bioconda::multiqc=1.15" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.15--pyhdfd78af_0' : + 'biocontainers/multiqc:1.15--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/samtools/faidx/main.nf b/modules/nf-core/samtools/faidx/main.nf index ce6580d2..59ed3088 100644 --- a/modules/nf-core/samtools/faidx/main.nf +++ b/modules/nf-core/samtools/faidx/main.nf @@ -2,18 +2,20 @@ process SAMTOOLS_FAIDX { tag "$fasta" label 'process_single' - conda "bioconda::samtools=1.16.1" + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.16.1--h6899075_1' : - 'quay.io/biocontainers/samtools:1.16.1--h6899075_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(fasta) + tuple val(meta2), path(fai) output: - tuple val(meta), path ("*.fai"), emit: fai - tuple val(meta), path ("*.gzi"), emit: gzi, optional: true - path "versions.yml" , emit: versions + tuple val(meta), path ("*.{fa,fasta}") , emit: fa , optional: true + tuple val(meta), path ("*.fai") , emit: fai, optional: true + tuple val(meta), path ("*.gzi") , emit: gzi, optional: true + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -23,8 +25,8 @@ process SAMTOOLS_FAIDX { """ samtools \\ faidx \\ - $args \\ - $fasta + $fasta \\ + $args cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -33,8 +35,12 @@ process SAMTOOLS_FAIDX { """ stub: + def match = (task.ext.args =~ /-o(?:utput)?\s(.*)\s?/).findAll() + def fastacmd = match[0] ? "touch ${match[0][1]}" : '' """ + ${fastacmd} touch ${fasta}.fai + cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/samtools/faidx/meta.yml b/modules/nf-core/samtools/faidx/meta.yml index fe2fe9a1..957b25e5 100644 --- a/modules/nf-core/samtools/faidx/meta.yml +++ b/modules/nf-core/samtools/faidx/meta.yml @@ -3,6 +3,7 @@ description: Index FASTA file keywords: - index - fasta + - faidx tools: - samtools: description: | @@ -17,12 +18,21 @@ input: - meta: type: map description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing reference information + e.g. [ id:'test' ] - fasta: type: file description: FASTA file pattern: "*.{fa,fasta}" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'test' ] + - fai: + type: file + description: FASTA index file + pattern: "*.{fai}" output: - meta: type: map diff --git a/modules/nf-core/samtools/flagstat/main.nf b/modules/nf-core/samtools/flagstat/main.nf index 2120cd7d..b75707ec 100644 --- a/modules/nf-core/samtools/flagstat/main.nf +++ b/modules/nf-core/samtools/flagstat/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_FLAGSTAT { tag "$meta.id" label 'process_single' - conda "bioconda::samtools=1.16.1" + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.16.1--h6899075_1' : - 'quay.io/biocontainers/samtools:1.16.1--h6899075_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(bam), path(bai) @@ -32,4 +32,15 @@ process SAMTOOLS_FLAGSTAT { samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.flagstat + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS + """ } diff --git a/modules/nf-core/samtools/flagstat/meta.yml b/modules/nf-core/samtools/flagstat/meta.yml index 95269063..954225df 100644 --- a/modules/nf-core/samtools/flagstat/meta.yml +++ b/modules/nf-core/samtools/flagstat/meta.yml @@ -14,7 +14,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/idxstats/main.nf b/modules/nf-core/samtools/idxstats/main.nf index a7b87d8b..83c7c34b 100644 --- a/modules/nf-core/samtools/idxstats/main.nf +++ b/modules/nf-core/samtools/idxstats/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_IDXSTATS { tag "$meta.id" label 'process_single' - conda "bioconda::samtools=1.16.1" + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.16.1--h6899075_1' : - 'quay.io/biocontainers/samtools:1.16.1--h6899075_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(bam), path(bai) @@ -33,4 +33,16 @@ process SAMTOOLS_IDXSTATS { samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + touch ${prefix}.idxstats + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS + """ } diff --git a/modules/nf-core/samtools/idxstats/meta.yml b/modules/nf-core/samtools/idxstats/meta.yml index 3710ab88..dda87e1e 100644 --- a/modules/nf-core/samtools/idxstats/meta.yml +++ b/modules/nf-core/samtools/idxstats/meta.yml @@ -15,7 +15,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf index 8b95687a..0b20aa4b 100644 --- a/modules/nf-core/samtools/index/main.nf +++ b/modules/nf-core/samtools/index/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_INDEX { tag "$meta.id" label 'process_low' - conda "bioconda::samtools=1.16.1" + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.16.1--h6899075_1' : - 'quay.io/biocontainers/samtools:1.16.1--h6899075_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(input) diff --git a/modules/nf-core/samtools/index/meta.yml b/modules/nf-core/samtools/index/meta.yml index e5cadbc2..8bd2fa6f 100644 --- a/modules/nf-core/samtools/index/meta.yml +++ b/modules/nf-core/samtools/index/meta.yml @@ -12,7 +12,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/merge/main.nf b/modules/nf-core/samtools/merge/main.nf index a80ff3a2..b73b7cb2 100644 --- a/modules/nf-core/samtools/merge/main.nf +++ b/modules/nf-core/samtools/merge/main.nf @@ -2,15 +2,15 @@ process SAMTOOLS_MERGE { tag "$meta.id" label 'process_low' - conda "bioconda::samtools=1.16.1" + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.16.1--h6899075_1' : - 'quay.io/biocontainers/samtools:1.16.1--h6899075_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(input_files, stageAs: "?/*") - path fasta - path fai + tuple val(meta2), path(fasta) + tuple val(meta3), path(fai) output: tuple val(meta), path("${prefix}.bam") , optional:true, emit: bam diff --git a/modules/nf-core/samtools/merge/meta.yml b/modules/nf-core/samtools/merge/meta.yml index 5bd84bc5..3a815f74 100644 --- a/modules/nf-core/samtools/merge/meta.yml +++ b/modules/nf-core/samtools/merge/meta.yml @@ -12,7 +12,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: @@ -25,13 +25,23 @@ input: type: file description: BAM/CRAM file pattern: "*.{bam,cram,sam}" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - fasta: - type: optional file - description: Reference file the CRAM was created with + type: file + description: Reference file the CRAM was created with (optional) pattern: "*.{fasta,fa}" + - meta3: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - fai: - type: optional file - description: Index of the reference file the CRAM was created with + type: file + description: Index of the reference file the CRAM was created with (optional) pattern: "*.fai" output: - meta: @@ -60,3 +70,4 @@ authors: - "@yuukiiwa " - "@maxulysse" - "@FriederikeHanssen" + - "@ramprasadn" diff --git a/modules/nf-core/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf index 84c167cd..2b7753fd 100644 --- a/modules/nf-core/samtools/sort/main.nf +++ b/modules/nf-core/samtools/sort/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_SORT { tag "$meta.id" label 'process_medium' - conda "bioconda::samtools=1.16.1" + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.16.1--h6899075_1' : - 'quay.io/biocontainers/samtools:1.16.1--h6899075_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(bam) @@ -23,7 +23,13 @@ process SAMTOOLS_SORT { def prefix = task.ext.prefix ?: "${meta.id}" if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" """ - samtools sort $args -@ $task.cpus -o ${prefix}.bam -T $prefix $bam + samtools sort \\ + $args \\ + -@ $task.cpus \\ + -o ${prefix}.bam \\ + -T $prefix \\ + $bam + cat <<-END_VERSIONS > versions.yml "${task.process}": samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') diff --git a/modules/nf-core/samtools/sort/meta.yml b/modules/nf-core/samtools/sort/meta.yml index 09289751..07328431 100644 --- a/modules/nf-core/samtools/sort/meta.yml +++ b/modules/nf-core/samtools/sort/meta.yml @@ -12,7 +12,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/stats/main.nf b/modules/nf-core/samtools/stats/main.nf index 0a2a3640..4a2607de 100644 --- a/modules/nf-core/samtools/stats/main.nf +++ b/modules/nf-core/samtools/stats/main.nf @@ -2,14 +2,14 @@ process SAMTOOLS_STATS { tag "$meta.id" label 'process_single' - conda "bioconda::samtools=1.16.1" + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.16.1--h6899075_1' : - 'quay.io/biocontainers/samtools:1.16.1--h6899075_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(input), path(input_index) - path fasta + tuple val(meta2), path(fasta) output: tuple val(meta), path("*.stats"), emit: stats diff --git a/modules/nf-core/samtools/stats/meta.yml b/modules/nf-core/samtools/stats/meta.yml index cac50b1c..90e6345f 100644 --- a/modules/nf-core/samtools/stats/meta.yml +++ b/modules/nf-core/samtools/stats/meta.yml @@ -13,7 +13,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: @@ -23,16 +23,21 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - input: - type: file - description: BAM/CRAM file from alignment - pattern: "*.{bam,cram}" + type: file + description: BAM/CRAM file from alignment + pattern: "*.{bam,cram}" - input_index: - type: file - description: BAI/CRAI file from alignment - pattern: "*.{bai,crai}" + type: file + description: BAI/CRAI file from alignment + pattern: "*.{bai,crai}" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] - fasta: - type: optional file - description: Reference file the CRAM was created with + type: file + description: Reference file the CRAM was created with (optional) pattern: "*.{fasta,fa}" output: - meta: @@ -51,3 +56,4 @@ output: authors: - "@drpatelh" - "@FriederikeHanssen" + - "@ramprasadn" diff --git a/modules/nf-core/snpeff/snpeff/main.nf b/modules/nf-core/snpeff/snpeff/main.nf index ad5bd4f4..e92c1597 100644 --- a/modules/nf-core/snpeff/snpeff/main.nf +++ b/modules/nf-core/snpeff/snpeff/main.nf @@ -5,12 +5,12 @@ process SNPEFF_SNPEFF { conda "bioconda::snpeff=5.1" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/snpeff:5.1--hdfd78af_2' : - 'quay.io/biocontainers/snpeff:5.1--hdfd78af_2' }" + 'biocontainers/snpeff:5.1--hdfd78af_2' }" input: tuple val(meta), path(vcf) val db - path cache + tuple val(meta2), path(cache) output: tuple val(meta), path("*.ann.vcf"), emit: vcf @@ -24,17 +24,17 @@ process SNPEFF_SNPEFF { script: def args = task.ext.args ?: '' - def avail_mem = 6 + def avail_mem = 6144 if (!task.memory) { log.info '[snpEff] Available memory not known - defaulting to 6GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } def prefix = task.ext.prefix ?: "${meta.id}" def cache_command = cache ? "-dataDir \${PWD}/${cache}" : "" """ snpEff \\ - -Xmx${avail_mem}g \\ + -Xmx${avail_mem}M \\ $db \\ $args \\ -csvStats ${prefix}.csv \\ diff --git a/modules/nf-core/snpeff/snpeff/meta.yml b/modules/nf-core/snpeff/snpeff/meta.yml index 28973c78..44bada23 100644 --- a/modules/nf-core/snpeff/snpeff/meta.yml +++ b/modules/nf-core/snpeff/snpeff/meta.yml @@ -2,6 +2,10 @@ name: SNPEFF_SNPEFF description: Genetic variant annotation and functional effect prediction toolbox keywords: - annotation + - effect prediction + - snpeff + - variant + - vcf tools: - snpeff: description: | @@ -21,7 +25,7 @@ input: description: | vcf to annotate - db: - type: value + type: string description: | which db to annotate with - cache: diff --git a/modules/nf-core/star/align/main.nf b/modules/nf-core/star/align/main.nf index 0e3bd713..d0e20384 100644 --- a/modules/nf-core/star/align/main.nf +++ b/modules/nf-core/star/align/main.nf @@ -5,30 +5,34 @@ process STAR_ALIGN { conda "bioconda::star=2.7.10a bioconda::samtools=1.16.1 conda-forge::gawk=5.1.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' : - 'quay.io/biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' }" + 'biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' }" input: - tuple val(meta), path(reads) - path index - path gtf + tuple val(meta), path(reads, stageAs: "input*/*") + tuple val(meta2), path(index) + tuple val(meta3), path(gtf) val star_ignore_sjdbgtf val seq_platform val seq_center output: - tuple val(meta), path('*d.out.bam') , emit: bam tuple val(meta), path('*Log.final.out') , emit: log_final tuple val(meta), path('*Log.out') , emit: log_out tuple val(meta), path('*Log.progress.out'), emit: log_progress path "versions.yml" , emit: versions + tuple val(meta), path('*d.out.bam') , optional:true, emit: bam tuple val(meta), path('*sortedByCoord.out.bam') , optional:true, emit: bam_sorted tuple val(meta), path('*toTranscriptome.out.bam'), optional:true, emit: bam_transcript tuple val(meta), path('*Aligned.unsort.out.bam') , optional:true, emit: bam_unsorted tuple val(meta), path('*fastq.gz') , optional:true, emit: fastq tuple val(meta), path('*.tab') , optional:true, emit: tab + tuple val(meta), path('*.SJ.out.tab') , optional:true, emit: spl_junc_tab + tuple val(meta), path('*.ReadsPerGene.out.tab') , optional:true, emit: read_per_gene_tab tuple val(meta), path('*.out.junction') , optional:true, emit: junction tuple val(meta), path('*.out.sam') , optional:true, emit: sam + tuple val(meta), path('*.wig') , optional:true, emit: wig + tuple val(meta), path('*.bg') , optional:true, emit: bedgraph when: task.ext.when == null || task.ext.when @@ -36,20 +40,23 @@ process STAR_ALIGN { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def reads1 = [], reads2 = [] + meta.single_end ? [reads].flatten().each{reads1 << it} : reads.eachWithIndex{ v, ix -> ( ix & 1 ? reads2 : reads1) << v } def ignore_gtf = star_ignore_sjdbgtf ? '' : "--sjdbGTFfile $gtf" def seq_platform = seq_platform ? "'PL:$seq_platform'" : "" - def seq_center = seq_center ? "--outSAMattrRGline ID:$prefix 'CN:$seq_center' 'SM:$prefix' $seq_platform " : "--outSAMattrRGline ID:$prefix 'SM:$prefix' $seq_platform " + def seq_center = seq_center ? "'CN:$seq_center'" : "" + def attrRG = args.contains("--outSAMattrRGline") ? "" : "--outSAMattrRGline 'ID:$prefix' $seq_center 'SM:$prefix' $seq_platform" def out_sam_type = (args.contains('--outSAMtype')) ? '' : '--outSAMtype BAM Unsorted' def mv_unsorted_bam = (args.contains('--outSAMtype BAM Unsorted SortedByCoordinate')) ? "mv ${prefix}.Aligned.out.bam ${prefix}.Aligned.unsort.out.bam" : '' """ STAR \\ --genomeDir $index \\ - --readFilesIn $reads \\ + --readFilesIn ${reads1.join(",")} ${reads2.join(",")} \\ --runThreadN $task.cpus \\ --outFileNamePrefix $prefix. \\ $out_sam_type \\ $ignore_gtf \\ - $seq_center \\ + $attrRG \\ $args $mv_unsorted_bam @@ -81,11 +88,16 @@ process STAR_ALIGN { touch ${prefix}.sortedByCoord.out.bam touch ${prefix}.toTranscriptome.out.bam touch ${prefix}.Aligned.unsort.out.bam + touch ${prefix}.Aligned.sortedByCoord.out.bam touch ${prefix}.unmapped_1.fastq.gz touch ${prefix}.unmapped_2.fastq.gz touch ${prefix}.tab + touch ${prefix}.SJ.out.tab + touch ${prefix}.ReadsPerGene.out.tab touch ${prefix}.Chimeric.out.junction touch ${prefix}.out.sam + touch ${prefix}.Signal.UniqueMultiple.str1.out.wig + touch ${prefix}.Signal.UniqueMultiple.str1.out.bg cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/star/align/meta.yml b/modules/nf-core/star/align/meta.yml index 7ee10f1c..3d8fed0c 100644 --- a/modules/nf-core/star/align/meta.yml +++ b/modules/nf-core/star/align/meta.yml @@ -25,10 +25,34 @@ input: description: | List of input FastQ files of size 1 and 2 for single-end and paired-end data, respectively. + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'test' ] - index: type: directory description: STAR genome index pattern: "star" + - meta3: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'test' ] + - gtf: + type: file + description: Annotation GTF file + pattern: "*.{gtf}" + - star_ignore_sjdbgtf: + type: boolean + description: Ignore annotation GTF file + - seq_platform: + type: string + description: Sequencing platform + - seq_center: + type: string + description: Sequencing center + output: - bam: type: file @@ -74,6 +98,14 @@ output: type: file description: STAR chimeric junction output file (optional) pattern: "*.out.junction" + - wig: + type: file + description: STAR output wiggle format file(s) (optional) + pattern: "*.wig" + - bedgraph: + type: file + description: STAR output bedGraph format file(s) (optional) + pattern: "*.bg" authors: - "@kevinmenden" diff --git a/modules/nf-core/star/genomegenerate/main.nf b/modules/nf-core/star/genomegenerate/main.nf index 91462489..43424042 100644 --- a/modules/nf-core/star/genomegenerate/main.nf +++ b/modules/nf-core/star/genomegenerate/main.nf @@ -5,15 +5,15 @@ process STAR_GENOMEGENERATE { conda "bioconda::star=2.7.10a bioconda::samtools=1.16.1 conda-forge::gawk=5.1.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' : - 'quay.io/biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' }" + 'biocontainers/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:1df389393721fc66f3fd8778ad938ac711951107-0' }" input: - path fasta - path gtf + tuple val(meta), path(fasta) + tuple val(meta2), path(gtf) output: - path "star" , emit: index - path "versions.yml", emit: versions + tuple val(meta), path("star") , emit: index + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/star/genomegenerate/meta.yml b/modules/nf-core/star/genomegenerate/meta.yml index 8181157a..eba2d9cf 100644 --- a/modules/nf-core/star/genomegenerate/meta.yml +++ b/modules/nf-core/star/genomegenerate/meta.yml @@ -15,14 +15,29 @@ tools: doi: 10.1093/bioinformatics/bts635 licence: ["MIT"] input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - fasta: type: file description: Fasta file of the reference genome + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'test' ] - gtf: type: file description: GTF file of the reference genome output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - index: type: directory description: Folder containing the star index files diff --git a/modules/nf-core/tabix/bgziptabix/main.nf b/modules/nf-core/tabix/bgziptabix/main.nf index d3a3bbff..d6c5a760 100644 --- a/modules/nf-core/tabix/bgziptabix/main.nf +++ b/modules/nf-core/tabix/bgziptabix/main.nf @@ -5,13 +5,14 @@ process TABIX_BGZIPTABIX { conda "bioconda::tabix=1.11" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0' : - 'quay.io/biocontainers/tabix:1.11--hdfd78af_0' }" + 'biocontainers/tabix:1.11--hdfd78af_0' }" input: tuple val(meta), path(input) output: - tuple val(meta), path("*.gz"), path("*.tbi"), emit: gz_tbi + tuple val(meta), path("*.gz"), path("*.tbi"), optional: true, emit: gz_tbi + tuple val(meta), path("*.gz"), path("*.csi"), optional: true, emit: gz_csi path "versions.yml" , emit: versions when: @@ -34,8 +35,9 @@ process TABIX_BGZIPTABIX { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.gz - touch ${prefix}.gz.tbi + touch ${prefix}.${input.getExtension()}.gz + touch ${prefix}.${input.getExtension()}.gz.tbi + touch ${prefix}.${input.getExtension()}.gz.csi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/bgziptabix/meta.yml b/modules/nf-core/tabix/bgziptabix/meta.yml index 49c03289..2761e271 100644 --- a/modules/nf-core/tabix/bgziptabix/meta.yml +++ b/modules/nf-core/tabix/bgziptabix/meta.yml @@ -37,9 +37,14 @@ output: type: file description: tabix index file pattern: "*.{gz.tbi}" + - csi: + type: file + description: tabix alternate index file + pattern: "*.{gz.csi}" - versions: type: file description: File containing software versions pattern: "versions.yml" authors: - "@maxulysse" + - "@DLBPointon" diff --git a/modules/nf-core/tabix/tabix/main.nf b/modules/nf-core/tabix/tabix/main.nf index 9a404db9..5bf332ef 100644 --- a/modules/nf-core/tabix/tabix/main.nf +++ b/modules/nf-core/tabix/tabix/main.nf @@ -5,7 +5,7 @@ process TABIX_TABIX { conda "bioconda::tabix=1.11" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0' : - 'quay.io/biocontainers/tabix:1.11--hdfd78af_0' }" + 'biocontainers/tabix:1.11--hdfd78af_0' }" input: tuple val(meta), path(tab) diff --git a/modules/nf-core/untar/main.nf b/modules/nf-core/untar/main.nf index 3384847a..61461c39 100644 --- a/modules/nf-core/untar/main.nf +++ b/modules/nf-core/untar/main.nf @@ -2,10 +2,10 @@ process UNTAR { tag "$archive" label 'process_single' - conda "conda-forge::sed=4.7 bioconda::grep=3.4 conda-forge::tar=1.34" + conda "conda-forge::sed=4.7 conda-forge::grep=3.11 conda-forge::tar=1.34" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'ubuntu:20.04' }" + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(archive) diff --git a/modules/nf-core/untar/meta.yml b/modules/nf-core/untar/meta.yml index ea7a3f38..db241a6e 100644 --- a/modules/nf-core/untar/meta.yml +++ b/modules/nf-core/untar/meta.yml @@ -3,6 +3,7 @@ description: Extract files. keywords: - untar - uncompress + - extract tools: - untar: description: | From 4976312eba55df37f941fe7e680dd14b2c75f472 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 12:34:27 +0200 Subject: [PATCH 10/34] prettier --- modules.json | 138 +++++++++++++-------------------------------------- 1 file changed, 35 insertions(+), 103 deletions(-) diff --git a/modules.json b/modules.json index 3e8235c2..1e399602 100755 --- a/modules.json +++ b/modules.json @@ -8,240 +8,172 @@ "bedtools/merge": { "branch": "master", "git_sha": "9e51255c4f8ec69fb6ccf68593392835f14fecb8", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "bedtools/sort": { "branch": "master", "git_sha": "9e51255c4f8ec69fb6ccf68593392835f14fecb8", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "cat/fastq": { "branch": "master", "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "custom/dumpsoftwareversions": { "branch": "master", "git_sha": "05c280924b6c768d484c7c443dad5e605c4ff4b4", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "ensemblvep/vep": { "branch": "master", "git_sha": "9f9e1fc31cb35876922070c0e601ae05abae5cae", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "fastqc": { "branch": "master", "git_sha": "9a4517e720bc812e95b56d23d15a1653b6db4f53", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/applybqsr": { "branch": "master", "git_sha": "240937a2a9c30298110753292be041188891f2cb", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/baserecalibrator": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/bedtointervallist": { "branch": "master", "git_sha": "2df2a11d5b12f2a73bca74f103691bc35d83c5fd", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/combinegvcfs": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/createsequencedictionary": { "branch": "master", "git_sha": "541811d779026c5d395925895fa5ed35e7216cc0", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/haplotypecaller": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/indexfeaturefile": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/intervallisttools": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/markduplicates": { "branch": "master", "git_sha": "0a261469640941da2488e1a5aa023b64db837c70", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/mergevcfs": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/splitncigarreads": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gatk4/variantfiltration": { "branch": "master", "git_sha": "2df2a11d5b12f2a73bca74f103691bc35d83c5fd", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gffread": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gunzip": { "branch": "master", "git_sha": "e06548bfa36ee31869b81041879dd6b3a83b1d57", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "multiqc": { "branch": "master", "git_sha": "a6e11ac655e744f7ebc724be669dd568ffdc0e80", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/faidx": { "branch": "master", "git_sha": "fd742419940e01ba1c5ecb172c3e32ec840662fe", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/flagstat": { "branch": "master", "git_sha": "570ec5bcfe19c49e16c9ca35a7a116563af6cc1c", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/idxstats": { "branch": "master", "git_sha": "e662ab16e0c11f1e62983e21de9871f59371a639", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/index": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/merge": { "branch": "master", "git_sha": "0460d316170f75f323111b4a2c0a2989f0c32013", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/sort": { "branch": "master", "git_sha": "a0f7be95788366c1923171e358da7d049eb440f9", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/stats": { "branch": "master", "git_sha": "735e1e04e7e01751d2d6e97055bbdb6f70683cc1", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "snpeff/snpeff": { "branch": "master", "git_sha": "4d584d5cf6ed5f7363a51cdb4b3eb25398e9e537", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "star/align": { "branch": "master", "git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "star/genomegenerate": { "branch": "master", "git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "tabix/bgziptabix": { "branch": "master", "git_sha": "591b71642820933dcb3c954c934b397bd00d8e5e", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "tabix/tabix": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "untar": { "branch": "master", "git_sha": "d0b4fc03af52a1cc8c6fb4493b921b57352b1dd8", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] } } }, @@ -250,4 +182,4 @@ } } } -} \ No newline at end of file +} From 0a001cd3180cb666e8c492996506d8e9df70234f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 12:36:28 +0200 Subject: [PATCH 11/34] fix lint --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 2a63dfa9..747ad37f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) @praveenraj2018 +Copyright (c) Praveen Raj, Maxime U Garcia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From a2856bd39b35374ea108de8b800f178f897bf831 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 14:23:47 +0200 Subject: [PATCH 12/34] fix path to file --- conf/test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/test.config b/conf/test.config index dd25aa41..68f97def 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,7 +20,7 @@ params { max_time = '6.h' // Input data - input = "https://raw.githubusercontent.com/nf-core/test-datasets/rnavar/samplesheet/v1.0/samplesheet.csv" + input = "${projectDir}/tests/csv/1.0/fastq_single.csv" // Genome references genome = 'WBcel235' From c205c24ae64f0c4de72138fcadd089bf7ec86a9e Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 14:55:48 +0200 Subject: [PATCH 13/34] add file --- tests/csv/1.0/fastq_single.csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/csv/1.0/fastq_single.csv diff --git a/tests/csv/1.0/fastq_single.csv b/tests/csv/1.0/fastq_single.csv new file mode 100644 index 00000000..125f9bfd --- /dev/null +++ b/tests/csv/1.0/fastq_single.csv @@ -0,0 +1,2 @@ +sample,fastq_1,fastq_2,strandedness +GM12878,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/homo_sapiens/illumina/fastq/test_rnaseq_1.fastq.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/homo_sapiens/illumina/fastq/test_rnaseq_2.fastq.gz,reverse From f7ae880151ac58cee924510c4835138e717c4b8f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 14:56:07 +0200 Subject: [PATCH 14/34] typo + code polish --- workflows/rnavar.nf | 144 +++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 82 deletions(-) diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index f504b785..8a558bb1 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -32,11 +32,11 @@ def checkPathParamList = [ params.star_index, ] -for (param in checkPathParamList) {if (param) file(param, checkIfExists: true)} +for(param in checkPathParamList) {if (param) file(param, checkIfExists: true)} // Check mandatory parameters if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } -if(!params.star_index && !params.gtf && !params.gff){ exit 1, "GTF|GFF3 file is required to build a STAR reference index! Use option --gtf|--gff to provide a GTF|GFF file." } +if (!params.star_index && !params.gtf && !params.gff){ exit 1, "GTF|GFF3 file is required to build a STAR reference index! Use option --gtf|--gff to provide a GTF|GFF file." } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -55,9 +55,9 @@ ch_multiqc_custom_methods_description = params.multiqc_methods_description ? fil ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { INPUT_CHECK } from '../subworkflows/local/input_check' // Validate the input samplesheet.csv and prepare input channels -include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome' // Build the genome index and other reference files -include { ANNOTATE } from '../subworkflows/local/annotate' // Annotate variants using snpEff or VEP or both +include { INPUT_CHECK } from '../subworkflows/local/input_check' // Validate the input samplesheet.csv and prepare input channels +include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome' // Build the genome index and other reference files +include { ANNOTATE } from '../subworkflows/local/annotate' // Annotate variants using snpEff or VEP or both /* ======================================================================================== @@ -88,7 +88,7 @@ include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/ ======================================================================================== */ -include { GTF2BED } from '../modules/local/gtf2bed' +include { GTF2BED } from '../modules/local/gtf2bed' /* ======================================================================================== @@ -96,10 +96,10 @@ include { GTF2BED } from '../modules/local/gtf2bed' ======================================================================================== */ -include { ALIGN_STAR } from '../subworkflows/nf-core/align_star' // Align reads to genome and sort and index the alignment file -include { MARKDUPLICATES } from '../subworkflows/nf-core/markduplicates' // Mark duplicates in the BAM file -include { SPLITNCIGAR } from '../subworkflows/nf-core/splitncigar' // Splits reads that contain Ns in their cigar string -include { RECALIBRATE } from '../subworkflows/nf-core/recalibrate' // Estimate and correct systematic bias +include { ALIGN_STAR } from '../subworkflows/nf-core/align_star' // Align reads to genome and sort and index the alignment file +include { MARKDUPLICATES } from '../subworkflows/nf-core/markduplicates' // Mark duplicates in the BAM file +include { SPLITNCIGAR } from '../subworkflows/nf-core/splitncigar' // Splits reads that contain Ns in their cigar string +include { RECALIBRATE } from '../subworkflows/nf-core/recalibrate' // Estimate and correct systematic bias /* ======================================================================================== @@ -109,8 +109,8 @@ include { RECALIBRATE } from '../subworkflows/nf-core/recalibr // Check STAR alignment parameters def prepareToolIndices = params.aligner -def seq_platform = params.seq_platform ? params.seq_platform : [] -def seq_center = params.seq_center ? params.seq_center : [] +def seq_platform = params.seq_platform ?: [] +def seq_center = params.seq_center ?: [] // Initialize file channels based on params ch_dbsnp = params.dbsnp ? Channel.fromPath(params.dbsnp).collect() : Channel.empty() @@ -147,7 +147,7 @@ workflow RNAVAR { // SUBWORKFLOW: Uncompress and prepare reference genome files // - PREPARE_GENOME ( + PREPARE_GENOME( prepareToolIndices, params.feature_type ) @@ -157,25 +157,24 @@ workflow RNAVAR { // // SUBWORKFLOW: Read in samplesheet, validate and stage input files // - INPUT_CHECK ( - file(params.input) - ) - .reads - .map { - meta, fastq -> - def meta_clone = meta.clone() - meta_clone.id = meta_clone.id.split('_')[0..-2].join('_') - [ meta_clone, fastq ] - } - .groupTuple(by: [0]) - .branch { - meta, fastq -> - single : fastq.size() == 1 - return [ meta, fastq.flatten() ] - multiple: fastq.size() > 1 - return [ meta, fastq.flatten() ] - } - .set { ch_fastq } + INPUT_CHECK(file(params.input)) + .reads + .map { + meta, fastq -> + def meta_clone = meta.clone() + meta_clone.id = meta_clone.id.split('_')[0..-2].join('_') + [ meta_clone, fastq ] + } + .groupTuple(by: [0]) + .branch { + meta, fastq -> + single : fastq.size() == 1 + return [ meta, fastq.flatten() ] + multiple: fastq.size() > 1 + return [ meta, fastq.flatten() ] + } + .set { ch_fastq } + ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ @@ -184,20 +183,17 @@ workflow RNAVAR { // // MODULE: Concatenate FastQ files from same sample if required // - CAT_FASTQ ( - ch_fastq.multiple - ) - .reads - .mix(ch_fastq.single) - .set { ch_cat_fastq } + CAT_FASTQ(ch_fastq.multiple) + .reads + .mix(ch_fastq.single) + .set { ch_cat_fastq } + ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) // // MODULE: Generate QC summary using FastQC // - FASTQC ( - ch_cat_fastq - ) + FASTQC(ch_cat_fastq) ch_reports = ch_reports.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) ch_versions = ch_versions.mix(FASTQC.out.versions.first()) @@ -205,10 +201,7 @@ workflow RNAVAR { // MODULE: Prepare the interval list from the GTF file using GATK4 BedToIntervalList // ch_interval_list = Channel.empty() - GATK4_BEDTOINTERVALLIST( - ch_genome_bed, - PREPARE_GENOME.out.dict - ) + GATK4_BEDTOINTERVALLIST(ch_genome_bed, PREPARE_GENOME.out.dict) ch_interval_list = GATK4_BEDTOINTERVALLIST.out.interval_list ch_versions = ch_versions.mix(GATK4_BEDTOINTERVALLIST.out.versions.first().ifEmpty(null)) @@ -217,9 +210,7 @@ workflow RNAVAR { // ch_interval_list_split = Channel.empty() if (!params.skip_intervallisttools) { - GATK4_INTERVALLISTTOOLS( - ch_interval_list - ) + GATK4_INTERVALLISTTOOLS(ch_interval_list) ch_interval_list_split = GATK4_INTERVALLISTTOOLS.out.interval_list.map{ meta, bed -> [bed] }.flatten() } else ch_interval_list_split = ch_interval_list @@ -237,7 +228,7 @@ workflow RNAVAR { ch_aligner_clustering_multiqc = Channel.empty() if (params.aligner == 'star') { - ALIGN_STAR ( + ALIGN_STAR( ch_cat_fastq, PREPARE_GENOME.out.star_index, PREPARE_GENOME.out.gtf, @@ -257,9 +248,7 @@ workflow RNAVAR { // // SUBWORKFLOW: Mark duplicates with GATK4 // - MARKDUPLICATES ( - ch_genome_bam - ) + MARKDUPLICATES(ch_genome_bam) ch_genome_bam = MARKDUPLICATES.out.bam_bai //Gather QC reports @@ -269,10 +258,10 @@ workflow RNAVAR { // // SUBWORKFLOW: SplitNCigarReads from GATK4 over the intervals - // Splits reads that contain Ns in their cigar string (e.g. spanning splicing events in RNAseq data). + // Splits reads that contain Ns in their cigar string(e.g. spanning splicing events in RNAseq data). // ch_splitncigar_bam_bai = Channel.empty() - SPLITNCIGAR ( + SPLITNCIGAR( ch_genome_bam, PREPARE_GENOME.out.fasta, PREPARE_GENOME.out.fai, @@ -287,7 +276,7 @@ workflow RNAVAR { // Generates a recalibration table based on various co-variates // ch_bam_variant_calling = Channel.empty() - if(!params.skip_baserecalibration) { + if (!params.skip_baserecalibration) { ch_bqsr_table = Channel.empty() // known_sites is made by grouping both the dbsnp and the known indels ressources // they can either or both be optional @@ -318,8 +307,8 @@ workflow RNAVAR { ch_interval_list_applybqsr = ch_interval_list.map{ meta, bed -> [bed] }.flatten() ch_bam_applybqsr.combine(ch_interval_list_applybqsr) - .map{ meta, bam, bai, table, interval -> [ meta, bam, bai, table, interval] - }.set{ch_applybqsr_bam_bai_interval} + .map{ meta, bam, bai, table, interval -> [ meta, bam, bai, table, interval]} + .set{ch_applybqsr_bam_bai_interval} // // MODULE: ApplyBaseRecalibrator from GATK4 @@ -375,10 +364,10 @@ workflow RNAVAR { ch_haplotypecaller_raw = GATK4_HAPLOTYPECALLER.out.vcf - .map{ meta, vcf -> - meta.id = meta.sample - [meta, vcf]} - .groupTuple() + .map{ meta, vcf -> + meta.id = meta.sample + [meta, vcf]} + .groupTuple() ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLER.out.versions.first().ifEmpty(null)) @@ -386,10 +375,7 @@ workflow RNAVAR { // MODULE: MergeVCFS from GATK4 // Merge multiple VCF files into one VCF // - GATK4_MERGEVCFS( - ch_haplotypecaller_raw, - PREPARE_GENOME.out.dict - ) + GATK4_MERGEVCFS(ch_haplotypecaller_raw, PREPARE_GENOME.out.dict) ch_haplotypecaller_vcf = GATK4_MERGEVCFS.out.vcf ch_versions = ch_versions.mix(GATK4_MERGEVCFS.out.versions.first().ifEmpty(null)) @@ -414,15 +400,13 @@ workflow RNAVAR { // MODULE: IndexFeatureFile from GATK4 // Index the gVCF files // - GATK4_INDEXFEATUREFILE( - GATK4_HAPLOTYPECALLERGVCF.out.vcf - ) + GATK4_INDEXFEATUREFILE(GATK4_HAPLOTYPECALLERGVCF.out.vcf) ch_haplotypecallergvcf_raw_index = GATK4_INDEXFEATUREFILE.out.index - .map{ meta, idx -> - meta.id = meta.sample - [meta, idx]} - .groupTuple() + .map{ meta, idx -> + meta.id = meta.sample + [meta, idx]} + .groupTuple() ch_versions = ch_versions.mix(GATK4_INDEXFEATUREFILE.out.versions.first().ifEmpty(null)) @@ -452,9 +436,7 @@ workflow RNAVAR { // // MODULE: Index the VCF using TABIX // - TABIXGVCF( - ch_haplotypecaller_gvcf - ) + TABIXGVCF(ch_haplotypecaller_gvcf) ch_haplotypecaller_gvcf_tbi = ch_haplotypecaller_gvcf .join(TABIXGVCF.out.tbi, by: [0], remainder: true) @@ -471,9 +453,7 @@ workflow RNAVAR { // // MODULE: Index the VCF using TABIX // - TABIX( - ch_haplotypecaller_vcf - ) + TABIX(ch_haplotypecaller_vcf) ch_haplotypecaller_vcf_tbi = ch_haplotypecaller_vcf .join(TABIX.out.tbi, by: [0], remainder: true) @@ -507,7 +487,7 @@ workflow RNAVAR { // // SUBWORKFLOW: Annotate variants using snpEff and Ensembl VEP if enabled. // - if((!params.skip_variantannotation) && (params.annotate_tools) && (params.annotate_tools.contains('merge') || params.annotate_tools.contains('snpeff') || params.annotate_tools.contains('vep'))) { + if ((!params.skip_variantannotation) &&(params.annotate_tools) &&(params.annotate_tools.contains('merge') || params.annotate_tools.contains('snpeff') || params.annotate_tools.contains('vep'))) { ANNOTATE( ch_final_vcf, params.annotate_tools, @@ -526,7 +506,7 @@ workflow RNAVAR { } ch_version_yaml = Channel.empty() - CUSTOM_DUMPSOFTWAREVERSIONS (ch_versions.unique().collectFile(name: 'collated_versions.yml')) + CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile(name: 'collated_versions.yml')) ch_version_yaml = CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect() // @@ -549,9 +529,9 @@ workflow RNAVAR { ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_reports.collect()) ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_config) - ch_multiqc_files = ch_multiqc_files.mix(ch_rnavar_logo)) + ch_multiqc_files = ch_multiqc_files.mix(ch_rnavar_logo) - MULTIQC ( + MULTIQC( ch_multiqc_files.collect(), ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), From 64a3dcd4012944759c1468f74fff95c0d2211816 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 14 Sep 2023 15:04:27 +0200 Subject: [PATCH 15/34] fix path to modules --- subworkflows/local/prepare_genome.nf | 32 ++++++++++----------- subworkflows/nf-core/align_star.nf | 2 +- subworkflows/nf-core/bam_sort_samtools.nf | 4 +-- subworkflows/nf-core/bam_stats_samtools.nf | 6 ++-- subworkflows/nf-core/ensemblvep_annotate.nf | 4 +-- subworkflows/nf-core/markduplicates.nf | 4 +-- subworkflows/nf-core/recalibrate.nf | 6 ++-- subworkflows/nf-core/snpeff_annotate.nf | 4 +-- subworkflows/nf-core/splitncigar.nf | 6 ++-- workflows/rnavar.nf | 32 ++++++++++----------- 10 files changed, 50 insertions(+), 50 deletions(-) diff --git a/subworkflows/local/prepare_genome.nf b/subworkflows/local/prepare_genome.nf index 74acecc8..c9e1e417 100755 --- a/subworkflows/local/prepare_genome.nf +++ b/subworkflows/local/prepare_genome.nf @@ -2,18 +2,18 @@ // Uncompress and prepare reference genome files // -include { GATK4_CREATESEQUENCEDICTIONARY } from '../../modules/nf-core/modules/gatk4/createsequencedictionary/main' //addParams(options: params.genome_options) -include { GFFREAD } from '../../modules/nf-core/modules/gffread/main' //addParams(options: params.gffread_options) -include { GTF2BED } from '../../modules/local/gtf2bed' -include { BEDTOOLS_SORT } from '../../modules/nf-core/modules/bedtools/sort/main' -include { BEDTOOLS_MERGE } from '../../modules/nf-core/modules/bedtools/merge/main' -include { GUNZIP as GUNZIP_FASTA } from '../../modules/nf-core/modules/gunzip/main' //addParams(options: params.genome_options) -include { GUNZIP as GUNZIP_GENE_BED } from '../../modules/nf-core/modules/gunzip/main' //addParams(options: params.genome_options) -include { GUNZIP as GUNZIP_GFF } from '../../modules/nf-core/modules/gunzip/main' //addParams(options: params.genome_options) -include { GUNZIP as GUNZIP_GTF } from '../../modules/nf-core/modules/gunzip/main' //addParams(options: params.genome_options) -include { SAMTOOLS_FAIDX } from '../../modules/nf-core/modules/samtools/faidx/main' //addParams(options: params.genome_options) -include { STAR_GENOMEGENERATE } from '../../modules/nf-core/modules/star/genomegenerate/main' //addParams(options: params.star_index_options) -include { UNTAR as UNTAR_STAR_INDEX } from '../../modules/nf-core/modules/untar/main' //addParams(options: params.star_untar_options) +include { GATK4_CREATESEQUENCEDICTIONARY } from '../../modules/nf-core/gatk4/createsequencedictionary/main' //addParams(options: params.genome_options) +include { GFFREAD } from '../../modules/nf-core/gffread/main' //addParams(options: params.gffread_options) +include { GTF2BED } from '../../modules/local/gtf2bed' +include { BEDTOOLS_SORT } from '../../modules/nf-core/bedtools/sort/main' +include { BEDTOOLS_MERGE } from '../../modules/nf-core/bedtools/merge/main' +include { GUNZIP as GUNZIP_FASTA } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) +include { GUNZIP as GUNZIP_GENE_BED } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) +include { GUNZIP as GUNZIP_GFF } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) +include { GUNZIP as GUNZIP_GTF } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) +include { SAMTOOLS_FAIDX } from '../../modules/nf-core/samtools/faidx/main' //addParams(options: params.genome_options) +include { STAR_GENOMEGENERATE } from '../../modules/nf-core/star/genomegenerate/main' //addParams(options: params.star_index_options) +include { UNTAR as UNTAR_STAR_INDEX } from '../../modules/nf-core/untar/main' //addParams(options: params.star_untar_options) workflow PREPARE_GENOME { @@ -89,19 +89,19 @@ workflow PREPARE_GENOME { ch_exon_bed = GTF2BED ( ch_gtf , feature_type).bed.collect() ch_versions = ch_versions.mix(GTF2BED.out.versions) } - + //ch_exon_bed.view() //ch_exon_bed.map{ it -> [[id:'exome'], it] } //ch_exon_bed.view() // Bedtools sort ch_bedtools_sort = BEDTOOLS_SORT(ch_exon_bed.map{ it -> [[id:'exome'], it] }, 'sorted').sorted.collect() ch_versions = ch_versions.mix(BEDTOOLS_SORT.out.versions) - - + + // Bedtools merge ch_bedtools_merge = BEDTOOLS_MERGE(ch_bedtools_sort).bed ch_versions = ch_versions.mix(BEDTOOLS_MERGE.out.versions) - + // Index the genome fasta ch_fasta_fai = Channel.empty() diff --git a/subworkflows/nf-core/align_star.nf b/subworkflows/nf-core/align_star.nf index 2a4069fd..5f165e0d 100644 --- a/subworkflows/nf-core/align_star.nf +++ b/subworkflows/nf-core/align_star.nf @@ -2,7 +2,7 @@ // Alignment with STAR // -include { STAR_ALIGN } from '../../modules/nf-core/modules/star/align/main' +include { STAR_ALIGN } from '../../modules/nf-core/star/align/main' include { BAM_SORT_SAMTOOLS } from './bam_sort_samtools' workflow ALIGN_STAR { diff --git a/subworkflows/nf-core/bam_sort_samtools.nf b/subworkflows/nf-core/bam_sort_samtools.nf index f473b759..2e84fa95 100644 --- a/subworkflows/nf-core/bam_sort_samtools.nf +++ b/subworkflows/nf-core/bam_sort_samtools.nf @@ -2,8 +2,8 @@ // Sort, index BAM file and run samtools stats, flagstat and idxstats // -include { SAMTOOLS_SORT } from '../../modules/nf-core/modules/samtools/sort/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/modules/samtools/index/main' +include { SAMTOOLS_SORT } from '../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' include { BAM_STATS_SAMTOOLS } from './bam_stats_samtools' workflow BAM_SORT_SAMTOOLS { diff --git a/subworkflows/nf-core/bam_stats_samtools.nf b/subworkflows/nf-core/bam_stats_samtools.nf index 68d632c3..c45a91ae 100644 --- a/subworkflows/nf-core/bam_stats_samtools.nf +++ b/subworkflows/nf-core/bam_stats_samtools.nf @@ -2,9 +2,9 @@ // Run SAMtools stats, flagstat and idxstats // -include { SAMTOOLS_STATS } from '../../modules/nf-core/modules/samtools/stats/main' -include { SAMTOOLS_IDXSTATS } from '../../modules/nf-core/modules/samtools/idxstats/main' -include { SAMTOOLS_FLAGSTAT } from '../../modules/nf-core/modules/samtools/flagstat/main' +include { SAMTOOLS_STATS } from '../../modules/nf-core/samtools/stats/main' +include { SAMTOOLS_IDXSTATS } from '../../modules/nf-core/samtools/idxstats/main' +include { SAMTOOLS_FLAGSTAT } from '../../modules/nf-core/samtools/flagstat/main' workflow BAM_STATS_SAMTOOLS { take: diff --git a/subworkflows/nf-core/ensemblvep_annotate.nf b/subworkflows/nf-core/ensemblvep_annotate.nf index 13dffee4..87cfc2b7 100644 --- a/subworkflows/nf-core/ensemblvep_annotate.nf +++ b/subworkflows/nf-core/ensemblvep_annotate.nf @@ -2,8 +2,8 @@ // Run VEP to annotate VCF files // -include { ENSEMBLVEP } from '../../modules/nf-core/modules/ensemblvep/main' -include { TABIX_BGZIPTABIX } from '../../modules/nf-core/modules/tabix/bgziptabix/main' +include { ENSEMBLVEP_VEP } from '../../modules/nf-core/ensemblvep/vep/main' +include { TABIX_BGZIPTABIX } from '../../modules/nf-core/tabix/bgziptabix/main' workflow ENSEMBLVEP_ANNOTATE { take: diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index 85b724dd..18d3dc39 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -3,8 +3,8 @@ // include { BAM_STATS_SAMTOOLS } from './bam_stats_samtools' -include { GATK4_MARKDUPLICATES } from '../../modules/nf-core/modules/gatk4/markduplicates/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/modules/samtools/index/main' +include { GATK4_MARKDUPLICATES } from '../../modules/nf-core/gatk4/markduplicates/main' +include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' workflow MARKDUPLICATES { take: diff --git a/subworkflows/nf-core/recalibrate.nf b/subworkflows/nf-core/recalibrate.nf index 664cf3f5..6d79257f 100644 --- a/subworkflows/nf-core/recalibrate.nf +++ b/subworkflows/nf-core/recalibrate.nf @@ -4,9 +4,9 @@ ======================================================================================== */ -include { GATK4_APPLYBQSR as APPLYBQSR } from '../../modules/nf-core/modules/gatk4/applybqsr/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/modules/samtools/index/main' -include { SAMTOOLS_STATS } from '../../modules/nf-core/modules/samtools/stats/main' +include { GATK4_APPLYBQSR as APPLYBQSR } from '../../modules/nf-core/gatk4/applybqsr/main' +include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' +include { SAMTOOLS_STATS } from '../../modules/nf-core/samtools/stats/main' workflow RECALIBRATE { take: diff --git a/subworkflows/nf-core/snpeff_annotate.nf b/subworkflows/nf-core/snpeff_annotate.nf index 9a8b65bc..e3dd8f0f 100644 --- a/subworkflows/nf-core/snpeff_annotate.nf +++ b/subworkflows/nf-core/snpeff_annotate.nf @@ -2,8 +2,8 @@ // Run snpEff to annotate VCF files // -include { SNPEFF } from '../../modules/nf-core/modules/snpeff/main' -include { TABIX_BGZIPTABIX } from '../../modules/nf-core/modules/tabix/bgziptabix/main' +include { SNPEFF_SNPEFF } from '../../modules/nf-core/snpeff/snpeff/main' +include { TABIX_BGZIPTABIX } from '../../modules/nf-core/tabix/bgziptabix/main' workflow SNPEFF_ANNOTATE { take: diff --git a/subworkflows/nf-core/splitncigar.nf b/subworkflows/nf-core/splitncigar.nf index 1b1895f7..88c63a38 100644 --- a/subworkflows/nf-core/splitncigar.nf +++ b/subworkflows/nf-core/splitncigar.nf @@ -2,9 +2,9 @@ // Subworkflow: Run GATK4 SplitNCigarReads with intervals, merge and index BAM file. // -include { GATK4_SPLITNCIGARREADS } from '../../modules/nf-core/modules/gatk4/splitncigarreads/main' -include { SAMTOOLS_MERGE } from '../../modules/nf-core/modules/samtools/merge/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/modules/samtools/index/main' +include { GATK4_SPLITNCIGARREADS } from '../../modules/nf-core/gatk4/splitncigarreads/main' +include { SAMTOOLS_MERGE } from '../../modules/nf-core/samtools/merge/main' +include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' workflow SPLITNCIGAR { take: diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 8a558bb1..d33ddb57 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -65,22 +65,22 @@ include { ANNOTATE } from '../subworkflows/local/annotate' ======================================================================================== */ -include { FASTQC } from '../modules/nf-core/modules/fastqc/main' -include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' -include { CAT_FASTQ } from '../modules/nf-core/modules/cat/fastq/main' -include { GATK4_BASERECALIBRATOR } from '../modules/nf-core/modules/gatk4/baserecalibrator/main' -include { GATK4_BEDTOINTERVALLIST } from '../modules/nf-core/modules/gatk4/bedtointervallist/main' -include { GATK4_INTERVALLISTTOOLS } from '../modules/nf-core/modules/gatk4/intervallisttools/main' -include { GATK4_HAPLOTYPECALLER } from '../modules/nf-core/modules/gatk4/haplotypecaller/main' -include { GATK4_HAPLOTYPECALLER as GATK4_HAPLOTYPECALLERGVCF } from '../modules/nf-core/modules/gatk4/haplotypecaller/main' -include { GATK4_MERGEVCFS } from '../modules/nf-core/modules/gatk4/mergevcfs/main' -include { GATK4_COMBINEGVCFS } from '../modules/nf-core/modules/gatk4/combinegvcfs/main' -include { GATK4_INDEXFEATUREFILE } from '../modules/nf-core/modules/gatk4/indexfeaturefile/main' -include { GATK4_VARIANTFILTRATION } from '../modules/nf-core/modules/gatk4/variantfiltration/main' -include { SAMTOOLS_INDEX } from '../modules/nf-core/modules/samtools/index/main' -include { TABIX_TABIX as TABIX } from '../modules/nf-core/modules/tabix/tabix/main' -include { TABIX_TABIX as TABIXGVCF } from '../modules/nf-core/modules/tabix/tabix/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' +include { FASTQC } from '../modules/nf-core/fastqc/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { CAT_FASTQ } from '../modules/nf-core/cat/fastq/main' +include { GATK4_BASERECALIBRATOR } from '../modules/nf-core/gatk4/baserecalibrator/main' +include { GATK4_BEDTOINTERVALLIST } from '../modules/nf-core/gatk4/bedtointervallist/main' +include { GATK4_INTERVALLISTTOOLS } from '../modules/nf-core/gatk4/intervallisttools/main' +include { GATK4_HAPLOTYPECALLER } from '../modules/nf-core/gatk4/haplotypecaller/main' +include { GATK4_HAPLOTYPECALLER as GATK4_HAPLOTYPECALLERGVCF } from '../modules/nf-core/gatk4/haplotypecaller/main' +include { GATK4_MERGEVCFS } from '../modules/nf-core/gatk4/mergevcfs/main' +include { GATK4_COMBINEGVCFS } from '../modules/nf-core/gatk4/combinegvcfs/main' +include { GATK4_INDEXFEATUREFILE } from '../modules/nf-core/gatk4/indexfeaturefile/main' +include { GATK4_VARIANTFILTRATION } from '../modules/nf-core/gatk4/variantfiltration/main' +include { SAMTOOLS_INDEX } from '../modules/nf-core/samtools/index/main' +include { TABIX_TABIX as TABIX } from '../modules/nf-core/tabix/tabix/main' +include { TABIX_TABIX as TABIXGVCF } from '../modules/nf-core/tabix/tabix/main' +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' /* ======================================================================================== From 519f412b0b9fe8b4dcc7678cc065d6efbeeab24a Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 18 Sep 2023 19:13:25 +0200 Subject: [PATCH 16/34] update all modules + fix path to subworkflows --- modules.json | 5 + modules/local/{gtf2bed.nf => gtf2bed/main.nf} | 0 modules/local/samplesheet_check.nf | 31 - modules/nf-core/mosdepth/main.nf | 80 ++ modules/nf-core/mosdepth/meta.yml | 109 +++ .../align_star/main.nf} | 4 +- .../local/{annotate.nf => annotate/main.nf} | 6 +- subworkflows/local/bam_markduplicates/main.nf | 43 + .../bam_sort_samtools/main.nf} | 6 +- .../bam_stats_samtools/main.nf} | 6 +- .../local/cram_qc_mosdepth_samtools/main.nf | 38 + .../ensemblvep_annotate/main.nf} | 4 +- subworkflows/local/input_check.nf | 46 - subworkflows/local/prepare_genome.nf | 175 ---- subworkflows/local/prepare_genome/main.nf | 136 +++ .../recalibrate/main.nf} | 6 +- .../snpeff_annotate/main.nf} | 4 +- .../splitncigar/main.nf} | 6 +- subworkflows/nf-core/markduplicates.nf | 54 -- workflows/rnavar.nf | 859 +++++++++--------- 20 files changed, 861 insertions(+), 757 deletions(-) rename modules/local/{gtf2bed.nf => gtf2bed/main.nf} (100%) delete mode 100644 modules/local/samplesheet_check.nf create mode 100644 modules/nf-core/mosdepth/main.nf create mode 100644 modules/nf-core/mosdepth/meta.yml rename subworkflows/{nf-core/align_star.nf => local/align_star/main.nf} (94%) rename subworkflows/local/{annotate.nf => annotate/main.nf} (87%) create mode 100644 subworkflows/local/bam_markduplicates/main.nf rename subworkflows/{nf-core/bam_sort_samtools.nf => local/bam_sort_samtools/main.nf} (87%) rename subworkflows/{nf-core/bam_stats_samtools.nf => local/bam_stats_samtools/main.nf} (78%) create mode 100644 subworkflows/local/cram_qc_mosdepth_samtools/main.nf rename subworkflows/{nf-core/ensemblvep_annotate.nf => local/ensemblvep_annotate/main.nf} (85%) delete mode 100644 subworkflows/local/input_check.nf delete mode 100755 subworkflows/local/prepare_genome.nf create mode 100755 subworkflows/local/prepare_genome/main.nf rename subworkflows/{nf-core/recalibrate.nf => local/recalibrate/main.nf} (85%) rename subworkflows/{nf-core/snpeff_annotate.nf => local/snpeff_annotate/main.nf} (82%) rename subworkflows/{nf-core/splitncigar.nf => local/splitncigar/main.nf} (87%) delete mode 100644 subworkflows/nf-core/markduplicates.nf diff --git a/modules.json b/modules.json index 1e399602..eb4ff72e 100755 --- a/modules.json +++ b/modules.json @@ -105,6 +105,11 @@ "git_sha": "e06548bfa36ee31869b81041879dd6b3a83b1d57", "installed_by": ["modules"] }, + "mosdepth": { + "branch": "master", + "git_sha": "ebb27711cd5f4de921244bfa81c676504072d31c", + "installed_by": ["modules"] + }, "multiqc": { "branch": "master", "git_sha": "a6e11ac655e744f7ebc724be669dd568ffdc0e80", diff --git a/modules/local/gtf2bed.nf b/modules/local/gtf2bed/main.nf similarity index 100% rename from modules/local/gtf2bed.nf rename to modules/local/gtf2bed/main.nf diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf deleted file mode 100644 index 852dcbf9..00000000 --- a/modules/local/samplesheet_check.nf +++ /dev/null @@ -1,31 +0,0 @@ -process SAMPLESHEET_CHECK { - tag "$samplesheet" - label 'process_single' - - conda "conda-forge::python=3.8.3" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/python:3.8.3' : - 'biocontainers/python:3.8.3' }" - - input: - path samplesheet - - output: - path '*.csv' , emit: csv - path "versions.yml", emit: versions - - when: - task.ext.when == null || task.ext.when - - script: // This script is bundled with the pipeline, in nf-core/rnavar/bin/ - """ - check_samplesheet.py \\ - $samplesheet \\ - samplesheet.valid.csv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS - """ -} diff --git a/modules/nf-core/mosdepth/main.nf b/modules/nf-core/mosdepth/main.nf new file mode 100644 index 00000000..74db3a27 --- /dev/null +++ b/modules/nf-core/mosdepth/main.nf @@ -0,0 +1,80 @@ +process MOSDEPTH { + tag "$meta.id" + label 'process_medium' + + conda "bioconda::mosdepth=0.3.3" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/mosdepth:0.3.3--hdfd78af_1' : + 'biocontainers/mosdepth:0.3.3--hdfd78af_1'}" + + input: + tuple val(meta), path(bam), path(bai), path(bed) + tuple val(meta2), path(fasta) + + output: + tuple val(meta), path('*.global.dist.txt') , emit: global_txt + tuple val(meta), path('*.summary.txt') , emit: summary_txt + tuple val(meta), path('*.region.dist.txt') , optional:true, emit: regions_txt + tuple val(meta), path('*.per-base.d4') , optional:true, emit: per_base_d4 + tuple val(meta), path('*.per-base.bed.gz') , optional:true, emit: per_base_bed + tuple val(meta), path('*.per-base.bed.gz.csi') , optional:true, emit: per_base_csi + tuple val(meta), path('*.regions.bed.gz') , optional:true, emit: regions_bed + tuple val(meta), path('*.regions.bed.gz.csi') , optional:true, emit: regions_csi + tuple val(meta), path('*.quantized.bed.gz') , optional:true, emit: quantized_bed + tuple val(meta), path('*.quantized.bed.gz.csi') , optional:true, emit: quantized_csi + tuple val(meta), path('*.thresholds.bed.gz') , optional:true, emit: thresholds_bed + tuple val(meta), path('*.thresholds.bed.gz.csi'), optional:true, emit: thresholds_csi + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def reference = fasta ? "--fasta ${fasta}" : "" + def interval = bed ? "--by ${bed}" : "" + if (bed && args.contains("--by")) { + error "'--by' can only be specified once when running mosdepth! Either remove input BED file definition or remove '--by' from 'ext.args' definition" + } + if (!bed && args.contains("--thresholds")) { + error "'--thresholds' can only be specified in conjunction with '--by'" + } + + """ + mosdepth \\ + --threads $task.cpus \\ + $interval \\ + $reference \\ + $args \\ + $prefix \\ + $bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + mosdepth: \$(mosdepth --version 2>&1 | sed 's/^.*mosdepth //; s/ .*\$//') + END_VERSIONS + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.global.dist.txt + touch ${prefix}.region.dist.txt + touch ${prefix}.summary.txt + touch ${prefix}.per-base.d4 + touch ${prefix}.per-base.bed.gz + touch ${prefix}.per-base.bed.gz.csi + touch ${prefix}.regions.bed.gz + touch ${prefix}.regions.bed.gz.csi + touch ${prefix}.quantized.bed.gz + touch ${prefix}.quantized.bed.gz.csi + touch ${prefix}.thresholds.bed.gz + touch ${prefix}.thresholds.bed.gz.csi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + mosdepth: \$(mosdepth --version 2>&1 | sed 's/^.*mosdepth //; s/ .*\$//') + END_VERSIONS + """ +} diff --git a/modules/nf-core/mosdepth/meta.yml b/modules/nf-core/mosdepth/meta.yml new file mode 100644 index 00000000..adf3893f --- /dev/null +++ b/modules/nf-core/mosdepth/meta.yml @@ -0,0 +1,109 @@ +name: mosdepth +description: Calculates genome-wide sequencing coverage. +keywords: + - mosdepth + - bam + - cram + - coverage +tools: + - mosdepth: + description: | + Fast BAM/CRAM depth calculation for WGS, exome, or targeted sequencing. + documentation: https://github.com/brentp/mosdepth + doi: 10.1093/bioinformatics/btx699 + licence: ["MIT"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: Input BAM/CRAM file + pattern: "*.{bam,cram}" + - bai: + type: file + description: Index for BAM/CRAM file + pattern: "*.{bai,crai}" + - meta2: + type: map + description: | + Groovy Map containing bed information + e.g. [ id:'test' ] + - bed: + type: file + description: BED file with intersected intervals + pattern: "*.{bed}" + - meta3: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'test' ] + - fasta: + type: file + description: Reference genome FASTA file + pattern: "*.{fa,fasta}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - global_txt: + type: file + description: Text file with global cumulative coverage distribution + pattern: "*.{global.dist.txt}" + - regions_txt: + type: file + description: Text file with region cumulative coverage distribution + pattern: "*.{region.dist.txt}" + - summary_txt: + type: file + description: Text file with summary mean depths per chromosome and regions + pattern: "*.{summary.txt}" + - per_base_bed: + type: file + description: BED file with per-base coverage + pattern: "*.{per-base.bed.gz}" + - per_base_csi: + type: file + description: Index file for BED file with per-base coverage + pattern: "*.{per-base.bed.gz.csi}" + - per_base_d4: + type: file + description: D4 file with per-base coverage + pattern: "*.{per-base.d4}" + - regions_bed: + type: file + description: BED file with per-region coverage + pattern: "*.{regions.bed.gz}" + - regions_csi: + type: file + description: Index file for BED file with per-region coverage + pattern: "*.{regions.bed.gz.csi}" + - quantized_bed: + type: file + description: BED file with binned coverage + pattern: "*.{quantized.bed.gz}" + - quantized_csi: + type: file + description: Index file for BED file with binned coverage + pattern: "*.{quantized.bed.gz.csi}" + - thresholds_bed: + type: file + description: BED file with the number of bases in each region that are covered at or above each threshold + pattern: "*.{thresholds.bed.gz}" + - thresholds_csi: + type: file + description: Index file for BED file with threshold coverage + pattern: "*.{thresholds.bed.gz.csi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@joseespinosa" + - "@drpatelh" + - "@ramprasadn" + - "@matthdsm" diff --git a/subworkflows/nf-core/align_star.nf b/subworkflows/local/align_star/main.nf similarity index 94% rename from subworkflows/nf-core/align_star.nf rename to subworkflows/local/align_star/main.nf index 5f165e0d..64ec6526 100644 --- a/subworkflows/nf-core/align_star.nf +++ b/subworkflows/local/align_star/main.nf @@ -2,8 +2,8 @@ // Alignment with STAR // -include { STAR_ALIGN } from '../../modules/nf-core/star/align/main' -include { BAM_SORT_SAMTOOLS } from './bam_sort_samtools' +include { STAR_ALIGN } from '../../../modules/nf-core/star/align/main' +include { BAM_SORT_SAMTOOLS } from '../bam_sort_samtools/main' workflow ALIGN_STAR { take: diff --git a/subworkflows/local/annotate.nf b/subworkflows/local/annotate/main.nf similarity index 87% rename from subworkflows/local/annotate.nf rename to subworkflows/local/annotate/main.nf index 9555dddb..8f2b5104 100644 --- a/subworkflows/local/annotate.nf +++ b/subworkflows/local/annotate/main.nf @@ -2,9 +2,9 @@ // ANNOTATION // -include { SNPEFF_ANNOTATE } from '../nf-core/snpeff_annotate' -include { ENSEMBLVEP_ANNOTATE as MERGE_ANNOTATE } from '../nf-core/ensemblvep_annotate' -include { ENSEMBLVEP_ANNOTATE } from '../nf-core/ensemblvep_annotate' +include { SNPEFF_ANNOTATE } from '../snpeff_annotate/main' +include { ENSEMBLVEP_ANNOTATE as MERGE_ANNOTATE } from '../ensemblvep_annotate/main' +include { ENSEMBLVEP_ANNOTATE } from '../ensemblvep_annotate/main' workflow ANNOTATE { take: diff --git a/subworkflows/local/bam_markduplicates/main.nf b/subworkflows/local/bam_markduplicates/main.nf new file mode 100644 index 00000000..6ee53041 --- /dev/null +++ b/subworkflows/local/bam_markduplicates/main.nf @@ -0,0 +1,43 @@ +// +// MARKDUPLICATES AND QC after mapping +// +// For all modules here: +// A when clause condition is defined in the conf/modules.config to determine if the module should be run + +include { CRAM_QC_MOSDEPTH_SAMTOOLS } from '../cram_qc_mosdepth_samtools/main' +include { GATK4_MARKDUPLICATES } from '../../../modules/nf-core/gatk4/markduplicates/main' + +workflow BAM_MARKDUPLICATES { + take: + bam // channel: [mandatory] [ meta, bam ] + fasta // channel: [mandatory] [ fasta ] + fasta_fai // channel: [mandatory] [ fasta_fai ] + intervals_bed_combined // channel: [optional] [ intervals_bed ] + + main: + versions = Channel.empty() + reports = Channel.empty() + + // RUN MARKDUPLICATES + GATK4_MARKDUPLICATES(bam, fasta, fasta_fai) + + // Join with the crai file + cram = GATK4_MARKDUPLICATES.out.cram.join(GATK4_MARKDUPLICATES.out.crai, failOnDuplicate: true, failOnMismatch: true) + + // QC on CRAM + CRAM_QC_MOSDEPTH_SAMTOOLS(cram, fasta, intervals_bed_combined) + + // Gather all reports generated + reports = reports.mix(GATK4_MARKDUPLICATES.out.metrics) + reports = reports.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.reports) + + // Gather versions of all tools used + versions = versions.mix(GATK4_MARKDUPLICATES.out.versions) + versions = versions.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.versions) + + emit: + cram + reports + + versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/bam_sort_samtools.nf b/subworkflows/local/bam_sort_samtools/main.nf similarity index 87% rename from subworkflows/nf-core/bam_sort_samtools.nf rename to subworkflows/local/bam_sort_samtools/main.nf index 2e84fa95..9b4f6e31 100644 --- a/subworkflows/nf-core/bam_sort_samtools.nf +++ b/subworkflows/local/bam_sort_samtools/main.nf @@ -2,9 +2,9 @@ // Sort, index BAM file and run samtools stats, flagstat and idxstats // -include { SAMTOOLS_SORT } from '../../modules/nf-core/samtools/sort/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' -include { BAM_STATS_SAMTOOLS } from './bam_stats_samtools' +include { SAMTOOLS_SORT } from '../../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' +include { BAM_STATS_SAMTOOLS } from '../bam_stats_samtools/main' workflow BAM_SORT_SAMTOOLS { take: diff --git a/subworkflows/nf-core/bam_stats_samtools.nf b/subworkflows/local/bam_stats_samtools/main.nf similarity index 78% rename from subworkflows/nf-core/bam_stats_samtools.nf rename to subworkflows/local/bam_stats_samtools/main.nf index c45a91ae..e94c1bd1 100644 --- a/subworkflows/nf-core/bam_stats_samtools.nf +++ b/subworkflows/local/bam_stats_samtools/main.nf @@ -2,9 +2,9 @@ // Run SAMtools stats, flagstat and idxstats // -include { SAMTOOLS_STATS } from '../../modules/nf-core/samtools/stats/main' -include { SAMTOOLS_IDXSTATS } from '../../modules/nf-core/samtools/idxstats/main' -include { SAMTOOLS_FLAGSTAT } from '../../modules/nf-core/samtools/flagstat/main' +include { SAMTOOLS_STATS } from '../../../modules/nf-core/samtools/stats/main' +include { SAMTOOLS_IDXSTATS } from '../../../modules/nf-core/samtools/idxstats/main' +include { SAMTOOLS_FLAGSTAT } from '../../../modules/nf-core/samtools/flagstat/main' workflow BAM_STATS_SAMTOOLS { take: diff --git a/subworkflows/local/cram_qc_mosdepth_samtools/main.nf b/subworkflows/local/cram_qc_mosdepth_samtools/main.nf new file mode 100644 index 00000000..fd070a68 --- /dev/null +++ b/subworkflows/local/cram_qc_mosdepth_samtools/main.nf @@ -0,0 +1,38 @@ +// +// QC on CRAM +// +// For all modules here: +// A when clause condition is defined in the conf/modules.config to determine if the module should be run + +include { SAMTOOLS_STATS } from '../../../modules/nf-core/samtools/stats/main' +include { MOSDEPTH } from '../../../modules/nf-core/mosdepth/main' + +workflow CRAM_QC_MOSDEPTH_SAMTOOLS { + take: + cram // channel: [mandatory] [ meta, cram, crai ] + fasta // channel: [mandatory] [ fasta ] + intervals + + main: + versions = Channel.empty() + reports = Channel.empty() + + // Reports run on cram + SAMTOOLS_STATS(cram, fasta.map{ it -> [ [ id:'fasta' ], it ] }) + + MOSDEPTH(cram.combine(intervals.map{ meta, bed -> [ bed?:[] ] }), fasta.map{ it -> [ [ id:'fasta' ], it ] }) + + // Gather all reports generated + reports = reports.mix(SAMTOOLS_STATS.out.stats) + reports = reports.mix(MOSDEPTH.out.global_txt) + reports = reports.mix(MOSDEPTH.out.regions_txt) + + // Gather versions of all tools used + versions = versions.mix(MOSDEPTH.out.versions) + versions = versions.mix(SAMTOOLS_STATS.out.versions.first()) + + emit: + reports + + versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/ensemblvep_annotate.nf b/subworkflows/local/ensemblvep_annotate/main.nf similarity index 85% rename from subworkflows/nf-core/ensemblvep_annotate.nf rename to subworkflows/local/ensemblvep_annotate/main.nf index 87cfc2b7..83187150 100644 --- a/subworkflows/nf-core/ensemblvep_annotate.nf +++ b/subworkflows/local/ensemblvep_annotate/main.nf @@ -2,8 +2,8 @@ // Run VEP to annotate VCF files // -include { ENSEMBLVEP_VEP } from '../../modules/nf-core/ensemblvep/vep/main' -include { TABIX_BGZIPTABIX } from '../../modules/nf-core/tabix/bgziptabix/main' +include { ENSEMBLVEP_VEP } from '../../../modules/nf-core/ensemblvep/vep/main' +include { TABIX_BGZIPTABIX } from '../../../modules/nf-core/tabix/bgziptabix/main' workflow ENSEMBLVEP_ANNOTATE { take: diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf deleted file mode 100644 index 95fc458e..00000000 --- a/subworkflows/local/input_check.nf +++ /dev/null @@ -1,46 +0,0 @@ -// -// Check input samplesheet and get read channels -// - -include { SAMPLESHEET_CHECK } from '../../modules/local/samplesheet_check' - -workflow INPUT_CHECK { - take: - samplesheet // file: /path/to/samplesheet.csv - - main: - SAMPLESHEET_CHECK ( - samplesheet - ) - .csv - .splitCsv ( header:true, sep:',' ) - .map { create_fastq_channel(it) } - .set { reads } - - emit: - reads // channel: [ val(meta), [ reads ] ] - versions = SAMPLESHEET_CHECK.out.versions // channel: [ versions.yml ] -} - -// Function to get list of [ meta, [ fastq_1, fastq_2 ] ] -def create_fastq_channel(LinkedHashMap row) { - // create meta map - def meta = [:] - meta.id = row.sample - meta.single_end = row.single_end.toBoolean() - - // add path(s) of the fastq file(s) to the meta map - def fastq_meta = [] - if (!file(row.fastq_1).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 1 FastQ file does not exist!\n${row.fastq_1}" - } - if (meta.single_end) { - fastq_meta = [ meta, [ file(row.fastq_1) ] ] - } else { - if (!file(row.fastq_2).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 2 FastQ file does not exist!\n${row.fastq_2}" - } - fastq_meta = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] - } - return fastq_meta -} diff --git a/subworkflows/local/prepare_genome.nf b/subworkflows/local/prepare_genome.nf deleted file mode 100755 index c9e1e417..00000000 --- a/subworkflows/local/prepare_genome.nf +++ /dev/null @@ -1,175 +0,0 @@ -// -// Uncompress and prepare reference genome files -// - -include { GATK4_CREATESEQUENCEDICTIONARY } from '../../modules/nf-core/gatk4/createsequencedictionary/main' //addParams(options: params.genome_options) -include { GFFREAD } from '../../modules/nf-core/gffread/main' //addParams(options: params.gffread_options) -include { GTF2BED } from '../../modules/local/gtf2bed' -include { BEDTOOLS_SORT } from '../../modules/nf-core/bedtools/sort/main' -include { BEDTOOLS_MERGE } from '../../modules/nf-core/bedtools/merge/main' -include { GUNZIP as GUNZIP_FASTA } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) -include { GUNZIP as GUNZIP_GENE_BED } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) -include { GUNZIP as GUNZIP_GFF } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) -include { GUNZIP as GUNZIP_GTF } from '../../modules/nf-core/gunzip/main' //addParams(options: params.genome_options) -include { SAMTOOLS_FAIDX } from '../../modules/nf-core/samtools/faidx/main' //addParams(options: params.genome_options) -include { STAR_GENOMEGENERATE } from '../../modules/nf-core/star/genomegenerate/main' //addParams(options: params.star_index_options) -include { UNTAR as UNTAR_STAR_INDEX } from '../../modules/nf-core/untar/main' //addParams(options: params.star_untar_options) - - -workflow PREPARE_GENOME { - take: - prepare_tool_indices - feature_type - - main: - - ch_versions = Channel.empty() - - // - // Uncompress genome fasta file if required - // - if (params.fasta.endsWith('.gz')) { - GUNZIP_FASTA ( - Channel.fromPath(params.fasta).map{ it -> [[id:it[0].baseName], it] } - ) - ch_fasta = GUNZIP_FASTA.out.gunzip.map{ meta, fasta -> [fasta] }.collect() - ch_versions = ch_versions.mix(GUNZIP_FASTA.out.versions) - } else { - ch_fasta = Channel.fromPath(params.fasta).collect() - } - - // - // Uncompress GTF annotation file or create from GFF3 if required - // - ch_gffread_version = Channel.empty() - if (params.gtf) { - if (params.gtf.endsWith('.gz')) { - GUNZIP_GTF ( - Channel.fromPath(params.gtf).map{ it -> [[id:it[0].baseName], it] } - ) - ch_gtf = GUNZIP_GTF.out.gunzip.map{ meta, gtf -> [gtf] }.collect() - ch_versions = ch_versions.mix(GUNZIP_GTF.out.versions) - } else { - ch_gtf = Channel.fromPath(params.gtf).collect() - } - } else if (params.gff) { - if (params.gff.endsWith('.gz')) { - GUNZIP_GFF ( - Channel.fromPath(params.gff).map{ it -> [[id:it[0].baseName], it] } - ) - ch_gff = GUNZIP_GFF.out.gunzip.map{ meta, gff -> [gff] }.collect() - ch_versions = ch_versions.mix(GUNZIP_GFF.out.versions) - } else { - ch_gff = Channel.fromPath(params.gff).collect() - } - - GFFREAD ( - ch_gff - ) - .gtf - .set { ch_gtf } - - ch_versions = ch_versions.mix(GFFREAD.out.versions) - } - - // - // Uncompress exon BED annotation file or create from GTF if required - // - if (params.exon_bed) { - if (params.exon_bed.endsWith('.gz')) { - GUNZIP_GENE_BED ( - Channel.fromPath(params.exon_bed).map{ it -> [[id:it[0].baseName], it] } - ) - ch_gene_bed = GUNZIP_GENE_BED.out.gunzip.map{ meta, bed -> [bed] }.collect() - ch_versions = ch_versions.mix(GUNZIP_GENE_BED.out.versions) - } else { - ch_gene_bed = Channel.fromPath(params.exon_bed).collect() - } - } else { - ch_exon_bed = GTF2BED ( ch_gtf , feature_type).bed.collect() - ch_versions = ch_versions.mix(GTF2BED.out.versions) - } - - //ch_exon_bed.view() - //ch_exon_bed.map{ it -> [[id:'exome'], it] } - //ch_exon_bed.view() - // Bedtools sort - ch_bedtools_sort = BEDTOOLS_SORT(ch_exon_bed.map{ it -> [[id:'exome'], it] }, 'sorted').sorted.collect() - ch_versions = ch_versions.mix(BEDTOOLS_SORT.out.versions) - - - // Bedtools merge - ch_bedtools_merge = BEDTOOLS_MERGE(ch_bedtools_sort).bed - ch_versions = ch_versions.mix(BEDTOOLS_MERGE.out.versions) - - - // Index the genome fasta - ch_fasta_fai = Channel.empty() - if (params.fasta_fai) ch_fasta_fai = Channel.fromPath(params.fasta_fai).collect() - if (!params.fasta_fai) { - SAMTOOLS_FAIDX( - ch_fasta.map{ it -> [[id:it[0].getName()], it]} - ) - ch_fasta_fai = SAMTOOLS_FAIDX.out.fai.map{ meta, fai -> [fai] }.collect() - ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) - } - - // Create dictionary file for the genome fasta - ch_fasta_dict = Channel.empty() - if (params.dict) ch_fasta_dict = Channel.fromPath(params.dict).collect() - else ch_fasta_dict = GATK4_CREATESEQUENCEDICTIONARY(ch_fasta).dict - - // - // Uncompress STAR index or generate from scratch if required - // - ch_star_index = Channel.empty() - if ('star' in prepare_tool_indices) { - if (params.star_index) { - if (params.star_index.endsWith('.tar.gz')) { - UNTAR_STAR_INDEX ( - Channel.fromPath(params.star_index).map{ it -> [[id:it[0].baseName], it] } - ) - ch_star_index = UNTAR_STAR_INDEX.out.untar.map{ meta, star_index -> [star_index] }.collect() - ch_versions = ch_versions.mix(UNTAR_STAR_INDEX.out.versions) - } else { - ch_star_index = Channel.fromPath(params.star_index).collect() - } - } - else { - STAR_GENOMEGENERATE ( - ch_fasta,ch_gtf - ) - .index - .set { ch_star_index } - ch_versions = ch_versions.mix(STAR_GENOMEGENERATE.out.versions) - } - - //if((!ch_star_index) || getIndexVersion(ch_star_index) != '2.7.4a'){ - // ch_star_index = STAR_GENOMEGENERATE(ch_fasta,ch_gtf).index - // ch_versions = ch_versions.mix(STAR_GENOMEGENERATE.out.versions) - //} - } - - - emit: - fasta = ch_fasta // path: genome.fasta - fai = ch_fasta_fai // path: genome.fasta.fai - dict = ch_fasta_dict // path: genome.fasta.dict - gtf = ch_gtf // path: genome.gtf - exon_bed = ch_exon_bed // path: exon.bed - bedtools_sort = ch_bedtools_sort // path: sort.bed - bedtools_merge = ch_bedtools_merge // path: merge.bed - star_index = ch_star_index // path: star/index/ - versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] -} - -def getIndexVersion( index_path ) { - genomeParameters = new File("$index_path/genomeParameters.txt") - if ( genomeParameters.exists() ) { - for(line: genomeParameters.readLines()){ - if(line.startsWith("versionGenome")){ - return line.split("\t")[1].trim() - } - } - } -} diff --git a/subworkflows/local/prepare_genome/main.nf b/subworkflows/local/prepare_genome/main.nf new file mode 100755 index 00000000..598dc4e4 --- /dev/null +++ b/subworkflows/local/prepare_genome/main.nf @@ -0,0 +1,136 @@ +// +// Uncompress and prepare reference genome files +// + +include { BEDTOOLS_MERGE } from '../../../modules/nf-core/bedtools/merge/main' +include { BEDTOOLS_SORT } from '../../../modules/nf-core/bedtools/sort/main' +include { GATK4_CREATESEQUENCEDICTIONARY } from '../../../modules/nf-core/gatk4/createsequencedictionary/main' +include { GFFREAD } from '../../../modules/nf-core/gffread/main' +include { GTF2BED } from '../../../modules/local/gtf2bed' +include { GUNZIP as GUNZIP_FASTA } from '../../../modules/nf-core/gunzip/main' +include { GUNZIP as GUNZIP_GENE_BED } from '../../../modules/nf-core/gunzip/main' +include { GUNZIP as GUNZIP_GFF } from '../../../modules/nf-core/gunzip/main' +include { GUNZIP as GUNZIP_GTF } from '../../../modules/nf-core/gunzip/main' +include { SAMTOOLS_FAIDX } from '../../../modules/nf-core/samtools/faidx/main' +include { STAR_GENOMEGENERATE } from '../../../modules/nf-core/star/genomegenerate/main' +include { UNTAR as UNTAR_STAR_INDEX } from '../../../modules/nf-core/untar/main' + +workflow PREPARE_GENOME { + take: + fasta // file: /path/to/genome.fasta + // gtf // file: /path/to/genome.gtf + // gff // file: /path/to/genome.gff + // exon_bed // file: /path/to/gene.bed + // prepare_tool_indices + // feature_type + + main: + ch_versions = Channel.empty() + + ch_fasta = fasta.map{ fasta -> [ [ id:fasta.baseName ], fasta ] } + + // + // Uncompress genome fasta file if required + // + // if (fasta.endsWith('.gz')) { + // ch_fasta = GUNZIP_FASTA([[:], fasta]).gunzip.map{ meta, fasta -> fasta } + // ch_versions = ch_versions.mix(GUNZIP_FASTA.out.versions) + // } else { + // ch_fasta = Channel.value(file(fasta)) + // } + + // + // Uncompress GTF annotation file or create from GFF3 if required + // + // if (gtf) { + // if (gtf.endsWith('.gz')) { + // ch_gtf = GUNZIP_GTF([[:], gtf]).gunzip.map{ meta, gtf -> gtf } + // ch_versions = ch_versions.mix(GUNZIP_GTF.out.versions) + // } else ch_gtf = Channel.value(file(gtf)) + // } else if (gff) { + // if (gff.endsWith('.gz')) { + // ch_gff = GUNZIP_GFF([[:], gff]).gunzip.map{ meta, gff -> gff } + // ch_versions = ch_versions.mix(GUNZIP_GFF.out.versions) + // } else ch_gff = Channel.value(file(gff)) + // ch_gtf = GFFREAD(ch_gff).gtf + // ch_versions = ch_versions.mix(GFFREAD.out.versions) + // } + + // + // Uncompress exon BED annotation file or create from GTF if required + // + // if (exon_bed) { + // if (exon_bed.endsWith('.gz')) { + // exonGENE_BED( + // Channel.fromPath(exon_bed).map{ it -> [[id:it[0].baseName], it] } + // ) + // ch_exon_bed = GUNZIP_GENE_BED.out.gunzip.map{ meta, bed -> [bed] }.collect() + // ch_versions = ch_versions.mix(GUNZIP_GENE_BED.out.versions) + // } else { + // ch_exon_bed = Channel.fromPath(exon_bed).collect() + // } + // } else { + // ch_exon_bed = GTF2BED( ch_gtf , feature_type).bed.collect() + // ch_versions = ch_versions.mix(GTF2BED.out.versions) + // } + + //ch_exon_bed.view() + //ch_exon_bed.map{ it -> [[id:'exome'], it] } + //ch_exon_bed.view() + // Bedtools sort + // ch_bedtools_sort = BEDTOOLS_SORT(ch_exon_bed.map{ it -> [[id:'exome'], it] }, 'sorted').sorted.collect() + // ch_versions = ch_versions.mix(BEDTOOLS_SORT.out.versions) + + + // // Bedtools merge + // ch_bedtools_merge = BEDTOOLS_MERGE(ch_bedtools_sort).bed + // ch_versions = ch_versions.mix(BEDTOOLS_MERGE.out.versions) + + + // Index the genome fasta + SAMTOOLS_FAIDX(ch_fasta, [['id':null], []]) + + ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) + + // Create dictionary file for the genome fasta + // ch_fasta_dict = Channel.empty() + // if (params.dict) ch_fasta_dict = Channel.fromPath(params.dict).collect() + // else ch_fasta_dict = GATK4_CREATESEQUENCEDICTIONARY(ch_fasta).dict + + // + // Uncompress STAR index or generate from scratch if required + // + // ch_star_index = Channel.empty() + // if ('star' in prepare_tool_indices) { + // if (params.star_index) { + // if (params.star_index.endsWith('.tar.gz')) { + // UNTAR_STAR_INDEX( + // Channel.fromPath(params.star_index).map{ it -> [[id:it[0].baseName], it] } + // ) + // ch_star_index = UNTAR_STAR_INDEX.out.untar.map{ meta, star_index -> [star_index] }.collect() + // ch_versions = ch_versions.mix(UNTAR_STAR_INDEX.out.versions) + // } else { + // ch_star_index = Channel.fromPath(params.star_index).collect() + // } + // } + // else { + // STAR_GENOMEGENERATE( + // ch_fasta,ch_gtf + // ) + // .index + // .set { ch_star_index } + // ch_versions = ch_versions.mix(STAR_GENOMEGENERATE.out.versions) + // } + // } + + + emit: + // fasta = ch_fasta // path: genome.fasta + fasta_fai = SAMTOOLS_FAIDX.out.fai.map{ meta, fai -> [fai] } // path: genome.fasta.fai dict = ch_fasta_dict // path: genome.fasta.dict + // gtf = ch_gtf // path: genome.gtf + // exon_bed = ch_exon_bed // path: exon.bed + // bedtools_sort = ch_bedtools_sort // path: sort.bed + // bedtools_merge = ch_bedtools_merge // path: merge.bed + // star_index = ch_star_index // path: star/index/ + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/recalibrate.nf b/subworkflows/local/recalibrate/main.nf similarity index 85% rename from subworkflows/nf-core/recalibrate.nf rename to subworkflows/local/recalibrate/main.nf index 6d79257f..3b4ab0d8 100644 --- a/subworkflows/nf-core/recalibrate.nf +++ b/subworkflows/local/recalibrate/main.nf @@ -4,9 +4,9 @@ ======================================================================================== */ -include { GATK4_APPLYBQSR as APPLYBQSR } from '../../modules/nf-core/gatk4/applybqsr/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' -include { SAMTOOLS_STATS } from '../../modules/nf-core/samtools/stats/main' +include { GATK4_APPLYBQSR as APPLYBQSR } from '../../../modules/nf-core/gatk4/applybqsr/main' +include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' +include { SAMTOOLS_STATS } from '../../../modules/nf-core/samtools/stats/main' workflow RECALIBRATE { take: diff --git a/subworkflows/nf-core/snpeff_annotate.nf b/subworkflows/local/snpeff_annotate/main.nf similarity index 82% rename from subworkflows/nf-core/snpeff_annotate.nf rename to subworkflows/local/snpeff_annotate/main.nf index e3dd8f0f..241cb190 100644 --- a/subworkflows/nf-core/snpeff_annotate.nf +++ b/subworkflows/local/snpeff_annotate/main.nf @@ -2,8 +2,8 @@ // Run snpEff to annotate VCF files // -include { SNPEFF_SNPEFF } from '../../modules/nf-core/snpeff/snpeff/main' -include { TABIX_BGZIPTABIX } from '../../modules/nf-core/tabix/bgziptabix/main' +include { SNPEFF_SNPEFF } from '../../../modules/nf-core/snpeff/snpeff/main' +include { TABIX_BGZIPTABIX } from '../../../modules/nf-core/tabix/bgziptabix/main' workflow SNPEFF_ANNOTATE { take: diff --git a/subworkflows/nf-core/splitncigar.nf b/subworkflows/local/splitncigar/main.nf similarity index 87% rename from subworkflows/nf-core/splitncigar.nf rename to subworkflows/local/splitncigar/main.nf index 88c63a38..9348a098 100644 --- a/subworkflows/nf-core/splitncigar.nf +++ b/subworkflows/local/splitncigar/main.nf @@ -2,9 +2,9 @@ // Subworkflow: Run GATK4 SplitNCigarReads with intervals, merge and index BAM file. // -include { GATK4_SPLITNCIGARREADS } from '../../modules/nf-core/gatk4/splitncigarreads/main' -include { SAMTOOLS_MERGE } from '../../modules/nf-core/samtools/merge/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' +include { GATK4_SPLITNCIGARREADS } from '../../../modules/nf-core/gatk4/splitncigarreads/main' +include { SAMTOOLS_MERGE } from '../../../modules/nf-core/samtools/merge/main' +include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' workflow SPLITNCIGAR { take: diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf deleted file mode 100644 index 18d3dc39..00000000 --- a/subworkflows/nf-core/markduplicates.nf +++ /dev/null @@ -1,54 +0,0 @@ -// -// GATK4 MarkDuplicates, index BAM file and run samtools stats, flagstat and idxstats -// - -include { BAM_STATS_SAMTOOLS } from './bam_stats_samtools' -include { GATK4_MARKDUPLICATES } from '../../modules/nf-core/gatk4/markduplicates/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' - -workflow MARKDUPLICATES { - take: - bam // channel: [ val(meta), [ bam ] ] - - main: - - ch_versions = Channel.empty() - - GATK4_MARKDUPLICATES ( - bam - ) - ch_versions = ch_versions.mix(GATK4_MARKDUPLICATES.out.versions.first()) - - // - // Index BAM file and run samtools stats, flagstat and idxstats - // - SAMTOOLS_INDEX ( - GATK4_MARKDUPLICATES.out.bam - ) - ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) - - GATK4_MARKDUPLICATES.out.bam - .join(SAMTOOLS_INDEX.out.bai, by: [0], remainder: true) - .join(SAMTOOLS_INDEX.out.csi, by: [0], remainder: true) - .map{meta, bam, bai, csi -> - if (bai) [meta, bam, bai] - else [meta, bam, csi]} - .set{ch_bam_bai} - - BAM_STATS_SAMTOOLS ( - ch_bam_bai - ) - ch_versions = ch_versions.mix(BAM_STATS_SAMTOOLS.out.versions.first()) - - emit: - bam = GATK4_MARKDUPLICATES.out.bam // channel: [ val(meta), [ bam ] ] - bam_bai = ch_bam_bai // channel: [ val(meta), [ bam ], [bai or csi] ] - metrics = GATK4_MARKDUPLICATES.out.metrics // channel: [ val(meta), [ metrics ] ] - - bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), [ bai ] ] - csi = SAMTOOLS_INDEX.out.csi // channel: [ val(meta), [ csi ] ] - stats = BAM_STATS_SAMTOOLS.out.stats // channel: [ val(meta), [ stats ] ] - flagstat = BAM_STATS_SAMTOOLS.out.flagstat // channel: [ val(meta), [ flagstat ] ] - idxstats = BAM_STATS_SAMTOOLS.out.idxstats // channel: [ val(meta), [ idxstats ] ] - versions = ch_versions // channel: [versions.yml] -} diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index d33ddb57..64264b44 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -55,9 +55,14 @@ ch_multiqc_custom_methods_description = params.multiqc_methods_description ? fil ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { INPUT_CHECK } from '../subworkflows/local/input_check' // Validate the input samplesheet.csv and prepare input channels -include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome' // Build the genome index and other reference files -include { ANNOTATE } from '../subworkflows/local/annotate' // Annotate variants using snpEff or VEP or both +include { GTF2BED } from '../modules/local/gtf2bed/main' + +include { ALIGN_STAR } from '../subworkflows/local/align_star/main' // Align reads to genome and sort and index the alignment file +include { ANNOTATE } from '../subworkflows/local/annotate/main' // Annotate variants using snpEff or VEP or both +include { BAM_MARKDUPLICATES } from '../subworkflows/local/bam_markduplicates/main' // Mark duplicates in the BAM file +include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome/main' // Build the genome index and other reference files +include { RECALIBRATE } from '../subworkflows/local/recalibrate/main' // Estimate and correct systematic bias +include { SPLITNCIGAR } from '../subworkflows/local/splitncigar/main' // Splits reads that contain Ns in their cigar string /* ======================================================================================== @@ -82,54 +87,38 @@ include { TABIX_TABIX as TABIX } from '../modules/ include { TABIX_TABIX as TABIXGVCF } from '../modules/nf-core/tabix/tabix/main' include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' -/* -======================================================================================== - IMPORT LOCAL MODULES -======================================================================================== -*/ - -include { GTF2BED } from '../modules/local/gtf2bed' - -/* -======================================================================================== - IMPORT NF-CORE SUBWORKFLOWS -======================================================================================== -*/ - -include { ALIGN_STAR } from '../subworkflows/nf-core/align_star' // Align reads to genome and sort and index the alignment file -include { MARKDUPLICATES } from '../subworkflows/nf-core/markduplicates' // Mark duplicates in the BAM file -include { SPLITNCIGAR } from '../subworkflows/nf-core/splitncigar' // Splits reads that contain Ns in their cigar string -include { RECALIBRATE } from '../subworkflows/nf-core/recalibrate' // Estimate and correct systematic bias - /* ======================================================================================== VARIABLES ======================================================================================== */ -// Check STAR alignment parameters -def prepareToolIndices = params.aligner -def seq_platform = params.seq_platform ?: [] -def seq_center = params.seq_center ?: [] - -// Initialize file channels based on params -ch_dbsnp = params.dbsnp ? Channel.fromPath(params.dbsnp).collect() : Channel.empty() -ch_dbsnp_tbi = params.dbsnp_tbi ? Channel.fromPath(params.dbsnp_tbi).collect() : Channel.empty() -ch_known_indels = params.known_indels ? Channel.fromPath(params.known_indels).collect() : Channel.empty() -ch_known_indels_tbi = params.known_indels_tbi ? Channel.fromPath(params.known_indels_tbi).collect() : Channel.empty() - -// Initialize variant annotation associated channels -ch_snpeff_db = params.snpeff_db ?: Channel.empty() -ch_vep_cache_version = params.vep_cache_version ?: Channel.empty() -ch_vep_genome = params.vep_genome ?: Channel.empty() -ch_vep_species = params.vep_species ?: Channel.empty() -ch_snpeff_cache = params.snpeff_cache ? Channel.fromPath(params.snpeff_cache).collect() : [] -ch_vep_cache = params.vep_cache ? Channel.fromPath(params.vep_cache).collect() : [] +// // Check STAR alignment parameters +// def prepareToolIndices = params.aligner +// def seq_platform = params.seq_platform ?: [] +// def seq_center = params.seq_center ?: [] + +// // Initialize file channels based on params +// ch_dbsnp = params.dbsnp ? Channel.fromPath(params.dbsnp).collect() : Channel.empty() +// ch_dbsnp_tbi = params.dbsnp_tbi ? Channel.fromPath(params.dbsnp_tbi).collect() : Channel.empty() +// ch_known_indels = params.known_indels ? Channel.fromPath(params.known_indels).collect() : Channel.empty() +// ch_known_indels_tbi = params.known_indels_tbi ? Channel.fromPath(params.known_indels_tbi).collect() : Channel.empty() + +// // Initialize variant annotation associated channels +// ch_snpeff_db = params.snpeff_db ?: Channel.empty() +// ch_vep_cache_version = params.vep_cache_version ?: Channel.empty() +// ch_vep_genome = params.vep_genome ?: Channel.empty() +// ch_vep_species = params.vep_species ?: Channel.empty() +// ch_snpeff_cache = params.snpeff_cache ? Channel.fromPath(params.snpeff_cache).collect() : [] +// ch_vep_cache = params.vep_cache ? Channel.fromPath(params.vep_cache).collect() : [] // MultiQC reporting -def multiqc_report = [] +// def multiqc_report = [] +// Initialize file channels based on params, defined in the params.genomes[params.genome] scope +ch_fasta = params.fasta ? Channel.fromPath(params.fasta).first() : Channel.empty() + /* ======================================================================================== RUN MAIN WORKFLOW RNAVAR @@ -148,397 +137,407 @@ workflow RNAVAR { // PREPARE_GENOME( - prepareToolIndices, - params.feature_type + ch_fasta + // params.gtf, + // params.gff, + // params.gene_bed, + // params.aligner, + // params.feature_type ) - ch_genome_bed = Channel.from([id:'genome.bed']).combine(PREPARE_GENOME.out.exon_bed) - ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) - - // - // SUBWORKFLOW: Read in samplesheet, validate and stage input files - // - INPUT_CHECK(file(params.input)) - .reads - .map { - meta, fastq -> - def meta_clone = meta.clone() - meta_clone.id = meta_clone.id.split('_')[0..-2].join('_') - [ meta_clone, fastq ] - } - .groupTuple(by: [0]) - .branch { - meta, fastq -> - single : fastq.size() == 1 - return [ meta, fastq.flatten() ] - multiple: fastq.size() > 1 - return [ meta, fastq.flatten() ] - } - .set { ch_fastq } - - ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) - // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") - // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ - // ! There is currently no tooling to help you write a sample sheet schema - - // - // MODULE: Concatenate FastQ files from same sample if required - // - CAT_FASTQ(ch_fastq.multiple) - .reads - .mix(ch_fastq.single) - .set { ch_cat_fastq } - ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) - - // - // MODULE: Generate QC summary using FastQC - // - FASTQC(ch_cat_fastq) - ch_reports = ch_reports.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) - ch_versions = ch_versions.mix(FASTQC.out.versions.first()) - - // - // MODULE: Prepare the interval list from the GTF file using GATK4 BedToIntervalList - // - ch_interval_list = Channel.empty() - GATK4_BEDTOINTERVALLIST(ch_genome_bed, PREPARE_GENOME.out.dict) - ch_interval_list = GATK4_BEDTOINTERVALLIST.out.interval_list - ch_versions = ch_versions.mix(GATK4_BEDTOINTERVALLIST.out.versions.first().ifEmpty(null)) - - // - // MODULE: Scatter one interval-list into many interval-files using GATK4 IntervalListTools - // - ch_interval_list_split = Channel.empty() - if (!params.skip_intervallisttools) { - GATK4_INTERVALLISTTOOLS(ch_interval_list) - ch_interval_list_split = GATK4_INTERVALLISTTOOLS.out.interval_list.map{ meta, bed -> [bed] }.flatten() - } - else ch_interval_list_split = ch_interval_list - - // - // SUBWORKFLOW: Perform read alignment using STAR aligner - // - ch_genome_bam = Channel.empty() - ch_genome_bam_index = Channel.empty() - ch_samtools_stats = Channel.empty() - ch_samtools_flagstat = Channel.empty() - ch_samtools_idxstats = Channel.empty() - ch_star_multiqc = Channel.empty() - ch_aligner_pca_multiqc = Channel.empty() - ch_aligner_clustering_multiqc = Channel.empty() - - if (params.aligner == 'star') { - ALIGN_STAR( - ch_cat_fastq, - PREPARE_GENOME.out.star_index, - PREPARE_GENOME.out.gtf, - params.star_ignore_sjdbgtf, - seq_platform, - seq_center - ) - ch_genome_bam = ALIGN_STAR.out.bam - ch_genome_bam_index = ALIGN_STAR.out.bai - ch_transcriptome_bam = ALIGN_STAR.out.bam_transcript - - // Gather QC reports - ch_reports = ch_reports.mix(ALIGN_STAR.out.stats.collect{it[1]}.ifEmpty([])) - ch_reports = ch_reports.mix(ALIGN_STAR.out.log_final.collect{it[1]}.ifEmpty([])) - ch_versions = ch_versions.mix(ALIGN_STAR.out.versions.first().ifEmpty(null)) - - // - // SUBWORKFLOW: Mark duplicates with GATK4 - // - MARKDUPLICATES(ch_genome_bam) - ch_genome_bam = MARKDUPLICATES.out.bam_bai - - //Gather QC reports - ch_reports = ch_reports.mix(MARKDUPLICATES.out.stats.collect{it[1]}.ifEmpty([])) - ch_reports = ch_reports.mix(MARKDUPLICATES.out.metrics.collect{it[1]}.ifEmpty([])) - ch_versions = ch_versions.mix(MARKDUPLICATES.out.versions.first().ifEmpty(null)) - - // - // SUBWORKFLOW: SplitNCigarReads from GATK4 over the intervals - // Splits reads that contain Ns in their cigar string(e.g. spanning splicing events in RNAseq data). - // - ch_splitncigar_bam_bai = Channel.empty() - SPLITNCIGAR( - ch_genome_bam, - PREPARE_GENOME.out.fasta, - PREPARE_GENOME.out.fai, - PREPARE_GENOME.out.dict, - ch_interval_list_split - ) - ch_splitncigar_bam_bai = SPLITNCIGAR.out.bam_bai - ch_versions = ch_versions.mix(SPLITNCIGAR.out.versions.first().ifEmpty(null)) - - // - // MODULE: BaseRecalibrator from GATK4 - // Generates a recalibration table based on various co-variates - // - ch_bam_variant_calling = Channel.empty() - if (!params.skip_baserecalibration) { - ch_bqsr_table = Channel.empty() - // known_sites is made by grouping both the dbsnp and the known indels ressources - // they can either or both be optional - ch_known_sites = ch_dbsnp.concat(ch_known_indels).collect() - ch_known_sites_tbi = ch_dbsnp_tbi.concat(ch_known_indels_tbi).collect() - - ch_interval_list_recalib = ch_interval_list.map{ meta, bed -> [bed] }.flatten() - ch_splitncigar_bam_bai.combine(ch_interval_list_recalib) - .map{ meta, bam, bai, interval -> [ meta, bam, bai, interval] - }.set{ch_splitncigar_bam_bai_interval} - - GATK4_BASERECALIBRATOR( - ch_splitncigar_bam_bai_interval, - PREPARE_GENOME.out.fasta, - PREPARE_GENOME.out.fai, - PREPARE_GENOME.out.dict, - ch_known_sites, - ch_known_sites_tbi - ) - ch_bqsr_table = GATK4_BASERECALIBRATOR.out.table - - // Gather QC reports - ch_reports = ch_reports.mix(ch_bqsr_table.map{ meta, table -> table}) - ch_versions = ch_versions.mix(GATK4_BASERECALIBRATOR.out.versions.first().ifEmpty(null)) - - ch_bam_applybqsr = ch_splitncigar_bam_bai.join(ch_bqsr_table, by: [0]) - ch_bam_recalibrated_qc = Channel.empty() - - ch_interval_list_applybqsr = ch_interval_list.map{ meta, bed -> [bed] }.flatten() - ch_bam_applybqsr.combine(ch_interval_list_applybqsr) - .map{ meta, bam, bai, table, interval -> [ meta, bam, bai, table, interval]} - .set{ch_applybqsr_bam_bai_interval} - - // - // MODULE: ApplyBaseRecalibrator from GATK4 - // Recalibrates the base qualities of the input reads based on the recalibration table produced by the GATK BaseRecalibrator tool. - // - RECALIBRATE( - params.skip_multiqc, - ch_applybqsr_bam_bai_interval, - PREPARE_GENOME.out.dict, - PREPARE_GENOME.out.fai, - PREPARE_GENOME.out.fasta - ) - - ch_bam_variant_calling = RECALIBRATE.out.bam - ch_bam_recalibrated_qc = RECALIBRATE.out.qc - - // Gather QC reports - ch_reports = ch_reports.mix(RECALIBRATE.out.qc.collect{it[1]}.ifEmpty([])) - ch_versions = ch_versions.mix(RECALIBRATE.out.versions.first().ifEmpty(null)) - } else { - ch_bam_variant_calling = ch_splitncigar_bam_bai - } - - interval_flag = params.no_intervals - // Run haplotyper even in the absence of dbSNP files - if (!params.dbsnp){ - ch_dbsnp = [] - ch_dbsnp_tbi = [] - } - - ch_haplotypecaller_vcf = Channel.empty() - ch_haplotypecaller_interval_bam = ch_bam_variant_calling.combine(ch_interval_list_split) - .map{ meta, bam, bai, interval_list -> - new_meta = meta.clone() - new_meta.id = meta.id + "_" + interval_list.baseName - new_meta.sample = meta.id - [new_meta, bam, bai, interval_list] - } - - // - // MODULE: HaplotypeCaller from GATK4 - // Calls germline SNPs and indels via local re-assembly of haplotypes. - // - - GATK4_HAPLOTYPECALLER( - ch_haplotypecaller_interval_bam, - PREPARE_GENOME.out.fasta, - PREPARE_GENOME.out.fai, - PREPARE_GENOME.out.dict, - ch_dbsnp, - ch_dbsnp_tbi - ) - - - ch_haplotypecaller_raw = GATK4_HAPLOTYPECALLER.out.vcf - .map{ meta, vcf -> - meta.id = meta.sample - [meta, vcf]} - .groupTuple() - - ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLER.out.versions.first().ifEmpty(null)) - - // - // MODULE: MergeVCFS from GATK4 - // Merge multiple VCF files into one VCF - // - GATK4_MERGEVCFS(ch_haplotypecaller_raw, PREPARE_GENOME.out.dict) - ch_haplotypecaller_vcf = GATK4_MERGEVCFS.out.vcf - ch_versions = ch_versions.mix(GATK4_MERGEVCFS.out.versions.first().ifEmpty(null)) - - if (params.generate_gvcf){ - GATK4_HAPLOTYPECALLERGVCF( - ch_haplotypecaller_interval_bam, - PREPARE_GENOME.out.fasta, - PREPARE_GENOME.out.fai, - PREPARE_GENOME.out.dict, - ch_dbsnp, - ch_dbsnp_tbi - ) - - ch_haplotypecallergvcf_raw = GATK4_HAPLOTYPECALLERGVCF.out.vcf - .map{ meta, vcf -> - meta.id = meta.sample - [meta, vcf]} - .groupTuple() - - ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLERGVCF.out.versions.first().ifEmpty(null)) - // - // MODULE: IndexFeatureFile from GATK4 - // Index the gVCF files - // - GATK4_INDEXFEATUREFILE(GATK4_HAPLOTYPECALLERGVCF.out.vcf) - - ch_haplotypecallergvcf_raw_index = GATK4_INDEXFEATUREFILE.out.index - .map{ meta, idx -> - meta.id = meta.sample - [meta, idx]} - .groupTuple() - - ch_versions = ch_versions.mix(GATK4_INDEXFEATUREFILE.out.versions.first().ifEmpty(null)) - - // - // MODULE: CombineGVCFS from GATK4 - // Merge multiple GVCF files into one GVCF - // - - //ch_haplotypecallergvcf_raw_tbi = ch_haplotypecallergvcf_raw - // .join(ch_haplotypecallergvcf_raw_index, by: [0], remainder: true) - // .map{meta, vcf, tbi -> - // [meta, vcf, tbi] - // } - - - - GATK4_COMBINEGVCFS( - ch_haplotypecallergvcf_raw, - ch_haplotypecallergvcf_raw_index, - PREPARE_GENOME.out.fasta, - PREPARE_GENOME.out.fai, - PREPARE_GENOME.out.dict - ) - ch_haplotypecaller_gvcf = GATK4_COMBINEGVCFS.out.combined_gvcf - ch_versions = ch_versions.mix(GATK4_COMBINEGVCFS.out.versions.first().ifEmpty(null)) - - // - // MODULE: Index the VCF using TABIX - // - TABIXGVCF(ch_haplotypecaller_gvcf) - - ch_haplotypecaller_gvcf_tbi = ch_haplotypecaller_gvcf - .join(TABIXGVCF.out.tbi, by: [0], remainder: true) - .join(TABIXGVCF.out.csi, by: [0], remainder: true) - .map{meta, vcf, tbi, csi -> - if (tbi) [meta, vcf, tbi] - else [meta, vcf, csi] - } - - ch_versions = ch_versions.mix(TABIXGVCF.out.versions.first().ifEmpty(null)) - - } - - // - // MODULE: Index the VCF using TABIX - // - TABIX(ch_haplotypecaller_vcf) - - ch_haplotypecaller_vcf_tbi = ch_haplotypecaller_vcf - .join(TABIX.out.tbi, by: [0], remainder: true) - .join(TABIX.out.csi, by: [0], remainder: true) - .map{meta, vcf, tbi, csi -> - if (tbi) [meta, vcf, tbi] - else [meta, vcf, csi] - } - - ch_versions = ch_versions.mix(TABIX.out.versions.first().ifEmpty(null)) - ch_final_vcf = ch_haplotypecaller_vcf - - // - // MODULE: VariantFiltration from GATK4 - // Filter variant calls based on certain criteria - // - if (!params.skip_variantfiltration && !params.bam_csi_index ) { - - GATK4_VARIANTFILTRATION( - ch_haplotypecaller_vcf_tbi, - PREPARE_GENOME.out.fasta, - PREPARE_GENOME.out.fai, - PREPARE_GENOME.out.dict - ) - - ch_filtered_vcf = GATK4_VARIANTFILTRATION.out.vcf - ch_final_vcf = ch_filtered_vcf - ch_versions = ch_versions.mix(GATK4_VARIANTFILTRATION.out.versions.first().ifEmpty(null)) - } - - // - // SUBWORKFLOW: Annotate variants using snpEff and Ensembl VEP if enabled. - // - if ((!params.skip_variantannotation) &&(params.annotate_tools) &&(params.annotate_tools.contains('merge') || params.annotate_tools.contains('snpeff') || params.annotate_tools.contains('vep'))) { - ANNOTATE( - ch_final_vcf, - params.annotate_tools, - ch_snpeff_db, - ch_snpeff_cache, - ch_vep_genome, - ch_vep_species, - ch_vep_cache_version, - ch_vep_cache) - - // Gather QC reports - ch_reports = ch_reports.mix(ANNOTATE.out.reports) - ch_versions = ch_versions.mix(ANNOTATE.out.versions.first().ifEmpty(null)) - } - - } - - ch_version_yaml = Channel.empty() - CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile(name: 'collated_versions.yml')) - ch_version_yaml = CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect() - - // - // MODULE: MultiQC - // Present summary of reads, alignment, duplicates, BSQR stats for all samples as well as workflow summary/parameters as single report - // - if (!params.skip_multiqc){ - workflow_summary = WorkflowRnavar.paramsSummaryMultiqc(workflow, summary_params) - ch_workflow_summary = Channel.value(workflow_summary) - - methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) - ch_methods_description = Channel.value(methods_description) - - ch_multiqc_files = Channel.empty() - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) - ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(ch_version_yaml) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(ch_reports.collect()) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_config) - ch_multiqc_files = ch_multiqc_files.mix(ch_rnavar_logo) - - MULTIQC( - ch_multiqc_files.collect(), - ch_multiqc_config.toList(), - ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() - ) - multiqc_report = MULTIQC.out.report.toList() - } + // ch_genome_bed = Channel.from([id:'genome.bed']).combine(PREPARE_GENOME.out.exon_bed) + // ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) + + // // + // // SUBWORKFLOW: Read in samplesheet, validate and stage input files + // // + // INPUT_CHECK(file(params.input)) + // .reads + // .map { + // meta, fastq -> + // def meta_clone = meta.clone() + // meta_clone.id = meta_clone.id.split('_')[0..-2].join('_') + // [ meta_clone, fastq ] + // } + // .groupTuple(by: [0]) + // .branch { + // meta, fastq -> + // single : fastq.size() == 1 + // return [ meta, fastq.flatten() ] + // multiple: fastq.size() > 1 + // return [ meta, fastq.flatten() ] + // } + // .set { ch_fastq } + + // ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) + // // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") + // // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ + // // ! There is currently no tooling to help you write a sample sheet schema + + // // + // // MODULE: Concatenate FastQ files from same sample if required + // // + // CAT_FASTQ(ch_fastq.multiple) + // .reads + // .mix(ch_fastq.single) + // .set { ch_cat_fastq } + + // ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) + + // // + // // MODULE: Generate QC summary using FastQC + // // + // FASTQC(ch_cat_fastq) + // ch_reports = ch_reports.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) + // ch_versions = ch_versions.mix(FASTQC.out.versions.first()) + + // // + // // MODULE: Prepare the interval list from the GTF file using GATK4 BedToIntervalList + // // + // ch_interval_list = Channel.empty() + // GATK4_BEDTOINTERVALLIST(ch_genome_bed, PREPARE_GENOME.out.dict) + // ch_interval_list = GATK4_BEDTOINTERVALLIST.out.interval_list + // ch_versions = ch_versions.mix(GATK4_BEDTOINTERVALLIST.out.versions.first().ifEmpty(null)) + + // // + // // MODULE: Scatter one interval-list into many interval-files using GATK4 IntervalListTools + // // + // ch_interval_list_split = Channel.empty() + // if (!params.skip_intervallisttools) { + // GATK4_INTERVALLISTTOOLS(ch_interval_list) + // ch_interval_list_split = GATK4_INTERVALLISTTOOLS.out.interval_list.map{ meta, bed -> [bed] }.flatten() + // } + // else ch_interval_list_split = ch_interval_list + + // // + // // SUBWORKFLOW: Perform read alignment using STAR aligner + // // + // ch_genome_bam = Channel.empty() + // ch_genome_bam_index = Channel.empty() + // ch_samtools_stats = Channel.empty() + // ch_samtools_flagstat = Channel.empty() + // ch_samtools_idxstats = Channel.empty() + // ch_star_multiqc = Channel.empty() + // ch_aligner_pca_multiqc = Channel.empty() + // ch_aligner_clustering_multiqc = Channel.empty() + + // if (params.aligner == 'star') { + // ALIGN_STAR( + // ch_cat_fastq, + // PREPARE_GENOME.out.star_index, + // PREPARE_GENOME.out.gtf, + // params.star_ignore_sjdbgtf, + // seq_platform, + // seq_center + // ) + // ch_genome_bam = ALIGN_STAR.out.bam + // ch_genome_bam_index = ALIGN_STAR.out.bai + // ch_transcriptome_bam = ALIGN_STAR.out.bam_transcript + + // // Gather QC reports + // ch_reports = ch_reports.mix(ALIGN_STAR.out.stats.collect{it[1]}.ifEmpty([])) + // ch_reports = ch_reports.mix(ALIGN_STAR.out.log_final.collect{it[1]}.ifEmpty([])) + // ch_versions = ch_versions.mix(ALIGN_STAR.out.versions.first().ifEmpty(null)) + + // // + // // SUBWORKFLOW: Mark duplicates with GATK4 + // // + // BAM_MARKDUPLICATES( + // ch_genome_bam, + // PREPARE_GENOME.out.fasta, + // PREPARE_GENOME.out.fai, + // []) + + // ch_genome_bam = BAM_MARKDUPLICATES.out.bam_bai + + // //Gather QC reports + // ch_reports = ch_reports.mix(BAM_MARKDUPLICATES.out.stats.collect{it[1]}.ifEmpty([])) + // ch_reports = ch_reports.mix(BAM_MARKDUPLICATES.out.metrics.collect{it[1]}.ifEmpty([])) + // ch_versions = ch_versions.mix(BAM_MARKDUPLICATES.out.versions.first().ifEmpty(null)) + + // // + // // SUBWORKFLOW: SplitNCigarReads from GATK4 over the intervals + // // Splits reads that contain Ns in their cigar string(e.g. spanning splicing events in RNAseq data). + // // + // ch_splitncigar_bam_bai = Channel.empty() + // SPLITNCIGAR( + // ch_genome_bam, + // PREPARE_GENOME.out.fasta, + // PREPARE_GENOME.out.fai, + // PREPARE_GENOME.out.dict, + // ch_interval_list_split + // ) + // ch_splitncigar_bam_bai = SPLITNCIGAR.out.bam_bai + // ch_versions = ch_versions.mix(SPLITNCIGAR.out.versions.first().ifEmpty(null)) + + // // + // // MODULE: BaseRecalibrator from GATK4 + // // Generates a recalibration table based on various co-variates + // // + // ch_bam_variant_calling = Channel.empty() + // if (!params.skip_baserecalibration) { + // ch_bqsr_table = Channel.empty() + // // known_sites is made by grouping both the dbsnp and the known indels ressources + // // they can either or both be optional + // ch_known_sites = ch_dbsnp.concat(ch_known_indels).collect() + // ch_known_sites_tbi = ch_dbsnp_tbi.concat(ch_known_indels_tbi).collect() + + // ch_interval_list_recalib = ch_interval_list.map{ meta, bed -> [bed] }.flatten() + // ch_splitncigar_bam_bai.combine(ch_interval_list_recalib) + // .map{ meta, bam, bai, interval -> [ meta, bam, bai, interval] + // }.set{ch_splitncigar_bam_bai_interval} + + // GATK4_BASERECALIBRATOR( + // ch_splitncigar_bam_bai_interval, + // PREPARE_GENOME.out.fasta, + // PREPARE_GENOME.out.fai, + // PREPARE_GENOME.out.dict, + // ch_known_sites, + // ch_known_sites_tbi + // ) + // ch_bqsr_table = GATK4_BASERECALIBRATOR.out.table + + // // Gather QC reports + // ch_reports = ch_reports.mix(ch_bqsr_table.map{ meta, table -> table}) + // ch_versions = ch_versions.mix(GATK4_BASERECALIBRATOR.out.versions.first().ifEmpty(null)) + + // ch_bam_applybqsr = ch_splitncigar_bam_bai.join(ch_bqsr_table, by: [0]) + // ch_bam_recalibrated_qc = Channel.empty() + + // ch_interval_list_applybqsr = ch_interval_list.map{ meta, bed -> [bed] }.flatten() + // ch_bam_applybqsr.combine(ch_interval_list_applybqsr) + // .map{ meta, bam, bai, table, interval -> [ meta, bam, bai, table, interval]} + // .set{ch_applybqsr_bam_bai_interval} + + // // + // // MODULE: ApplyBaseRecalibrator from GATK4 + // // Recalibrates the base qualities of the input reads based on the recalibration table produced by the GATK BaseRecalibrator tool. + // // + // RECALIBRATE( + // params.skip_multiqc, + // ch_applybqsr_bam_bai_interval, + // PREPARE_GENOME.out.dict, + // PREPARE_GENOME.out.fai, + // PREPARE_GENOME.out.fasta + // ) + + // ch_bam_variant_calling = RECALIBRATE.out.bam + // ch_bam_recalibrated_qc = RECALIBRATE.out.qc + + // // Gather QC reports + // ch_reports = ch_reports.mix(RECALIBRATE.out.qc.collect{it[1]}.ifEmpty([])) + // ch_versions = ch_versions.mix(RECALIBRATE.out.versions.first().ifEmpty(null)) + // } else { + // ch_bam_variant_calling = ch_splitncigar_bam_bai + // } + + // interval_flag = params.no_intervals + // // Run haplotyper even in the absence of dbSNP files + // if (!params.dbsnp){ + // ch_dbsnp = [] + // ch_dbsnp_tbi = [] + // } + + // ch_haplotypecaller_vcf = Channel.empty() + // ch_haplotypecaller_interval_bam = ch_bam_variant_calling.combine(ch_interval_list_split) + // .map{ meta, bam, bai, interval_list -> + // new_meta = meta.clone() + // new_meta.id = meta.id + "_" + interval_list.baseName + // new_meta.sample = meta.id + // [new_meta, bam, bai, interval_list] + // } + + // // + // // MODULE: HaplotypeCaller from GATK4 + // // Calls germline SNPs and indels via local re-assembly of haplotypes. + // // + + // GATK4_HAPLOTYPECALLER( + // ch_haplotypecaller_interval_bam, + // PREPARE_GENOME.out.fasta, + // PREPARE_GENOME.out.fai, + // PREPARE_GENOME.out.dict, + // ch_dbsnp, + // ch_dbsnp_tbi + // ) + + + // ch_haplotypecaller_raw = GATK4_HAPLOTYPECALLER.out.vcf + // .map{ meta, vcf -> + // meta.id = meta.sample + // [meta, vcf]} + // .groupTuple() + + // ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLER.out.versions.first().ifEmpty(null)) + + // // + // // MODULE: MergeVCFS from GATK4 + // // Merge multiple VCF files into one VCF + // // + // GATK4_MERGEVCFS(ch_haplotypecaller_raw, PREPARE_GENOME.out.dict) + // ch_haplotypecaller_vcf = GATK4_MERGEVCFS.out.vcf + // ch_versions = ch_versions.mix(GATK4_MERGEVCFS.out.versions.first().ifEmpty(null)) + + // if (params.generate_gvcf){ + // GATK4_HAPLOTYPECALLERGVCF( + // ch_haplotypecaller_interval_bam, + // PREPARE_GENOME.out.fasta, + // PREPARE_GENOME.out.fai, + // PREPARE_GENOME.out.dict, + // ch_dbsnp, + // ch_dbsnp_tbi + // ) + + // ch_haplotypecallergvcf_raw = GATK4_HAPLOTYPECALLERGVCF.out.vcf + // .map{ meta, vcf -> + // meta.id = meta.sample + // [meta, vcf]} + // .groupTuple() + + // ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLERGVCF.out.versions.first().ifEmpty(null)) + // // + // // MODULE: IndexFeatureFile from GATK4 + // // Index the gVCF files + // // + // GATK4_INDEXFEATUREFILE(GATK4_HAPLOTYPECALLERGVCF.out.vcf) + + // ch_haplotypecallergvcf_raw_index = GATK4_INDEXFEATUREFILE.out.index + // .map{ meta, idx -> + // meta.id = meta.sample + // [meta, idx]} + // .groupTuple() + + // ch_versions = ch_versions.mix(GATK4_INDEXFEATUREFILE.out.versions.first().ifEmpty(null)) + + // // + // // MODULE: CombineGVCFS from GATK4 + // // Merge multiple GVCF files into one GVCF + // // + + // //ch_haplotypecallergvcf_raw_tbi = ch_haplotypecallergvcf_raw + // // .join(ch_haplotypecallergvcf_raw_index, by: [0], remainder: true) + // // .map{meta, vcf, tbi -> + // // [meta, vcf, tbi] + // // } + + + + // GATK4_COMBINEGVCFS( + // ch_haplotypecallergvcf_raw, + // ch_haplotypecallergvcf_raw_index, + // PREPARE_GENOME.out.fasta, + // PREPARE_GENOME.out.fai, + // PREPARE_GENOME.out.dict + // ) + // ch_haplotypecaller_gvcf = GATK4_COMBINEGVCFS.out.combined_gvcf + // ch_versions = ch_versions.mix(GATK4_COMBINEGVCFS.out.versions.first().ifEmpty(null)) + + // // + // // MODULE: Index the VCF using TABIX + // // + // TABIXGVCF(ch_haplotypecaller_gvcf) + + // ch_haplotypecaller_gvcf_tbi = ch_haplotypecaller_gvcf + // .join(TABIXGVCF.out.tbi, by: [0], remainder: true) + // .join(TABIXGVCF.out.csi, by: [0], remainder: true) + // .map{meta, vcf, tbi, csi -> + // if (tbi) [meta, vcf, tbi] + // else [meta, vcf, csi] + // } + + // ch_versions = ch_versions.mix(TABIXGVCF.out.versions.first().ifEmpty(null)) + + // } + + // // + // // MODULE: Index the VCF using TABIX + // // + // TABIX(ch_haplotypecaller_vcf) + + // ch_haplotypecaller_vcf_tbi = ch_haplotypecaller_vcf + // .join(TABIX.out.tbi, by: [0], remainder: true) + // .join(TABIX.out.csi, by: [0], remainder: true) + // .map{meta, vcf, tbi, csi -> + // if (tbi) [meta, vcf, tbi] + // else [meta, vcf, csi] + // } + + // ch_versions = ch_versions.mix(TABIX.out.versions.first().ifEmpty(null)) + // ch_final_vcf = ch_haplotypecaller_vcf + + // // + // // MODULE: VariantFiltration from GATK4 + // // Filter variant calls based on certain criteria + // // + // if (!params.skip_variantfiltration && !params.bam_csi_index ) { + + // GATK4_VARIANTFILTRATION( + // ch_haplotypecaller_vcf_tbi, + // PREPARE_GENOME.out.fasta, + // PREPARE_GENOME.out.fai, + // PREPARE_GENOME.out.dict + // ) + + // ch_filtered_vcf = GATK4_VARIANTFILTRATION.out.vcf + // ch_final_vcf = ch_filtered_vcf + // ch_versions = ch_versions.mix(GATK4_VARIANTFILTRATION.out.versions.first().ifEmpty(null)) + // } + + // // + // // SUBWORKFLOW: Annotate variants using snpEff and Ensembl VEP if enabled. + // // + // if ((!params.skip_variantannotation) &&(params.annotate_tools) &&(params.annotate_tools.contains('merge') || params.annotate_tools.contains('snpeff') || params.annotate_tools.contains('vep'))) { + // ANNOTATE( + // ch_final_vcf, + // params.annotate_tools, + // ch_snpeff_db, + // ch_snpeff_cache, + // ch_vep_genome, + // ch_vep_species, + // ch_vep_cache_version, + // ch_vep_cache) + + // // Gather QC reports + // ch_reports = ch_reports.mix(ANNOTATE.out.reports) + // ch_versions = ch_versions.mix(ANNOTATE.out.versions.first().ifEmpty(null)) + // } + + // } + + // ch_version_yaml = Channel.empty() + // CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile(name: 'collated_versions.yml')) + // ch_version_yaml = CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect() + + // // + // // MODULE: MultiQC + // // Present summary of reads, alignment, duplicates, BSQR stats for all samples as well as workflow summary/parameters as single report + // // + // if (!params.skip_multiqc){ + // workflow_summary = WorkflowRnavar.paramsSummaryMultiqc(workflow, summary_params) + // ch_workflow_summary = Channel.value(workflow_summary) + + // methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) + // ch_methods_description = Channel.value(methods_description) + + // ch_multiqc_files = Channel.empty() + // ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + // ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) + // ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) + // ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) + // ch_multiqc_files = ch_multiqc_files.mix(ch_version_yaml) + // ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) + // ch_multiqc_files = ch_multiqc_files.mix(ch_reports.collect()) + // ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_config) + // ch_multiqc_files = ch_multiqc_files.mix(ch_rnavar_logo) + + // MULTIQC( + // ch_multiqc_files.collect(), + // ch_multiqc_config.toList(), + // ch_multiqc_custom_config.toList(), + // ch_multiqc_logo.toList() + // ) + // multiqc_report = MULTIQC.out.report.toList() + // } } /* From b3427dfc391c3040c3fbda1baa8e4d999975890f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Tue, 19 Sep 2023 16:45:57 +0200 Subject: [PATCH 17/34] improve prepare_genome subworkflow --- modules/local/gtf2bed/main.nf | 3 +- subworkflows/local/prepare_genome/main.nf | 131 ++++------------------ workflows/rnavar.nf | 18 +-- 3 files changed, 35 insertions(+), 117 deletions(-) diff --git a/modules/local/gtf2bed/main.nf b/modules/local/gtf2bed/main.nf index b0c7e42a..b18a63d4 100755 --- a/modules/local/gtf2bed/main.nf +++ b/modules/local/gtf2bed/main.nf @@ -8,8 +8,9 @@ process GTF2BED { 'biocontainers/r-base:3.5.0'}" input: - path gtf + tuple val(meta), path(gtf) val feature_type + output: path '*.bed' , emit: bed path "versions.yml", emit: versions diff --git a/subworkflows/local/prepare_genome/main.nf b/subworkflows/local/prepare_genome/main.nf index 598dc4e4..d3cde79e 100755 --- a/subworkflows/local/prepare_genome/main.nf +++ b/subworkflows/local/prepare_genome/main.nf @@ -1,5 +1,5 @@ // -// Uncompress and prepare reference genome files +// Prepare reference genome files // include { BEDTOOLS_MERGE } from '../../../modules/nf-core/bedtools/merge/main' @@ -7,130 +7,43 @@ include { BEDTOOLS_SORT } from '../../../modules/nf-core/bedtoo include { GATK4_CREATESEQUENCEDICTIONARY } from '../../../modules/nf-core/gatk4/createsequencedictionary/main' include { GFFREAD } from '../../../modules/nf-core/gffread/main' include { GTF2BED } from '../../../modules/local/gtf2bed' -include { GUNZIP as GUNZIP_FASTA } from '../../../modules/nf-core/gunzip/main' -include { GUNZIP as GUNZIP_GENE_BED } from '../../../modules/nf-core/gunzip/main' -include { GUNZIP as GUNZIP_GFF } from '../../../modules/nf-core/gunzip/main' -include { GUNZIP as GUNZIP_GTF } from '../../../modules/nf-core/gunzip/main' include { SAMTOOLS_FAIDX } from '../../../modules/nf-core/samtools/faidx/main' include { STAR_GENOMEGENERATE } from '../../../modules/nf-core/star/genomegenerate/main' -include { UNTAR as UNTAR_STAR_INDEX } from '../../../modules/nf-core/untar/main' workflow PREPARE_GENOME { take: - fasta // file: /path/to/genome.fasta - // gtf // file: /path/to/genome.gtf - // gff // file: /path/to/genome.gff - // exon_bed // file: /path/to/gene.bed + ch_exon_bed // file: /path/to/gene.bed + ch_fasta // file: /path/to/genome.fasta + ch_gff // file: /path/to/genome.gff + ch_gtf // file: /path/to/genome.gtf + feature_type // prepare_tool_indices - // feature_type main: ch_versions = Channel.empty() - ch_fasta = fasta.map{ fasta -> [ [ id:fasta.baseName ], fasta ] } - - // - // Uncompress genome fasta file if required - // - // if (fasta.endsWith('.gz')) { - // ch_fasta = GUNZIP_FASTA([[:], fasta]).gunzip.map{ meta, fasta -> fasta } - // ch_versions = ch_versions.mix(GUNZIP_FASTA.out.versions) - // } else { - // ch_fasta = Channel.value(file(fasta)) - // } - - // - // Uncompress GTF annotation file or create from GFF3 if required - // - // if (gtf) { - // if (gtf.endsWith('.gz')) { - // ch_gtf = GUNZIP_GTF([[:], gtf]).gunzip.map{ meta, gtf -> gtf } - // ch_versions = ch_versions.mix(GUNZIP_GTF.out.versions) - // } else ch_gtf = Channel.value(file(gtf)) - // } else if (gff) { - // if (gff.endsWith('.gz')) { - // ch_gff = GUNZIP_GFF([[:], gff]).gunzip.map{ meta, gff -> gff } - // ch_versions = ch_versions.mix(GUNZIP_GFF.out.versions) - // } else ch_gff = Channel.value(file(gff)) - // ch_gtf = GFFREAD(ch_gff).gtf - // ch_versions = ch_versions.mix(GFFREAD.out.versions) - // } - - // - // Uncompress exon BED annotation file or create from GTF if required - // - // if (exon_bed) { - // if (exon_bed.endsWith('.gz')) { - // exonGENE_BED( - // Channel.fromPath(exon_bed).map{ it -> [[id:it[0].baseName], it] } - // ) - // ch_exon_bed = GUNZIP_GENE_BED.out.gunzip.map{ meta, bed -> [bed] }.collect() - // ch_versions = ch_versions.mix(GUNZIP_GENE_BED.out.versions) - // } else { - // ch_exon_bed = Channel.fromPath(exon_bed).collect() - // } - // } else { - // ch_exon_bed = GTF2BED( ch_gtf , feature_type).bed.collect() - // ch_versions = ch_versions.mix(GTF2BED.out.versions) - // } - - //ch_exon_bed.view() - //ch_exon_bed.map{ it -> [[id:'exome'], it] } - //ch_exon_bed.view() - // Bedtools sort - // ch_bedtools_sort = BEDTOOLS_SORT(ch_exon_bed.map{ it -> [[id:'exome'], it] }, 'sorted').sorted.collect() - // ch_versions = ch_versions.mix(BEDTOOLS_SORT.out.versions) - - - // // Bedtools merge - // ch_bedtools_merge = BEDTOOLS_MERGE(ch_bedtools_sort).bed - // ch_versions = ch_versions.mix(BEDTOOLS_MERGE.out.versions) - - - // Index the genome fasta + GATK4_CREATESEQUENCEDICTIONARY(ch_fasta) + GFFREAD(ch_gff) SAMTOOLS_FAIDX(ch_fasta, [['id':null], []]) - ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) + ch_gtf = ch_gtf.mix(GFFREAD.out.gtf) - // Create dictionary file for the genome fasta - // ch_fasta_dict = Channel.empty() - // if (params.dict) ch_fasta_dict = Channel.fromPath(params.dict).collect() - // else ch_fasta_dict = GATK4_CREATESEQUENCEDICTIONARY(ch_fasta).dict - - // - // Uncompress STAR index or generate from scratch if required - // - // ch_star_index = Channel.empty() - // if ('star' in prepare_tool_indices) { - // if (params.star_index) { - // if (params.star_index.endsWith('.tar.gz')) { - // UNTAR_STAR_INDEX( - // Channel.fromPath(params.star_index).map{ it -> [[id:it[0].baseName], it] } - // ) - // ch_star_index = UNTAR_STAR_INDEX.out.untar.map{ meta, star_index -> [star_index] }.collect() - // ch_versions = ch_versions.mix(UNTAR_STAR_INDEX.out.versions) - // } else { - // ch_star_index = Channel.fromPath(params.star_index).collect() - // } - // } - // else { - // STAR_GENOMEGENERATE( - // ch_fasta,ch_gtf - // ) - // .index - // .set { ch_star_index } - // ch_versions = ch_versions.mix(STAR_GENOMEGENERATE.out.versions) - // } - // } + GTF2BED(ch_gtf, feature_type) + STAR_GENOMEGENERATE(ch_fasta, ch_gtf) + ch_versions = ch_versions.mix(GATK4_CREATESEQUENCEDICTIONARY.out.versions) + ch_versions = ch_versions.mix(GFFREAD.out.versions) + ch_versions = ch_versions.mix(GTF2BED.out.versions) + ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) + ch_versions = ch_versions.mix(STAR_GENOMEGENERATE.out.versions) emit: - // fasta = ch_fasta // path: genome.fasta - fasta_fai = SAMTOOLS_FAIDX.out.fai.map{ meta, fai -> [fai] } // path: genome.fasta.fai dict = ch_fasta_dict // path: genome.fasta.dict - // gtf = ch_gtf // path: genome.gtf - // exon_bed = ch_exon_bed // path: exon.bed + dict = GATK4_CREATESEQUENCEDICTIONARY.out.dict // path: genome.fasta.dict + exon_bed = GTF2BED.out.bed.collect() // path: exon.bed + fasta_fai = SAMTOOLS_FAIDX.out.fai.map{ meta, fai -> [fai] } // path: genome.fasta.fai + gtf = ch_gtf // path: genome.gtf + star_index = STAR_GENOMEGENERATE.out.index // path: star/index/ + versions = ch_versions // channel: [ versions.yml ] // bedtools_sort = ch_bedtools_sort // path: sort.bed // bedtools_merge = ch_bedtools_merge // path: merge.bed - // star_index = ch_star_index // path: star/index/ - versions = ch_versions // channel: [ versions.yml ] } diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 64264b44..b2796ae2 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -117,7 +117,10 @@ include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/ // Initialize file channels based on params, defined in the params.genomes[params.genome] scope -ch_fasta = params.fasta ? Channel.fromPath(params.fasta).first() : Channel.empty() +ch_exon_bed = params.exon_bed ? Channel.fromPath(params.exon_bed) : Channel.empty() +ch_fasta = params.fasta ? Channel.fromPath(params.fasta).map{ fasta -> [ [ id:fasta.baseName ], fasta ] }.first() : Channel.empty() +ch_gff = params.gff ? Channel.fromPath(params.gff).first() : Channel.empty() +ch_gtf = params.gtf ? Channel.fromPath(params.gtf).map{ gtf -> [ [ id:gtf.baseName ], gtf ] }.first() : Channel.empty() /* ======================================================================================== @@ -129,20 +132,21 @@ workflow RNAVAR { // To gather all QC reports for MultiQC ch_reports = Channel.empty() + // To gather used softwares versions for MultiQC ch_versions = Channel.empty() // - // SUBWORKFLOW: Uncompress and prepare reference genome files + // Prepare reference genome files // PREPARE_GENOME( - ch_fasta - // params.gtf, - // params.gff, - // params.gene_bed, + ch_exon_bed, + ch_fasta, + ch_gff, + ch_gtf, // params.aligner, - // params.feature_type + params.feature_type ) // ch_genome_bed = Channel.from([id:'genome.bed']).combine(PREPARE_GENOME.out.exon_bed) From 3663ecbff77e440f39d39c45e0a63e20b30294c5 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 11:45:11 +0200 Subject: [PATCH 18/34] use samplesheet from nf-validation --- assets/schema_input.json | 27 +++++++--- conf/modules.config | 8 --- nextflow_schema.json | 9 ++-- subworkflows/local/prepare_genome/main.nf | 1 - workflows/rnavar.nf | 66 +++++++++-------------- 5 files changed, 50 insertions(+), 61 deletions(-) diff --git a/assets/schema_input.json b/assets/schema_input.json index 6fd501da..dc7fcba6 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -10,15 +10,27 @@ "sample": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample name must be provided and cannot contain spaces" + "errorMessage": "Sample ID must be provided and cannot contain spaces", + "meta": ["sample"] }, "fastq_1": { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$", - "errorMessage": "FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" + "errorMessage": "FastQ file for reads 1 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.f(ast)?q\\.gz$" + }, + { + "type": "string", + "maxLength": 0 + } + ], + "format": "file-path", + "exists": true }, "fastq_2": { "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", + "dependentRequired": ["fastq_1"], "anyOf": [ { "type": "string", @@ -28,12 +40,15 @@ "type": "string", "maxLength": 0 } - ] + ], + "format": "file-path", + "exists": true }, "strandedness": { "type": "string", "errorMessage": "Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded'", - "enum": ["forward", "reverse", "unstranded"] + "enum": ["forward", "reverse", "unstranded"], + "meta": ["strandedness"] } }, "required": ["sample", "fastq_1", "strandedness"] diff --git a/conf/modules.config b/conf/modules.config index f05b3322..d5e4f84e 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -18,14 +18,6 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] - withName: 'NFCORE_RNAVAR:RNAVAR:INPUT_CHECK:SAMPLESHEET_CHECK' { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: CUSTOM_DUMPSOFTWAREVERSIONS { publishDir = [ path: { "${params.outdir}/pipeline_info" }, diff --git a/nextflow_schema.json b/nextflow_schema.json index 7fb6d183..7ce1ee3d 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -13,14 +13,15 @@ "required": ["input", "outdir"], "properties": { "input": { + "description": "Path to comma-separated file containing information about the samples in the experiment.", + "help_text": "A design file with information about the samples in your experiment. Use this parameter to specify the location of the input files. It has to be a comma-separated file with a header row. See [usage docs](https://nf-co.re/sarek/usage#input).\n\nIf no input file is specified, sarek will attempt to locate one in the `{outdir}` directory. If no input should be supplied, i.e. when --step is supplied or --build_from_index, then set --input false", + "fa_icon": "fas fa-file-csv", + "schema": "assets/schema_input.json", "type": "string", "format": "file-path", "exists": true, "mimetype": "text/csv", - "pattern": "^\\S+\\.csv$", - "description": "Path to comma-separated file containing information about the samples in the experiment.", - "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/rnavar/usage#samplesheet-input).", - "fa_icon": "fas fa-file-csv" + "pattern": "^\\S+\\.csv$" }, "outdir": { "type": "string", diff --git a/subworkflows/local/prepare_genome/main.nf b/subworkflows/local/prepare_genome/main.nf index d3cde79e..d20d8fe6 100755 --- a/subworkflows/local/prepare_genome/main.nf +++ b/subworkflows/local/prepare_genome/main.nf @@ -17,7 +17,6 @@ workflow PREPARE_GENOME { ch_gff // file: /path/to/genome.gff ch_gtf // file: /path/to/genome.gtf feature_type - // prepare_tool_indices main: ch_versions = Channel.empty() diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index b2796ae2..94c2cff8 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryLog; paramsSummaryMap; fromSamplesheet } from 'plugin/nf-validation' def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) def citation = '\n' + WorkflowMain.citation(workflow) + '\n' @@ -116,6 +116,7 @@ include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/ // def multiqc_report = [] + // Initialize file channels based on params, defined in the params.genomes[params.genome] scope ch_exon_bed = params.exon_bed ? Channel.fromPath(params.exon_bed) : Channel.empty() ch_fasta = params.fasta ? Channel.fromPath(params.fasta).map{ fasta -> [ [ id:fasta.baseName ], fasta ] }.first() : Channel.empty() @@ -129,71 +130,52 @@ ch_gtf = params.gtf ? Channel.fromPath(params.gtf).map{ gtf -> [ [ id: */ workflow RNAVAR { - // To gather all QC reports for MultiQC ch_reports = Channel.empty() // To gather used softwares versions for MultiQC ch_versions = Channel.empty() - // + ch_from_samplesheet = Channel.empty() + + if (params.input) ch_from_samplesheet = Channel.fromSamplesheet("input") + + ch_fastq = ch_from_samplesheet.map{ meta, fastq_1, fastq_2 -> + if (fastq_2) return [ meta + [id: meta.sample], [ fastq_1, fastq_2 ] ] + else return [ meta + [id: meta.sample], [ fastq_1 ] ] + }.groupTuple() + .branch { meta, fastq -> + single : fastq.size() == 1 + return [ meta, fastq.flatten() ] + multiple: fastq.size() > 1 + return [ meta, fastq.flatten() ] + } + // Prepare reference genome files - // PREPARE_GENOME( ch_exon_bed, ch_fasta, ch_gff, ch_gtf, - // params.aligner, params.feature_type ) // ch_genome_bed = Channel.from([id:'genome.bed']).combine(PREPARE_GENOME.out.exon_bed) // ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) - // // - // // SUBWORKFLOW: Read in samplesheet, validate and stage input files - // // - // INPUT_CHECK(file(params.input)) - // .reads - // .map { - // meta, fastq -> - // def meta_clone = meta.clone() - // meta_clone.id = meta_clone.id.split('_')[0..-2].join('_') - // [ meta_clone, fastq ] - // } - // .groupTuple(by: [0]) - // .branch { - // meta, fastq -> - // single : fastq.size() == 1 - // return [ meta, fastq.flatten() ] - // multiple: fastq.size() > 1 - // return [ meta, fastq.flatten() ] - // } - // .set { ch_fastq } + // // MODULE: Concatenate FastQ files from same sample if required - // ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) - // // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") - // // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ - // // ! There is currently no tooling to help you write a sample sheet schema + CAT_FASTQ(ch_fastq.multiple) - // // - // // MODULE: Concatenate FastQ files from same sample if required - // // - // CAT_FASTQ(ch_fastq.multiple) - // .reads - // .mix(ch_fastq.single) - // .set { ch_cat_fastq } + ch_cat_fastq = CAT_FASTQ.out.reads.mix(ch_fastq.single) - // ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) - // // // // MODULE: Generate QC summary using FastQC - // // - // FASTQC(ch_cat_fastq) - // ch_reports = ch_reports.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) - // ch_versions = ch_versions.mix(FASTQC.out.versions.first()) + FASTQC(ch_cat_fastq) + ch_reports = ch_reports.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) + ch_versions = ch_versions.mix(FASTQC.out.versions.first()) // // // // MODULE: Prepare the interval list from the GTF file using GATK4 BedToIntervalList From 4e44c19ba915d4f0f11cb19aeee165d389a14ea1 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 11:59:35 +0200 Subject: [PATCH 19/34] one more step --- subworkflows/local/prepare_genome/main.nf | 13 ++++++------- workflows/rnavar.nf | 18 +++++++++++------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/subworkflows/local/prepare_genome/main.nf b/subworkflows/local/prepare_genome/main.nf index d20d8fe6..efb649ae 100755 --- a/subworkflows/local/prepare_genome/main.nf +++ b/subworkflows/local/prepare_genome/main.nf @@ -12,7 +12,6 @@ include { STAR_GENOMEGENERATE } from '../../../modules/nf-core/star/g workflow PREPARE_GENOME { take: - ch_exon_bed // file: /path/to/gene.bed ch_fasta // file: /path/to/genome.fasta ch_gff // file: /path/to/genome.gff ch_gtf // file: /path/to/genome.gtf @@ -37,12 +36,12 @@ workflow PREPARE_GENOME { ch_versions = ch_versions.mix(STAR_GENOMEGENERATE.out.versions) emit: - dict = GATK4_CREATESEQUENCEDICTIONARY.out.dict // path: genome.fasta.dict - exon_bed = GTF2BED.out.bed.collect() // path: exon.bed - fasta_fai = SAMTOOLS_FAIDX.out.fai.map{ meta, fai -> [fai] } // path: genome.fasta.fai - gtf = ch_gtf // path: genome.gtf - star_index = STAR_GENOMEGENERATE.out.index // path: star/index/ - versions = ch_versions // channel: [ versions.yml ] + dict = GATK4_CREATESEQUENCEDICTIONARY.out.dict // path: genome.fasta.dict + exon_bed = GTF2BED.out.bed.map{ bed -> [ [ id:bed.baseName ], bed ] }.collect() // path: exon.bed + fasta_fai = SAMTOOLS_FAIDX.out.fai.map{ meta, fai -> [fai] } // path: genome.fasta.fai + gtf = ch_gtf // path: genome.gtf + star_index = STAR_GENOMEGENERATE.out.index // path: star/index/ + versions = ch_versions // channel: [ versions.yml ] // bedtools_sort = ch_bedtools_sort // path: sort.bed // bedtools_merge = ch_bedtools_merge // path: merge.bed } diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 94c2cff8..ab429c34 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -154,15 +154,19 @@ workflow RNAVAR { // Prepare reference genome files PREPARE_GENOME( - ch_exon_bed, ch_fasta, ch_gff, ch_gtf, params.feature_type ) + ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) - // ch_genome_bed = Channel.from([id:'genome.bed']).combine(PREPARE_GENOME.out.exon_bed) - // ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) + ch_genome_bed = params.exon_bed ? Channel.fromPath(params.exon_bed).map{ it -> [ [id:'exon_bed'], it ] }.collect() + : PREPARE_GENOME.out.exon_bed + ch_dict = params.dict ? Channel.fromPath(params.dict).map{ it -> [ [id:'dict'], it ] }.collect() + : PREPARE_GENOME.out.dict + ch_fasta_fai = params.fasta_fai ? Channel.fromPath(params.fasta_fai).collect() + : PREPARE_GENOME.out.fasta_fai // // MODULE: Concatenate FastQ files from same sample if required @@ -180,10 +184,10 @@ workflow RNAVAR { // // // // MODULE: Prepare the interval list from the GTF file using GATK4 BedToIntervalList // // - // ch_interval_list = Channel.empty() - // GATK4_BEDTOINTERVALLIST(ch_genome_bed, PREPARE_GENOME.out.dict) - // ch_interval_list = GATK4_BEDTOINTERVALLIST.out.interval_list - // ch_versions = ch_versions.mix(GATK4_BEDTOINTERVALLIST.out.versions.first().ifEmpty(null)) + + GATK4_BEDTOINTERVALLIST(ch_genome_bed, ch_dict) + ch_interval_list = GATK4_BEDTOINTERVALLIST.out.interval_list + ch_versions = ch_versions.mix(GATK4_BEDTOINTERVALLIST.out.versions.first().ifEmpty(null)) // // // // MODULE: Scatter one interval-list into many interval-files using GATK4 IntervalListTools From 65b3343a0554e06b617444222d428f3f5a65f219 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 12:23:00 +0200 Subject: [PATCH 20/34] markduplicates --- subworkflows/local/bam_markduplicates/main.nf | 8 +- subworkflows/local/bam_stats_samtools/main.nf | 16 +- workflows/rnavar.nf | 169 +++++++++--------- 3 files changed, 92 insertions(+), 101 deletions(-) diff --git a/subworkflows/local/bam_markduplicates/main.nf b/subworkflows/local/bam_markduplicates/main.nf index 6ee53041..fac00488 100644 --- a/subworkflows/local/bam_markduplicates/main.nf +++ b/subworkflows/local/bam_markduplicates/main.nf @@ -18,22 +18,22 @@ workflow BAM_MARKDUPLICATES { versions = Channel.empty() reports = Channel.empty() - // RUN MARKDUPLICATES + // RUN MARKUPDUPLICATES GATK4_MARKDUPLICATES(bam, fasta, fasta_fai) // Join with the crai file cram = GATK4_MARKDUPLICATES.out.cram.join(GATK4_MARKDUPLICATES.out.crai, failOnDuplicate: true, failOnMismatch: true) // QC on CRAM - CRAM_QC_MOSDEPTH_SAMTOOLS(cram, fasta, intervals_bed_combined) + // CRAM_QC_MOSDEPTH_SAMTOOLS(cram, fasta, intervals_bed_combined) // Gather all reports generated reports = reports.mix(GATK4_MARKDUPLICATES.out.metrics) - reports = reports.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.reports) + // reports = reports.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.reports) // Gather versions of all tools used versions = versions.mix(GATK4_MARKDUPLICATES.out.versions) - versions = versions.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.versions) + // versions = versions.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.versions) emit: cram diff --git a/subworkflows/local/bam_stats_samtools/main.nf b/subworkflows/local/bam_stats_samtools/main.nf index e94c1bd1..94bb2037 100644 --- a/subworkflows/local/bam_stats_samtools/main.nf +++ b/subworkflows/local/bam_stats_samtools/main.nf @@ -13,21 +13,13 @@ workflow BAM_STATS_SAMTOOLS { main: ch_versions = Channel.empty() - SAMTOOLS_STATS ( - ch_bam_bai, - [] - ) - ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) + SAMTOOLS_FLAGSTAT(ch_bam_bai) + SAMTOOLS_IDXSTATS(ch_bam_bai) + SAMTOOLS_STATS(ch_bam_bai, [[],[]]) - SAMTOOLS_FLAGSTAT ( - ch_bam_bai - ) ch_versions = ch_versions.mix(SAMTOOLS_FLAGSTAT.out.versions.first()) - - SAMTOOLS_IDXSTATS ( - ch_bam_bai - ) ch_versions = ch_versions.mix(SAMTOOLS_IDXSTATS.out.versions.first()) + ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) emit: stats = SAMTOOLS_STATS.out.stats // channel: [ val(meta), [ stats ] ] diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index ab429c34..7f84af92 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -95,8 +95,8 @@ include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/ // // Check STAR alignment parameters // def prepareToolIndices = params.aligner -// def seq_platform = params.seq_platform ?: [] -// def seq_center = params.seq_center ?: [] +def seq_platform = params.seq_platform ?: [] +def seq_center = params.seq_center ?: [] // // Initialize file channels based on params // ch_dbsnp = params.dbsnp ? Channel.fromPath(params.dbsnp).collect() : Channel.empty() @@ -164,11 +164,11 @@ workflow RNAVAR { ch_genome_bed = params.exon_bed ? Channel.fromPath(params.exon_bed).map{ it -> [ [id:'exon_bed'], it ] }.collect() : PREPARE_GENOME.out.exon_bed ch_dict = params.dict ? Channel.fromPath(params.dict).map{ it -> [ [id:'dict'], it ] }.collect() - : PREPARE_GENOME.out.dict + : ch_dict ch_fasta_fai = params.fasta_fai ? Channel.fromPath(params.fasta_fai).collect() : PREPARE_GENOME.out.fasta_fai - // // MODULE: Concatenate FastQ files from same sample if required + // MODULE: Concatenate FastQ files from same sample if required CAT_FASTQ(ch_fastq.multiple) @@ -176,74 +176,73 @@ workflow RNAVAR { ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) - // // MODULE: Generate QC summary using FastQC + // MODULE: Generate QC summary using FastQC FASTQC(ch_cat_fastq) ch_reports = ch_reports.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) ch_versions = ch_versions.mix(FASTQC.out.versions.first()) - // // - // // MODULE: Prepare the interval list from the GTF file using GATK4 BedToIntervalList - // // + // + // MODULE: Prepare the interval list from the GTF file using GATK4 BedToIntervalList + // GATK4_BEDTOINTERVALLIST(ch_genome_bed, ch_dict) ch_interval_list = GATK4_BEDTOINTERVALLIST.out.interval_list ch_versions = ch_versions.mix(GATK4_BEDTOINTERVALLIST.out.versions.first().ifEmpty(null)) - // // - // // MODULE: Scatter one interval-list into many interval-files using GATK4 IntervalListTools - // // - // ch_interval_list_split = Channel.empty() - // if (!params.skip_intervallisttools) { - // GATK4_INTERVALLISTTOOLS(ch_interval_list) - // ch_interval_list_split = GATK4_INTERVALLISTTOOLS.out.interval_list.map{ meta, bed -> [bed] }.flatten() - // } - // else ch_interval_list_split = ch_interval_list - - // // - // // SUBWORKFLOW: Perform read alignment using STAR aligner - // // - // ch_genome_bam = Channel.empty() - // ch_genome_bam_index = Channel.empty() - // ch_samtools_stats = Channel.empty() - // ch_samtools_flagstat = Channel.empty() - // ch_samtools_idxstats = Channel.empty() - // ch_star_multiqc = Channel.empty() - // ch_aligner_pca_multiqc = Channel.empty() - // ch_aligner_clustering_multiqc = Channel.empty() - - // if (params.aligner == 'star') { - // ALIGN_STAR( - // ch_cat_fastq, - // PREPARE_GENOME.out.star_index, - // PREPARE_GENOME.out.gtf, - // params.star_ignore_sjdbgtf, - // seq_platform, - // seq_center - // ) - // ch_genome_bam = ALIGN_STAR.out.bam - // ch_genome_bam_index = ALIGN_STAR.out.bai - // ch_transcriptome_bam = ALIGN_STAR.out.bam_transcript - - // // Gather QC reports - // ch_reports = ch_reports.mix(ALIGN_STAR.out.stats.collect{it[1]}.ifEmpty([])) - // ch_reports = ch_reports.mix(ALIGN_STAR.out.log_final.collect{it[1]}.ifEmpty([])) - // ch_versions = ch_versions.mix(ALIGN_STAR.out.versions.first().ifEmpty(null)) - - // // - // // SUBWORKFLOW: Mark duplicates with GATK4 - // // - // BAM_MARKDUPLICATES( - // ch_genome_bam, - // PREPARE_GENOME.out.fasta, - // PREPARE_GENOME.out.fai, - // []) - - // ch_genome_bam = BAM_MARKDUPLICATES.out.bam_bai - - // //Gather QC reports - // ch_reports = ch_reports.mix(BAM_MARKDUPLICATES.out.stats.collect{it[1]}.ifEmpty([])) - // ch_reports = ch_reports.mix(BAM_MARKDUPLICATES.out.metrics.collect{it[1]}.ifEmpty([])) - // ch_versions = ch_versions.mix(BAM_MARKDUPLICATES.out.versions.first().ifEmpty(null)) + // + // MODULE: Scatter one interval-list into many interval-files using GATK4 IntervalListTools + // + ch_interval_list_split = Channel.empty() + if (!params.skip_intervallisttools) { + GATK4_INTERVALLISTTOOLS(ch_interval_list) + ch_interval_list_split = GATK4_INTERVALLISTTOOLS.out.interval_list.map{ meta, bed -> [bed] }.flatten() + } + else ch_interval_list_split = ch_interval_list + + // + // SUBWORKFLOW: Perform read alignment using STAR aligner + // + ch_genome_bam = Channel.empty() + ch_genome_bam_index = Channel.empty() + ch_samtools_stats = Channel.empty() + ch_samtools_flagstat = Channel.empty() + ch_samtools_idxstats = Channel.empty() + ch_star_multiqc = Channel.empty() + ch_aligner_pca_multiqc = Channel.empty() + ch_aligner_clustering_multiqc = Channel.empty() + + if (params.aligner == 'star') { + ALIGN_STAR( + ch_cat_fastq, + PREPARE_GENOME.out.star_index, + PREPARE_GENOME.out.gtf, + params.star_ignore_sjdbgtf, + seq_platform, + seq_center + ) + ch_genome_bam = ALIGN_STAR.out.bam + ch_genome_bam_index = ALIGN_STAR.out.bai + ch_transcriptome_bam = ALIGN_STAR.out.bam_transcript + + // Gather QC reports + ch_reports = ch_reports.mix(ALIGN_STAR.out.stats.collect{it[1]}.ifEmpty([])) + ch_reports = ch_reports.mix(ALIGN_STAR.out.log_final.collect{it[1]}.ifEmpty([])) + ch_versions = ch_versions.mix(ALIGN_STAR.out.versions.first().ifEmpty(null)) + + // + // SUBWORKFLOW: Mark duplicates with GATK4 + // + BAM_MARKDUPLICATES( + ch_genome_bam, + ch_fasta.map{ meta, fasta -> [fasta] }, + ch_fasta_fai, + []) + + ch_genome_bam = BAM_MARKDUPLICATES.out.cram + + //Gather QC reports + ch_reports = ch_reports.mix(BAM_MARKDUPLICATES.out.reports.collect{it[1]}.ifEmpty([])) + ch_versions = ch_versions.mix(BAM_MARKDUPLICATES.out.versions.first().ifEmpty(null)) // // // // SUBWORKFLOW: SplitNCigarReads from GATK4 over the intervals @@ -252,9 +251,9 @@ workflow RNAVAR { // ch_splitncigar_bam_bai = Channel.empty() // SPLITNCIGAR( // ch_genome_bam, - // PREPARE_GENOME.out.fasta, - // PREPARE_GENOME.out.fai, - // PREPARE_GENOME.out.dict, + // ch_fasta, + // ch_fasta_fai, + // ch_dict, // ch_interval_list_split // ) // ch_splitncigar_bam_bai = SPLITNCIGAR.out.bam_bai @@ -279,9 +278,9 @@ workflow RNAVAR { // GATK4_BASERECALIBRATOR( // ch_splitncigar_bam_bai_interval, - // PREPARE_GENOME.out.fasta, - // PREPARE_GENOME.out.fai, - // PREPARE_GENOME.out.dict, + // ch_fasta, + // ch_fasta_fai, + // ch_dict, // ch_known_sites, // ch_known_sites_tbi // ) @@ -306,8 +305,8 @@ workflow RNAVAR { // RECALIBRATE( // params.skip_multiqc, // ch_applybqsr_bam_bai_interval, - // PREPARE_GENOME.out.dict, - // PREPARE_GENOME.out.fai, + // ch_dict, + // ch_fasta_fai, // PREPARE_GENOME.out.fasta // ) @@ -319,7 +318,7 @@ workflow RNAVAR { // ch_versions = ch_versions.mix(RECALIBRATE.out.versions.first().ifEmpty(null)) // } else { // ch_bam_variant_calling = ch_splitncigar_bam_bai - // } + } // interval_flag = params.no_intervals // // Run haplotyper even in the absence of dbSNP files @@ -344,9 +343,9 @@ workflow RNAVAR { // GATK4_HAPLOTYPECALLER( // ch_haplotypecaller_interval_bam, - // PREPARE_GENOME.out.fasta, - // PREPARE_GENOME.out.fai, - // PREPARE_GENOME.out.dict, + // ch_fasta, + // ch_fasta_fai, + // ch_dict, // ch_dbsnp, // ch_dbsnp_tbi // ) @@ -364,16 +363,16 @@ workflow RNAVAR { // // MODULE: MergeVCFS from GATK4 // // Merge multiple VCF files into one VCF // // - // GATK4_MERGEVCFS(ch_haplotypecaller_raw, PREPARE_GENOME.out.dict) + // GATK4_MERGEVCFS(ch_haplotypecaller_raw, ch_dict) // ch_haplotypecaller_vcf = GATK4_MERGEVCFS.out.vcf // ch_versions = ch_versions.mix(GATK4_MERGEVCFS.out.versions.first().ifEmpty(null)) // if (params.generate_gvcf){ // GATK4_HAPLOTYPECALLERGVCF( // ch_haplotypecaller_interval_bam, - // PREPARE_GENOME.out.fasta, - // PREPARE_GENOME.out.fai, - // PREPARE_GENOME.out.dict, + // ch_fasta, + // ch_fasta_fai, + // ch_dict, // ch_dbsnp, // ch_dbsnp_tbi // ) @@ -415,9 +414,9 @@ workflow RNAVAR { // GATK4_COMBINEGVCFS( // ch_haplotypecallergvcf_raw, // ch_haplotypecallergvcf_raw_index, - // PREPARE_GENOME.out.fasta, - // PREPARE_GENOME.out.fai, - // PREPARE_GENOME.out.dict + // ch_fasta, + // ch_fasta_fai, + // ch_dict // ) // ch_haplotypecaller_gvcf = GATK4_COMBINEGVCFS.out.combined_gvcf // ch_versions = ch_versions.mix(GATK4_COMBINEGVCFS.out.versions.first().ifEmpty(null)) @@ -463,9 +462,9 @@ workflow RNAVAR { // GATK4_VARIANTFILTRATION( // ch_haplotypecaller_vcf_tbi, - // PREPARE_GENOME.out.fasta, - // PREPARE_GENOME.out.fai, - // PREPARE_GENOME.out.dict + // ch_fasta, + // ch_fasta_fai, + // ch_dict // ) // ch_filtered_vcf = GATK4_VARIANTFILTRATION.out.vcf From fc9cbc2b45abe22ce4c21465b94cbff7a895f830 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 13:14:47 +0200 Subject: [PATCH 21/34] splitncigar --- conf/modules.config | 8 ++-- subworkflows/local/align_star/main.nf | 5 +-- subworkflows/local/bam_markduplicates/main.nf | 38 ++++++++++------- subworkflows/local/bam_sort_samtools/main.nf | 4 +- subworkflows/local/bam_stats_samtools/main.nf | 5 +-- subworkflows/local/splitncigar/main.nf | 41 +++++++++---------- workflows/rnavar.nf | 32 +++++++-------- 7 files changed, 65 insertions(+), 68 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index d5e4f84e..b536b83f 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -204,19 +204,19 @@ process { publishDir = [ enabled: false ] } - withName: '.*:MARKDUPLICATES:GATK4_MARKDUPLICATES' { + withName: 'GATK4_MARKDUPLICATES' { ext.args = [ '--ASSUME_SORTED true', '--VALIDATION_STRINGENCY LENIENT', params.remove_duplicates ? '--REMOVE_DUPLICATES true' : '' ].join(' ').trim() - ext.prefix = {"${meta.id}.markdup.sorted"} + ext.prefix = {"${meta.id}.md.bam"} publishDir = [ [ path: { "${params.outdir}/preprocessing/${meta.id}" }, mode: params.publish_dir_mode, enabled: true, - pattern: "*.{bam}" + pattern: "*.{bam,bai}" ], [ path: { "${params.outdir}/reports/stats/${meta.id}" }, @@ -229,7 +229,7 @@ process { withName: '.*:MARKDUPLICATES:SAMTOOLS_INDEX' { ext.args = params.bam_csi_index ? '-c' : '' - ext.prefix = {"${meta.id}.markdup.sorted"} + ext.prefix = {"${meta.id}.md"} publishDir = [ path: { "${params.outdir}/preprocessing/${meta.id}" }, mode: params.publish_dir_mode, diff --git a/subworkflows/local/align_star/main.nf b/subworkflows/local/align_star/main.nf index 64ec6526..4949686f 100644 --- a/subworkflows/local/align_star/main.nf +++ b/subworkflows/local/align_star/main.nf @@ -52,10 +52,7 @@ workflow ALIGN_STAR { bam = BAM_SORT_SAMTOOLS.out.bam // channel: [ val(meta), [ bam ] ] bai = BAM_SORT_SAMTOOLS.out.bai // channel: [ val(meta), [ bai ] ] csi = BAM_SORT_SAMTOOLS.out.csi // channel: [ val(meta), [ csi ] ] - stats = BAM_SORT_SAMTOOLS.out.stats // channel: [ val(meta), [ stats ] ] - flagstat = BAM_SORT_SAMTOOLS.out.flagstat // channel: [ val(meta), [ flagstat ] ] - idxstats = BAM_SORT_SAMTOOLS.out.idxstats // channel: [ val(meta), [ idxstats ] ] - + reports = BAM_SORT_SAMTOOLS.out.reports versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/local/bam_markduplicates/main.nf b/subworkflows/local/bam_markduplicates/main.nf index fac00488..37d901b9 100644 --- a/subworkflows/local/bam_markduplicates/main.nf +++ b/subworkflows/local/bam_markduplicates/main.nf @@ -4,8 +4,9 @@ // For all modules here: // A when clause condition is defined in the conf/modules.config to determine if the module should be run -include { CRAM_QC_MOSDEPTH_SAMTOOLS } from '../cram_qc_mosdepth_samtools/main' -include { GATK4_MARKDUPLICATES } from '../../../modules/nf-core/gatk4/markduplicates/main' +include { BAM_STATS_SAMTOOLS } from '../bam_stats_samtools/main' +include { GATK4_MARKDUPLICATES } from '../../../modules/nf-core/gatk4/markduplicates/main' +include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' workflow BAM_MARKDUPLICATES { take: @@ -15,29 +16,36 @@ workflow BAM_MARKDUPLICATES { intervals_bed_combined // channel: [optional] [ intervals_bed ] main: - versions = Channel.empty() - reports = Channel.empty() + ch_versions = Channel.empty() + ch_reports = Channel.empty() // RUN MARKUPDUPLICATES GATK4_MARKDUPLICATES(bam, fasta, fasta_fai) - // Join with the crai file - cram = GATK4_MARKDUPLICATES.out.cram.join(GATK4_MARKDUPLICATES.out.crai, failOnDuplicate: true, failOnMismatch: true) + SAMTOOLS_INDEX(GATK4_MARKDUPLICATES.out.bam) - // QC on CRAM - // CRAM_QC_MOSDEPTH_SAMTOOLS(cram, fasta, intervals_bed_combined) + ch_bam_index = GATK4_MARKDUPLICATES.out.bam + .join(SAMTOOLS_INDEX.out.bai, remainder: true) + .join(SAMTOOLS_INDEX.out.csi, remainder: true) + .map{meta, bam, bai, csi -> + if (bai) [meta, bam, bai] + else [meta, bam, csi] + } + + BAM_STATS_SAMTOOLS(ch_bam_index) // Gather all reports generated - reports = reports.mix(GATK4_MARKDUPLICATES.out.metrics) - // reports = reports.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.reports) + ch_reports = ch_reports.mix(GATK4_MARKDUPLICATES.out.metrics) + ch_reports = ch_reports.mix(BAM_STATS_SAMTOOLS.out.reports) // Gather versions of all tools used - versions = versions.mix(GATK4_MARKDUPLICATES.out.versions) - // versions = versions.mix(CRAM_QC_MOSDEPTH_SAMTOOLS.out.versions) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) + ch_versions = ch_versions.mix(GATK4_MARKDUPLICATES.out.versions) + ch_versions = ch_versions.mix(BAM_STATS_SAMTOOLS.out.versions) emit: - cram - reports + bam = ch_bam_index + reports = ch_reports - versions // channel: [ versions.yml ] + versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/local/bam_sort_samtools/main.nf b/subworkflows/local/bam_sort_samtools/main.nf index 9b4f6e31..2680e618 100644 --- a/subworkflows/local/bam_sort_samtools/main.nf +++ b/subworkflows/local/bam_sort_samtools/main.nf @@ -47,8 +47,6 @@ workflow BAM_SORT_SAMTOOLS { bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), [ bai ] ] csi = SAMTOOLS_INDEX.out.csi // channel: [ val(meta), [ csi ] ] - stats = BAM_STATS_SAMTOOLS.out.stats // channel: [ val(meta), [ stats ] ] - flagstat = BAM_STATS_SAMTOOLS.out.flagstat // channel: [ val(meta), [ flagstat ] ] - idxstats = BAM_STATS_SAMTOOLS.out.idxstats // channel: [ val(meta), [ idxstats ] ] + reports = BAM_STATS_SAMTOOLS.out.reports versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/local/bam_stats_samtools/main.nf b/subworkflows/local/bam_stats_samtools/main.nf index 94bb2037..6b664686 100644 --- a/subworkflows/local/bam_stats_samtools/main.nf +++ b/subworkflows/local/bam_stats_samtools/main.nf @@ -22,9 +22,6 @@ workflow BAM_STATS_SAMTOOLS { ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) emit: - stats = SAMTOOLS_STATS.out.stats // channel: [ val(meta), [ stats ] ] - flagstat = SAMTOOLS_FLAGSTAT.out.flagstat // channel: [ val(meta), [ flagstat ] ] - idxstats = SAMTOOLS_IDXSTATS.out.idxstats // channel: [ val(meta), [ idxstats ] ] - + reports = SAMTOOLS_STATS.out.stats.mix(SAMTOOLS_FLAGSTAT.out.flagstat, SAMTOOLS_IDXSTATS.out.idxstats) versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/local/splitncigar/main.nf b/subworkflows/local/splitncigar/main.nf index 9348a098..cbc30618 100644 --- a/subworkflows/local/splitncigar/main.nf +++ b/subworkflows/local/splitncigar/main.nf @@ -9,52 +9,49 @@ include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index workflow SPLITNCIGAR { take: bam // channel: [ val(meta), [ bam ], [bai] ] - fasta // channel: [ fasta ] - fasta_fai // channel: [ fai ] - fasta_dict // channel: [ dict ] + ch_fasta // channel: [ fasta ] + ch_fai // channel: [ fai ] + ch_dict // channel: [ dict ] intervals // channel: [ interval_list] main: - ch_versions = Channel.empty() - bam.combine(intervals) - .map{ meta, bam, bai, intervals -> + bam_interval = bam.combine(intervals).map{ meta, bam, bai, intervals -> new_meta = meta.clone() new_meta.id = meta.id + "_" + intervals.baseName new_meta.sample = meta.id [new_meta, bam, bai, intervals] - }.set{bam_interval} + } - GATK4_SPLITNCIGARREADS ( + GATK4_SPLITNCIGARREADS( bam_interval, - fasta, - fasta_fai, - fasta_dict + ch_fasta.map{ meta, fasta -> [fasta] }, + ch_fai, + ch_dict.map{ meta, dict -> [dict] }, ) bam_splitncigar = GATK4_SPLITNCIGARREADS.out.bam ch_versions = ch_versions.mix(GATK4_SPLITNCIGARREADS.out.versions.first()) - bam_splitncigar + bam_splitncigar_interval = bam_splitncigar .map{ meta, bam -> new_meta = meta.clone() new_meta.id = meta.sample [new_meta, bam] - }.groupTuple().set{bam_splitncigar_interval} + }.groupTuple() + + SAMTOOLS_MERGE(bam_splitncigar_interval, + ch_fasta, + ch_fai.map{ fai -> [[id:fai.baseName], fai] }) - SAMTOOLS_MERGE ( - bam_splitncigar_interval, - fasta - ) splitncigar_bam = SAMTOOLS_MERGE.out.bam ch_versions = ch_versions.mix(SAMTOOLS_MERGE.out.versions.first()) - SAMTOOLS_INDEX ( - splitncigar_bam - ) + SAMTOOLS_INDEX(splitncigar_bam) + splitncigar_bam_bai = splitncigar_bam - .join(SAMTOOLS_INDEX.out.bai, by: [0], remainder: true) - .join(SAMTOOLS_INDEX.out.csi, by: [0], remainder: true) + .join(SAMTOOLS_INDEX.out.bai, remainder: true) + .join(SAMTOOLS_INDEX.out.csi, remainder: true) .map{meta, bam, bai, csi -> if (bai) [meta, bam, bai] else [meta, bam, csi] diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 7f84af92..0cbbb699 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -225,7 +225,7 @@ workflow RNAVAR { ch_transcriptome_bam = ALIGN_STAR.out.bam_transcript // Gather QC reports - ch_reports = ch_reports.mix(ALIGN_STAR.out.stats.collect{it[1]}.ifEmpty([])) + ch_reports = ch_reports.mix(ALIGN_STAR.out.reports) ch_reports = ch_reports.mix(ALIGN_STAR.out.log_final.collect{it[1]}.ifEmpty([])) ch_versions = ch_versions.mix(ALIGN_STAR.out.versions.first().ifEmpty(null)) @@ -238,26 +238,26 @@ workflow RNAVAR { ch_fasta_fai, []) - ch_genome_bam = BAM_MARKDUPLICATES.out.cram + ch_genome_bam = BAM_MARKDUPLICATES.out.bam //Gather QC reports ch_reports = ch_reports.mix(BAM_MARKDUPLICATES.out.reports.collect{it[1]}.ifEmpty([])) ch_versions = ch_versions.mix(BAM_MARKDUPLICATES.out.versions.first().ifEmpty(null)) - // // - // // SUBWORKFLOW: SplitNCigarReads from GATK4 over the intervals - // // Splits reads that contain Ns in their cigar string(e.g. spanning splicing events in RNAseq data). - // // - // ch_splitncigar_bam_bai = Channel.empty() - // SPLITNCIGAR( - // ch_genome_bam, - // ch_fasta, - // ch_fasta_fai, - // ch_dict, - // ch_interval_list_split - // ) - // ch_splitncigar_bam_bai = SPLITNCIGAR.out.bam_bai - // ch_versions = ch_versions.mix(SPLITNCIGAR.out.versions.first().ifEmpty(null)) + // + // SUBWORKFLOW: SplitNCigarReads from GATK4 over the intervals + // Splits reads that contain Ns in their cigar string(e.g. spanning splicing events in RNAseq data). + // + ch_splitncigar_bam_bai = Channel.empty() + SPLITNCIGAR( + ch_genome_bam, + ch_fasta, + ch_fasta_fai, + ch_dict, + ch_interval_list_split + ) + ch_splitncigar_bam_bai = SPLITNCIGAR.out.bam_bai + ch_versions = ch_versions.mix(SPLITNCIGAR.out.versions.first().ifEmpty(null)) // // // // MODULE: BaseRecalibrator from GATK4 From d8e927f01e011787a13cae89d90647a3c74b3226 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 13:24:50 +0200 Subject: [PATCH 22/34] baserecalibrator --- workflows/rnavar.nf | 61 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 0cbbb699..694357e1 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -99,10 +99,10 @@ def seq_platform = params.seq_platform ?: [] def seq_center = params.seq_center ?: [] // // Initialize file channels based on params -// ch_dbsnp = params.dbsnp ? Channel.fromPath(params.dbsnp).collect() : Channel.empty() -// ch_dbsnp_tbi = params.dbsnp_tbi ? Channel.fromPath(params.dbsnp_tbi).collect() : Channel.empty() -// ch_known_indels = params.known_indels ? Channel.fromPath(params.known_indels).collect() : Channel.empty() -// ch_known_indels_tbi = params.known_indels_tbi ? Channel.fromPath(params.known_indels_tbi).collect() : Channel.empty() +ch_dbsnp = params.dbsnp ? Channel.fromPath(params.dbsnp).collect() : Channel.empty() +ch_dbsnp_tbi = params.dbsnp_tbi ? Channel.fromPath(params.dbsnp_tbi).collect() : Channel.empty() +ch_known_indels = params.known_indels ? Channel.fromPath(params.known_indels).collect() : Channel.empty() +ch_known_indels_tbi = params.known_indels_tbi ? Channel.fromPath(params.known_indels_tbi).collect() : Channel.empty() // // Initialize variant annotation associated channels // ch_snpeff_db = params.snpeff_db ?: Channel.empty() @@ -259,32 +259,31 @@ workflow RNAVAR { ch_splitncigar_bam_bai = SPLITNCIGAR.out.bam_bai ch_versions = ch_versions.mix(SPLITNCIGAR.out.versions.first().ifEmpty(null)) - // // - // // MODULE: BaseRecalibrator from GATK4 - // // Generates a recalibration table based on various co-variates - // // - // ch_bam_variant_calling = Channel.empty() - // if (!params.skip_baserecalibration) { - // ch_bqsr_table = Channel.empty() - // // known_sites is made by grouping both the dbsnp and the known indels ressources - // // they can either or both be optional - // ch_known_sites = ch_dbsnp.concat(ch_known_indels).collect() - // ch_known_sites_tbi = ch_dbsnp_tbi.concat(ch_known_indels_tbi).collect() - - // ch_interval_list_recalib = ch_interval_list.map{ meta, bed -> [bed] }.flatten() - // ch_splitncigar_bam_bai.combine(ch_interval_list_recalib) - // .map{ meta, bam, bai, interval -> [ meta, bam, bai, interval] - // }.set{ch_splitncigar_bam_bai_interval} - - // GATK4_BASERECALIBRATOR( - // ch_splitncigar_bam_bai_interval, - // ch_fasta, - // ch_fasta_fai, - // ch_dict, - // ch_known_sites, - // ch_known_sites_tbi - // ) - // ch_bqsr_table = GATK4_BASERECALIBRATOR.out.table + // + // MODULE: BaseRecalibrator from GATK4 + // Generates a recalibration table based on various co-variates + // + ch_bam_variant_calling = Channel.empty() + if (!params.skip_baserecalibration) { + ch_bqsr_table = Channel.empty() + // known_sites is made by grouping both the dbsnp and the known indels ressources + // they can either or both be optional + ch_known_sites = ch_dbsnp.concat(ch_known_indels).collect() + ch_known_sites_tbi = ch_dbsnp_tbi.concat(ch_known_indels_tbi).collect() + + ch_interval_list_recalib = ch_interval_list.map{ meta, bed -> [bed] }.flatten() + ch_splitncigar_bam_bai_interval = ch_splitncigar_bam_bai.combine(ch_interval_list_recalib) + .map{ meta, bam, bai, interval -> [ meta, bam, bai, interval] } + + GATK4_BASERECALIBRATOR( + ch_splitncigar_bam_bai_interval, + ch_fasta.map{ meta, fasta -> [fasta] }, + ch_fasta_fai, + ch_dict.map{ meta, dict -> [dict] }, + ch_known_sites, + ch_known_sites_tbi + ) + ch_bqsr_table = GATK4_BASERECALIBRATOR.out.table // // Gather QC reports // ch_reports = ch_reports.mix(ch_bqsr_table.map{ meta, table -> table}) @@ -491,7 +490,7 @@ workflow RNAVAR { // ch_versions = ch_versions.mix(ANNOTATE.out.versions.first().ifEmpty(null)) // } - // } + } // ch_version_yaml = Channel.empty() // CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile(name: 'collated_versions.yml')) From 1ce336931d53c33551d23dd284bddb76e8e8cc54 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 13:32:17 +0200 Subject: [PATCH 23/34] recalibrate --- subworkflows/local/recalibrate/main.nf | 7 ++-- workflows/rnavar.nf | 46 +++++++++++++------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/subworkflows/local/recalibrate/main.nf b/subworkflows/local/recalibrate/main.nf index 3b4ab0d8..518e9fb2 100644 --- a/subworkflows/local/recalibrate/main.nf +++ b/subworkflows/local/recalibrate/main.nf @@ -33,9 +33,8 @@ workflow RECALIBRATE { bam_recalibrated = APPLYBQSR.out.bam ch_versions = ch_versions.mix(APPLYBQSR.out.versions.first()) - SAMTOOLS_INDEX ( - bam_recalibrated - ) + SAMTOOLS_INDEX(bam_recalibrated) + bam_recalibrated_index = bam_recalibrated .join(SAMTOOLS_INDEX.out.bai, by: [0], remainder: true) .join(SAMTOOLS_INDEX.out.csi, by: [0], remainder: true) @@ -49,7 +48,7 @@ workflow RECALIBRATE { samtools_stats = Channel.empty() if (!skip_samtools) { - SAMTOOLS_STATS(bam_recalibrated_index, []) + SAMTOOLS_STATS(bam_recalibrated_index, [[], []]) samtools_stats = SAMTOOLS_STATS.out.stats ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) } diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 694357e1..ce32bc22 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -285,29 +285,29 @@ workflow RNAVAR { ) ch_bqsr_table = GATK4_BASERECALIBRATOR.out.table - // // Gather QC reports - // ch_reports = ch_reports.mix(ch_bqsr_table.map{ meta, table -> table}) - // ch_versions = ch_versions.mix(GATK4_BASERECALIBRATOR.out.versions.first().ifEmpty(null)) - - // ch_bam_applybqsr = ch_splitncigar_bam_bai.join(ch_bqsr_table, by: [0]) - // ch_bam_recalibrated_qc = Channel.empty() - - // ch_interval_list_applybqsr = ch_interval_list.map{ meta, bed -> [bed] }.flatten() - // ch_bam_applybqsr.combine(ch_interval_list_applybqsr) - // .map{ meta, bam, bai, table, interval -> [ meta, bam, bai, table, interval]} - // .set{ch_applybqsr_bam_bai_interval} - - // // - // // MODULE: ApplyBaseRecalibrator from GATK4 - // // Recalibrates the base qualities of the input reads based on the recalibration table produced by the GATK BaseRecalibrator tool. - // // - // RECALIBRATE( - // params.skip_multiqc, - // ch_applybqsr_bam_bai_interval, - // ch_dict, - // ch_fasta_fai, - // PREPARE_GENOME.out.fasta - // ) + // Gather QC reports + ch_reports = ch_reports.mix(ch_bqsr_table.map{ meta, table -> table}) + ch_versions = ch_versions.mix(GATK4_BASERECALIBRATOR.out.versions.first().ifEmpty(null)) + + ch_bam_applybqsr = ch_splitncigar_bam_bai.join(ch_bqsr_table, by: [0]) + ch_bam_recalibrated_qc = Channel.empty() + + ch_interval_list_applybqsr = ch_interval_list.map{ meta, bed -> [bed] }.flatten() + ch_bam_applybqsr.combine(ch_interval_list_applybqsr) + .map{ meta, bam, bai, table, interval -> [ meta, bam, bai, table, interval]} + .set{ch_applybqsr_bam_bai_interval} + + // + // MODULE: ApplyBaseRecalibrator from GATK4 + // Recalibrates the base qualities of the input reads based on the recalibration table produced by the GATK BaseRecalibrator tool. + // + RECALIBRATE( + params.skip_multiqc, + ch_applybqsr_bam_bai_interval, + ch_dict.map{ meta, dict -> [dict] }, + ch_fasta_fai, + ch_fasta.map{ meta, fasta -> [fasta] } + ) // ch_bam_variant_calling = RECALIBRATE.out.bam // ch_bam_recalibrated_qc = RECALIBRATE.out.qc From 6669292e6a2c5e3459946d944d90222c87828856 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 13:38:31 +0200 Subject: [PATCH 24/34] haplotypecaller --- workflows/rnavar.nf | 89 ++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index ce32bc22..90d1b825 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -309,62 +309,59 @@ workflow RNAVAR { ch_fasta.map{ meta, fasta -> [fasta] } ) - // ch_bam_variant_calling = RECALIBRATE.out.bam - // ch_bam_recalibrated_qc = RECALIBRATE.out.qc + ch_bam_variant_calling = RECALIBRATE.out.bam + ch_bam_recalibrated_qc = RECALIBRATE.out.qc - // // Gather QC reports - // ch_reports = ch_reports.mix(RECALIBRATE.out.qc.collect{it[1]}.ifEmpty([])) - // ch_versions = ch_versions.mix(RECALIBRATE.out.versions.first().ifEmpty(null)) - // } else { - // ch_bam_variant_calling = ch_splitncigar_bam_bai + // Gather QC reports + ch_reports = ch_reports.mix(RECALIBRATE.out.qc.collect{it[1]}.ifEmpty([])) + ch_versions = ch_versions.mix(RECALIBRATE.out.versions.first().ifEmpty(null)) + } else { + ch_bam_variant_calling = ch_splitncigar_bam_bai } - // interval_flag = params.no_intervals - // // Run haplotyper even in the absence of dbSNP files - // if (!params.dbsnp){ - // ch_dbsnp = [] - // ch_dbsnp_tbi = [] - // } + interval_flag = params.no_intervals + // Run haplotyper even in the absence of dbSNP files + if (!params.dbsnp){ + ch_dbsnp = [] + ch_dbsnp_tbi = [] + } - // ch_haplotypecaller_vcf = Channel.empty() - // ch_haplotypecaller_interval_bam = ch_bam_variant_calling.combine(ch_interval_list_split) - // .map{ meta, bam, bai, interval_list -> - // new_meta = meta.clone() - // new_meta.id = meta.id + "_" + interval_list.baseName - // new_meta.sample = meta.id - // [new_meta, bam, bai, interval_list] - // } + ch_haplotypecaller_vcf = Channel.empty() + ch_haplotypecaller_interval_bam = ch_bam_variant_calling.combine(ch_interval_list_split) + .map{ meta, bam, bai, interval_list -> + [meta + [id:meta.id + "_" + interval_list.baseName], bam, bai, interval_list, []] + } - // // - // // MODULE: HaplotypeCaller from GATK4 - // // Calls germline SNPs and indels via local re-assembly of haplotypes. - // // + // + // MODULE: HaplotypeCaller from GATK4 + // Calls germline SNPs and indels via local re-assembly of haplotypes. + // - // GATK4_HAPLOTYPECALLER( - // ch_haplotypecaller_interval_bam, - // ch_fasta, - // ch_fasta_fai, - // ch_dict, - // ch_dbsnp, - // ch_dbsnp_tbi - // ) + GATK4_HAPLOTYPECALLER( + ch_haplotypecaller_interval_bam, + ch_fasta.map{ meta, fasta -> [fasta] }, + ch_fasta_fai, + ch_dict.map{ meta, dict -> [dict] }, + ch_dbsnp, + ch_dbsnp_tbi + ) - // ch_haplotypecaller_raw = GATK4_HAPLOTYPECALLER.out.vcf - // .map{ meta, vcf -> - // meta.id = meta.sample - // [meta, vcf]} - // .groupTuple() + ch_haplotypecaller_raw = GATK4_HAPLOTYPECALLER.out.vcf + .map{ meta, vcf -> + meta.id = meta.sample + [meta, vcf]} + .groupTuple() - // ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLER.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLER.out.versions.first().ifEmpty(null)) - // // - // // MODULE: MergeVCFS from GATK4 - // // Merge multiple VCF files into one VCF - // // - // GATK4_MERGEVCFS(ch_haplotypecaller_raw, ch_dict) - // ch_haplotypecaller_vcf = GATK4_MERGEVCFS.out.vcf - // ch_versions = ch_versions.mix(GATK4_MERGEVCFS.out.versions.first().ifEmpty(null)) + // + // MODULE: MergeVCFS from GATK4 + // Merge multiple VCF files into one VCF + // + GATK4_MERGEVCFS(ch_haplotypecaller_raw, ch_dict) + ch_haplotypecaller_vcf = GATK4_MERGEVCFS.out.vcf + ch_versions = ch_versions.mix(GATK4_MERGEVCFS.out.versions.first().ifEmpty(null)) // if (params.generate_gvcf){ // GATK4_HAPLOTYPECALLERGVCF( From 40128c7fd876c83c84960f624a186905e78565ce Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 13:50:29 +0200 Subject: [PATCH 25/34] HAPLOTYPECALLERGVCF --- workflows/rnavar.nf | 116 +++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 62 deletions(-) diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 90d1b825..9d7e8aac 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -289,7 +289,7 @@ workflow RNAVAR { ch_reports = ch_reports.mix(ch_bqsr_table.map{ meta, table -> table}) ch_versions = ch_versions.mix(GATK4_BASERECALIBRATOR.out.versions.first().ifEmpty(null)) - ch_bam_applybqsr = ch_splitncigar_bam_bai.join(ch_bqsr_table, by: [0]) + ch_bam_applybqsr = ch_splitncigar_bam_bai.join(ch_bqsr_table) ch_bam_recalibrated_qc = Channel.empty() ch_interval_list_applybqsr = ch_interval_list.map{ meta, bed -> [bed] }.flatten() @@ -363,76 +363,68 @@ workflow RNAVAR { ch_haplotypecaller_vcf = GATK4_MERGEVCFS.out.vcf ch_versions = ch_versions.mix(GATK4_MERGEVCFS.out.versions.first().ifEmpty(null)) - // if (params.generate_gvcf){ - // GATK4_HAPLOTYPECALLERGVCF( - // ch_haplotypecaller_interval_bam, - // ch_fasta, - // ch_fasta_fai, - // ch_dict, - // ch_dbsnp, - // ch_dbsnp_tbi - // ) - - // ch_haplotypecallergvcf_raw = GATK4_HAPLOTYPECALLERGVCF.out.vcf - // .map{ meta, vcf -> - // meta.id = meta.sample - // [meta, vcf]} - // .groupTuple() - - // ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLERGVCF.out.versions.first().ifEmpty(null)) - // // - // // MODULE: IndexFeatureFile from GATK4 - // // Index the gVCF files - // // - // GATK4_INDEXFEATUREFILE(GATK4_HAPLOTYPECALLERGVCF.out.vcf) + if (params.generate_gvcf){ + GATK4_HAPLOTYPECALLERGVCF( + ch_haplotypecaller_interval_bam, + ch_fasta.map{ meta, fasta -> [fasta] }, + ch_fasta_fai, + ch_dict.map{ meta, dict -> [dict] }, + ch_dbsnp, + ch_dbsnp_tbi + ) - // ch_haplotypecallergvcf_raw_index = GATK4_INDEXFEATUREFILE.out.index - // .map{ meta, idx -> - // meta.id = meta.sample - // [meta, idx]} - // .groupTuple() + ch_haplotypecallergvcf_raw = GATK4_HAPLOTYPECALLERGVCF.out.vcf + .map{ meta, vcf -> + meta.id = meta.sample + [meta, vcf] + }.groupTuple() - // ch_versions = ch_versions.mix(GATK4_INDEXFEATUREFILE.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(GATK4_HAPLOTYPECALLERGVCF.out.versions.first().ifEmpty(null)) + // + // MODULE: IndexFeatureFile from GATK4 + // Index the gVCF files + // + GATK4_INDEXFEATUREFILE(GATK4_HAPLOTYPECALLERGVCF.out.vcf) - // // - // // MODULE: CombineGVCFS from GATK4 - // // Merge multiple GVCF files into one GVCF - // // + ch_haplotypecallergvcf_raw_index = GATK4_INDEXFEATUREFILE.out.index + .map{ meta, idx -> + meta.id = meta.sample + [meta, idx] + }.groupTuple() - // //ch_haplotypecallergvcf_raw_tbi = ch_haplotypecallergvcf_raw - // // .join(ch_haplotypecallergvcf_raw_index, by: [0], remainder: true) - // // .map{meta, vcf, tbi -> - // // [meta, vcf, tbi] - // // } + ch_versions = ch_versions.mix(GATK4_INDEXFEATUREFILE.out.versions.first().ifEmpty(null)) + // MODULE: CombineGVCFS from GATK4 + // Merge multiple GVCF files into one GVCF + ch_haplotypecallergvcf_raw_tbi = ch_haplotypecallergvcf_raw + .join(ch_haplotypecallergvcf_raw_index, remainder: true) - // GATK4_COMBINEGVCFS( - // ch_haplotypecallergvcf_raw, - // ch_haplotypecallergvcf_raw_index, - // ch_fasta, - // ch_fasta_fai, - // ch_dict - // ) - // ch_haplotypecaller_gvcf = GATK4_COMBINEGVCFS.out.combined_gvcf - // ch_versions = ch_versions.mix(GATK4_COMBINEGVCFS.out.versions.first().ifEmpty(null)) + GATK4_COMBINEGVCFS( + ch_haplotypecallergvcf_raw_tbi, + ch_fasta.map{ meta, fasta -> [fasta] }, + ch_fasta_fai, + ch_dict.map{ meta, dict -> [dict] } + ) + ch_haplotypecaller_gvcf = GATK4_COMBINEGVCFS.out.combined_gvcf + ch_versions = ch_versions.mix(GATK4_COMBINEGVCFS.out.versions.first().ifEmpty(null)) - // // - // // MODULE: Index the VCF using TABIX - // // - // TABIXGVCF(ch_haplotypecaller_gvcf) + // + // MODULE: Index the VCF using TABIX + // + TABIXGVCF(ch_haplotypecaller_gvcf) - // ch_haplotypecaller_gvcf_tbi = ch_haplotypecaller_gvcf - // .join(TABIXGVCF.out.tbi, by: [0], remainder: true) - // .join(TABIXGVCF.out.csi, by: [0], remainder: true) - // .map{meta, vcf, tbi, csi -> - // if (tbi) [meta, vcf, tbi] - // else [meta, vcf, csi] - // } + ch_haplotypecaller_gvcf_tbi = ch_haplotypecaller_gvcf + .join(TABIXGVCF.out.tbi, remainder: true) + .join(TABIXGVCF.out.csi, remainder: true) + .map{meta, vcf, tbi, csi -> + if (tbi) [meta, vcf, tbi] + else [meta, vcf, csi] + } - // ch_versions = ch_versions.mix(TABIXGVCF.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(TABIXGVCF.out.versions.first().ifEmpty(null)) - // } + } // // // // MODULE: Index the VCF using TABIX @@ -440,8 +432,8 @@ workflow RNAVAR { // TABIX(ch_haplotypecaller_vcf) // ch_haplotypecaller_vcf_tbi = ch_haplotypecaller_vcf - // .join(TABIX.out.tbi, by: [0], remainder: true) - // .join(TABIX.out.csi, by: [0], remainder: true) + // .join(TABIX.out.tbi, remainder: true) + // .join(TABIX.out.csi, remainder: true) // .map{meta, vcf, tbi, csi -> // if (tbi) [meta, vcf, tbi] // else [meta, vcf, csi] From 0bf28e0d18bdf1af119fcfcd03807ddb7d997880 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 14:13:29 +0200 Subject: [PATCH 26/34] multiqc --- subworkflows/local/bam_sort_samtools/main.nf | 9 +- subworkflows/local/bam_stats_samtools/main.nf | 7 +- workflows/rnavar.nf | 124 +++++++++--------- 3 files changed, 70 insertions(+), 70 deletions(-) diff --git a/subworkflows/local/bam_sort_samtools/main.nf b/subworkflows/local/bam_sort_samtools/main.nf index 2680e618..feabd8c9 100644 --- a/subworkflows/local/bam_sort_samtools/main.nf +++ b/subworkflows/local/bam_sort_samtools/main.nf @@ -37,9 +37,10 @@ workflow BAM_SORT_SAMTOOLS { } .set { ch_bam_bai } - BAM_STATS_SAMTOOLS ( - ch_bam_bai - ) + BAM_STATS_SAMTOOLS(ch_bam_bai) + + reports = BAM_STATS_SAMTOOLS.out.reports.map{meta, logs -> [logs] } + ch_versions = ch_versions.mix(BAM_STATS_SAMTOOLS.out.versions.first()) emit: @@ -47,6 +48,6 @@ workflow BAM_SORT_SAMTOOLS { bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), [ bai ] ] csi = SAMTOOLS_INDEX.out.csi // channel: [ val(meta), [ csi ] ] - reports = BAM_STATS_SAMTOOLS.out.reports + reports versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/local/bam_stats_samtools/main.nf b/subworkflows/local/bam_stats_samtools/main.nf index 6b664686..09fdd265 100644 --- a/subworkflows/local/bam_stats_samtools/main.nf +++ b/subworkflows/local/bam_stats_samtools/main.nf @@ -12,6 +12,7 @@ workflow BAM_STATS_SAMTOOLS { main: ch_versions = Channel.empty() + ch_reports = Channel.empty() SAMTOOLS_FLAGSTAT(ch_bam_bai) SAMTOOLS_IDXSTATS(ch_bam_bai) @@ -21,7 +22,11 @@ workflow BAM_STATS_SAMTOOLS { ch_versions = ch_versions.mix(SAMTOOLS_IDXSTATS.out.versions.first()) ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) + ch_reports = ch_reports.mix(SAMTOOLS_STATS.out.stats) + ch_reports = ch_reports.mix(SAMTOOLS_FLAGSTAT.out.flagstat) + ch_reports = ch_reports.mix(SAMTOOLS_IDXSTATS.out.idxstats) + emit: - reports = SAMTOOLS_STATS.out.stats.mix(SAMTOOLS_FLAGSTAT.out.flagstat, SAMTOOLS_IDXSTATS.out.idxstats) + reports = ch_reports versions = ch_versions // channel: [ versions.yml ] } diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index 9d7e8aac..d72839cd 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -178,7 +178,7 @@ workflow RNAVAR { // MODULE: Generate QC summary using FastQC FASTQC(ch_cat_fastq) - ch_reports = ch_reports.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) + ch_reports = ch_reports.mix(FASTQC.out.zip.collect{ meta, logs -> logs }) ch_versions = ch_versions.mix(FASTQC.out.versions.first()) // @@ -426,39 +426,39 @@ workflow RNAVAR { } - // // - // // MODULE: Index the VCF using TABIX - // // - // TABIX(ch_haplotypecaller_vcf) + // + // MODULE: Index the VCF using TABIX + // + TABIX(ch_haplotypecaller_vcf) + + ch_haplotypecaller_vcf_tbi = ch_haplotypecaller_vcf + .join(TABIX.out.tbi, remainder: true) + .join(TABIX.out.csi, remainder: true) + .map{meta, vcf, tbi, csi -> + if (tbi) [meta, vcf, tbi] + else [meta, vcf, csi] + } - // ch_haplotypecaller_vcf_tbi = ch_haplotypecaller_vcf - // .join(TABIX.out.tbi, remainder: true) - // .join(TABIX.out.csi, remainder: true) - // .map{meta, vcf, tbi, csi -> - // if (tbi) [meta, vcf, tbi] - // else [meta, vcf, csi] - // } + ch_versions = ch_versions.mix(TABIX.out.versions.first().ifEmpty(null)) + ch_final_vcf = ch_haplotypecaller_vcf - // ch_versions = ch_versions.mix(TABIX.out.versions.first().ifEmpty(null)) - // ch_final_vcf = ch_haplotypecaller_vcf + // + // MODULE: VariantFiltration from GATK4 + // Filter variant calls based on certain criteria + // + if (!params.skip_variantfiltration && !params.bam_csi_index ) { - // // - // // MODULE: VariantFiltration from GATK4 - // // Filter variant calls based on certain criteria - // // - // if (!params.skip_variantfiltration && !params.bam_csi_index ) { - - // GATK4_VARIANTFILTRATION( - // ch_haplotypecaller_vcf_tbi, - // ch_fasta, - // ch_fasta_fai, - // ch_dict - // ) - - // ch_filtered_vcf = GATK4_VARIANTFILTRATION.out.vcf - // ch_final_vcf = ch_filtered_vcf - // ch_versions = ch_versions.mix(GATK4_VARIANTFILTRATION.out.versions.first().ifEmpty(null)) - // } + GATK4_VARIANTFILTRATION( + ch_haplotypecaller_vcf_tbi, + ch_fasta, + ch_fasta_fai.map{ it -> [ [id:'fai'], it ] }, + ch_dict + ) + + ch_filtered_vcf = GATK4_VARIANTFILTRATION.out.vcf + ch_final_vcf = ch_filtered_vcf + ch_versions = ch_versions.mix(GATK4_VARIANTFILTRATION.out.versions.first().ifEmpty(null)) + } // // // // SUBWORKFLOW: Annotate variants using snpEff and Ensembl VEP if enabled. @@ -481,40 +481,34 @@ workflow RNAVAR { } - // ch_version_yaml = Channel.empty() - // CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile(name: 'collated_versions.yml')) - // ch_version_yaml = CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect() - - // // - // // MODULE: MultiQC - // // Present summary of reads, alignment, duplicates, BSQR stats for all samples as well as workflow summary/parameters as single report - // // - // if (!params.skip_multiqc){ - // workflow_summary = WorkflowRnavar.paramsSummaryMultiqc(workflow, summary_params) - // ch_workflow_summary = Channel.value(workflow_summary) - - // methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) - // ch_methods_description = Channel.value(methods_description) - - // ch_multiqc_files = Channel.empty() - // ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - // ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) - // ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) - // ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) - // ch_multiqc_files = ch_multiqc_files.mix(ch_version_yaml) - // ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) - // ch_multiqc_files = ch_multiqc_files.mix(ch_reports.collect()) - // ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_config) - // ch_multiqc_files = ch_multiqc_files.mix(ch_rnavar_logo) - - // MULTIQC( - // ch_multiqc_files.collect(), - // ch_multiqc_config.toList(), - // ch_multiqc_custom_config.toList(), - // ch_multiqc_logo.toList() - // ) - // multiqc_report = MULTIQC.out.report.toList() - // } + ch_version_yaml = Channel.empty() + CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile(name: 'collated_versions.yml')) + ch_version_yaml = CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect() + + // + // MODULE: MultiQC + // Present summary of reads, alignment, duplicates, BSQR stats for all samples as well as workflow summary/parameters as single report + // + if (!params.skip_multiqc){ + workflow_summary = WorkflowRnavar.paramsSummaryMultiqc(workflow, summary_params) + ch_workflow_summary = Channel.value(workflow_summary) + + methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) + ch_methods_description = Channel.value(methods_description) + + ch_reports.view() + + multiqc_files = Channel.empty() + multiqc_files = multiqc_files.mix(ch_version_yaml) + multiqc_files = multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + multiqc_files = multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) + multiqc_files = multiqc_files.mix(ch_reports.collect().ifEmpty([])) + + MULTIQC(multiqc_files.collect(), ch_multiqc_config.collect().ifEmpty([]), ch_multiqc_custom_config.collect().ifEmpty([]), ch_multiqc_logo.collect().ifEmpty([])) + + multiqc_report = MULTIQC.out.report.toList() + ch_versions = ch_versions.mix(MULTIQC.out.versions) + } } /* From 90600fe72e7f302b91f47dd54420ed469f220f27 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 14:15:17 +0200 Subject: [PATCH 27/34] update tests --- tests/test_annotation.yml | 54 ++++++++++++++++----------------- tests/test_bamcsiindex.yml | 2 +- tests/test_default.yml | 2 +- tests/test_removeduplicates.yml | 2 +- tests/test_skipbasecalib.yml | 2 +- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/test_annotation.yml b/tests/test_annotation.yml index b6fb529d..80bdadf3 100644 --- a/tests/test_annotation.yml +++ b/tests/test_annotation.yml @@ -1,27 +1,27 @@ -- name: Run snpEff - command: nextflow run main.nf -profile test,docker --annotate_tools snpeff - tags: - - annotation - - snpeff - files: - - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz - - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz.tbi - - path: results/reports/multiqc_report.html -- name: Run VEP - command: nextflow run main.nf -profile test,docker --annotate_tools vep --skip_multiqc - tags: - - annotation - - vep - files: - - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz - - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz.tbi -- name: Run snpEff followed by VEP - command: nextflow run main.nf -profile test,docker --annotate_tools merge --skip_multiqc - tags: - - annotation - - merge - - snpeff - - vep - files: - - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz - - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz.tbi +# - name: Run snpEff +# command: nextflow run main.nf -profile test,docker --annotate_tools snpeff --outdir results +# tags: +# - annotation +# - snpeff +# files: +# - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz +# - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz.tbi +# - path: results/reports/multiqc_report.html +# - name: Run VEP +# command: nextflow run main.nf -profile test,docker --annotate_tools vep --skip_multiqc --outdir results +# tags: +# - annotation +# - vep +# files: +# - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz +# - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz.tbi +# - name: Run snpEff followed by VEP +# command: nextflow run main.nf -profile test,docker --annotate_tools merge --skip_multiqc --outdir results +# tags: +# - annotation +# - merge +# - snpeff +# - vep +# files: +# - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz +# - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz.tbi diff --git a/tests/test_bamcsiindex.yml b/tests/test_bamcsiindex.yml index 06fb817f..a06de700 100644 --- a/tests/test_bamcsiindex.yml +++ b/tests/test_bamcsiindex.yml @@ -1,5 +1,5 @@ - name: Run pipeline with bam_csi_index for large genomes - command: nextflow run main.nf -profile test,docker --bam_csi_index + command: nextflow run main.nf -profile test,docker --bam_csi_index --outdir results tags: - bamcsiindex files: diff --git a/tests/test_default.yml b/tests/test_default.yml index 36cb35fa..776ec6f3 100644 --- a/tests/test_default.yml +++ b/tests/test_default.yml @@ -1,5 +1,5 @@ - name: Run default pipeline - command: nextflow run main.nf -profile test,docker + command: nextflow run main.nf -profile test,docker --outdir results tags: - default - preprocessing diff --git a/tests/test_removeduplicates.yml b/tests/test_removeduplicates.yml index 8889ad6a..d0650728 100644 --- a/tests/test_removeduplicates.yml +++ b/tests/test_removeduplicates.yml @@ -1,5 +1,5 @@ - name: Run pipeline with duplicate reads removed - command: nextflow run main.nf -profile test,docker --remove_duplicates true + command: nextflow run main.nf -profile test,docker --remove_duplicates true --outdir results tags: - removeduplicates - preprocessing diff --git a/tests/test_skipbasecalib.yml b/tests/test_skipbasecalib.yml index 7b5b2132..67f1c463 100644 --- a/tests/test_skipbasecalib.yml +++ b/tests/test_skipbasecalib.yml @@ -1,5 +1,5 @@ - name: Run pipeline without base calibration step - command: nextflow run main.nf -profile test,docker --skip_baserecalibration true + command: nextflow run main.nf -profile test,docker --skip_baserecalibration true --outdir results tags: - skipbasecalib - preprocessing From 1cd7988ecfa283d587f6b5576f92d6ea788b44d0 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 14:17:45 +0200 Subject: [PATCH 28/34] remove annotation tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3319d00..cc8f7dfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" test: - "default" - - "annotation" + # - "annotation" - "removeduplicates" - "skipbasecalib" - "bamcsiindex" From 05cb821e582e7c4faf4af43c07a7573e1b0e2f22 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 14:29:27 +0200 Subject: [PATCH 29/34] fix test csi --- conf/modules.config | 1 - workflows/rnavar.nf | 2 -- 2 files changed, 3 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index b536b83f..16dd423b 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -326,7 +326,6 @@ process { withName: GATK4_MERGEVCFS { ext.prefix = {"${meta.id}.haplotypecaller"} - ext.args = params.bam_csi_index ? '--CREATE_INDEX false' : '' publishDir = [ path: { "${params.outdir}/variant_calling/${meta.id}" }, mode: params.publish_dir_mode, diff --git a/workflows/rnavar.nf b/workflows/rnavar.nf index d72839cd..698e0c25 100755 --- a/workflows/rnavar.nf +++ b/workflows/rnavar.nf @@ -496,8 +496,6 @@ workflow RNAVAR { methods_description = WorkflowRnavar.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) ch_methods_description = Channel.value(methods_description) - ch_reports.view() - multiqc_files = Channel.empty() multiqc_files = multiqc_files.mix(ch_version_yaml) multiqc_files = multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) From e32e8e894075ceb5b134ae0869158c3832b58095 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 14:37:28 +0200 Subject: [PATCH 30/34] fix tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc8f7dfa..fc55fff3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,4 +50,4 @@ jobs: run: python -m pip install --upgrade pip pytest-workflow - name: Run pipeline with tests settings - run: pytest --tag ${{matrix.test}} --kwdof + run: TMPDIR=~ PROFILE=docker pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes From 8858819e1d21090ec95307c4665a2a4fa9f58793 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 14:44:02 +0200 Subject: [PATCH 31/34] uncomment --- tests/test_annotation.yml | 54 +++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/test_annotation.yml b/tests/test_annotation.yml index 80bdadf3..28ef4322 100644 --- a/tests/test_annotation.yml +++ b/tests/test_annotation.yml @@ -1,27 +1,27 @@ -# - name: Run snpEff -# command: nextflow run main.nf -profile test,docker --annotate_tools snpeff --outdir results -# tags: -# - annotation -# - snpeff -# files: -# - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz -# - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz.tbi -# - path: results/reports/multiqc_report.html -# - name: Run VEP -# command: nextflow run main.nf -profile test,docker --annotate_tools vep --skip_multiqc --outdir results -# tags: -# - annotation -# - vep -# files: -# - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz -# - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz.tbi -# - name: Run snpEff followed by VEP -# command: nextflow run main.nf -profile test,docker --annotate_tools merge --skip_multiqc --outdir results -# tags: -# - annotation -# - merge -# - snpeff -# - vep -# files: -# - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz -# - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz.tbi +- name: Run snpEff + command: nextflow run main.nf -profile test,docker --annotate_tools snpeff --outdir results + tags: + - annotation + - snpeff + files: + - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz + - path: results/variant_annotation/GM12878/GM12878_snpEff.ann.vcf.gz.tbi + - path: results/reports/multiqc_report.html +- name: Run VEP + command: nextflow run main.nf -profile test,docker --annotate_tools vep --skip_multiqc --outdir results + tags: + - annotation + - vep + files: + - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz + - path: results/variant_annotation/GM12878/GM12878_VEP.ann.vcf.gz.tbi +- name: Run snpEff followed by VEP + command: nextflow run main.nf -profile test,docker --annotate_tools merge --skip_multiqc --outdir results + tags: + - annotation + - merge + - snpeff + - vep + files: + - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz + - path: results/variant_annotation/GM12878/GM12878_snpEff_VEP.ann.vcf.gz.tbi From 80503cce89e7de4c44a3de598214f84608f40d42 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 15:46:12 +0200 Subject: [PATCH 32/34] fix csi tests --- conf/modules.config | 4 ++-- tests/test_bamcsiindex.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 16dd423b..55f7dcc9 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -216,7 +216,7 @@ process { path: { "${params.outdir}/preprocessing/${meta.id}" }, mode: params.publish_dir_mode, enabled: true, - pattern: "*.{bam,bai}" + pattern: "*.{bam,bai,csi}" ], [ path: { "${params.outdir}/reports/stats/${meta.id}" }, @@ -227,7 +227,7 @@ process { ] } - withName: '.*:MARKDUPLICATES:SAMTOOLS_INDEX' { + withName: '.*:BAM_MARKDUPLICATES:SAMTOOLS_INDEX' { ext.args = params.bam_csi_index ? '-c' : '' ext.prefix = {"${meta.id}.md"} publishDir = [ diff --git a/tests/test_bamcsiindex.yml b/tests/test_bamcsiindex.yml index a06de700..7235d9f9 100644 --- a/tests/test_bamcsiindex.yml +++ b/tests/test_bamcsiindex.yml @@ -4,7 +4,7 @@ - bamcsiindex files: - path: results/reports/multiqc_report.html - - path: results/preprocessing/GM12878/GM12878.markdup.sorted.bam - - path: results/preprocessing/GM12878/GM12878.markdup.sorted.bam.csi + - path: results/preprocessing/GM12878/GM12878.md.bam + - path: results/preprocessing/GM12878/GM12878.md.bam.csi - path: results/variant_calling/GM12878/GM12878.haplotypecaller.vcf.gz - path: results/variant_calling/GM12878/GM12878.haplotypecaller.vcf.gz.csi From cfeeeb4c66f1d2eed435032dc2368e795a721e26 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 20 Sep 2023 18:04:05 +0200 Subject: [PATCH 33/34] fix comments from review --- assets/schema_input.json | 14 +-- bin/check_samplesheet.py | 259 --------------------------------------- 2 files changed, 3 insertions(+), 270 deletions(-) delete mode 100755 bin/check_samplesheet.py diff --git a/assets/schema_input.json b/assets/schema_input.json index dc7fcba6..7cbc45b8 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -14,17 +14,9 @@ "meta": ["sample"] }, "fastq_1": { - "errorMessage": "FastQ file for reads 1 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", - "anyOf": [ - { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$" - }, - { - "type": "string", - "maxLength": 0 - } - ], + "errorMessage": "FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", + "type": "string", + "pattern": "^\\S+\\.f(ast)?q\\.gz$", "format": "file-path", "exists": true }, diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py deleted file mode 100755 index 9f85605d..00000000 --- a/bin/check_samplesheet.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python - - -"""Provide a command line tool to validate and transform tabular samplesheets.""" - - -import argparse -import csv -import logging -import sys -from collections import Counter -from pathlib import Path - -logger = logging.getLogger() - - -class RowChecker: - """ - Define a service that can validate and transform each given row. - - Attributes: - modified (list): A list of dicts, where each dict corresponds to a previously - validated and transformed row. The order of rows is maintained. - - """ - - VALID_FORMATS = ( - ".fq.gz", - ".fastq.gz", - ) - - def __init__( - self, - sample_col="sample", - first_col="fastq_1", - second_col="fastq_2", - single_col="single_end", - **kwargs, - ): - """ - Initialize the row checker with the expected column names. - - Args: - sample_col (str): The name of the column that contains the sample name - (default "sample"). - first_col (str): The name of the column that contains the first (or only) - FASTQ file path (default "fastq_1"). - second_col (str): The name of the column that contains the second (if any) - FASTQ file path (default "fastq_2"). - single_col (str): The name of the new column that will be inserted and - records whether the sample contains single- or paired-end sequencing - reads (default "single_end"). - - """ - super().__init__(**kwargs) - self._sample_col = sample_col - self._first_col = first_col - self._second_col = second_col - self._single_col = single_col - self._seen = set() - self.modified = [] - - def validate_and_transform(self, row): - """ - Perform all validations on the given row and insert the read pairing status. - - Args: - row (dict): A mapping from column headers (keys) to elements of that row - (values). - - """ - self._validate_sample(row) - self._validate_first(row) - self._validate_second(row) - self._validate_pair(row) - self._seen.add((row[self._sample_col], row[self._first_col])) - self.modified.append(row) - - def _validate_sample(self, row): - """Assert that the sample name exists and convert spaces to underscores.""" - if len(row[self._sample_col]) <= 0: - raise AssertionError("Sample input is required.") - # Sanitize samples slightly. - row[self._sample_col] = row[self._sample_col].replace(" ", "_") - - def _validate_first(self, row): - """Assert that the first FASTQ entry is non-empty and has the right format.""" - if len(row[self._first_col]) <= 0: - raise AssertionError("At least the first FASTQ file is required.") - self._validate_fastq_format(row[self._first_col]) - - def _validate_second(self, row): - """Assert that the second FASTQ entry has the right format if it exists.""" - if len(row[self._second_col]) > 0: - self._validate_fastq_format(row[self._second_col]) - - def _validate_pair(self, row): - """Assert that read pairs have the same file extension. Report pair status.""" - if row[self._first_col] and row[self._second_col]: - row[self._single_col] = False - first_col_suffix = Path(row[self._first_col]).suffixes[-2:] - second_col_suffix = Path(row[self._second_col]).suffixes[-2:] - if first_col_suffix != second_col_suffix: - raise AssertionError("FASTQ pairs must have the same file extensions.") - else: - row[self._single_col] = True - - def _validate_fastq_format(self, filename): - """Assert that a given filename has one of the expected FASTQ extensions.""" - if not any(filename.endswith(extension) for extension in self.VALID_FORMATS): - raise AssertionError( - f"The FASTQ file has an unrecognized extension: {filename}\n" - f"It should be one of: {', '.join(self.VALID_FORMATS)}" - ) - - def validate_unique_samples(self): - """ - Assert that the combination of sample name and FASTQ filename is unique. - - In addition to the validation, also rename all samples to have a suffix of _T{n}, where n is the - number of times the same sample exist, but with different FASTQ files, e.g., multiple runs per experiment. - - """ - if len(self._seen) != len(self.modified): - raise AssertionError("The pair of sample name and FASTQ must be unique.") - seen = Counter() - for row in self.modified: - sample = row[self._sample_col] - seen[sample] += 1 - row[self._sample_col] = f"{sample}_T{seen[sample]}" - - -def read_head(handle, num_lines=10): - """Read the specified number of lines from the current position in the file.""" - lines = [] - for idx, line in enumerate(handle): - if idx == num_lines: - break - lines.append(line) - return "".join(lines) - - -def sniff_format(handle): - """ - Detect the tabular format. - - Args: - handle (text file): A handle to a `text file`_ object. The read position is - expected to be at the beginning (index 0). - - Returns: - csv.Dialect: The detected tabular format. - - .. _text file: - https://docs.python.org/3/glossary.html#term-text-file - - """ - peek = read_head(handle) - handle.seek(0) - sniffer = csv.Sniffer() - dialect = sniffer.sniff(peek) - return dialect - - -def check_samplesheet(file_in, file_out): - """ - Check that the tabular samplesheet has the structure expected by nf-core pipelines. - - Validate the general shape of the table, expected columns, and each row. Also add - an additional column which records whether one or two FASTQ reads were found. - - Args: - file_in (pathlib.Path): The given tabular samplesheet. The format can be either - CSV, TSV, or any other format automatically recognized by ``csv.Sniffer``. - file_out (pathlib.Path): Where the validated and transformed samplesheet should - be created; always in CSV format. - - Example: - This function checks that the samplesheet follows the following structure, - see also the `rnavar test samplesheet`_:: - - sample,fastq_1,fastq_2 - SAMPLE_PE,SAMPLE_PE_RUN1_1.fastq.gz,SAMPLE_PE_RUN1_2.fastq.gz - SAMPLE_PE,SAMPLE_PE_RUN2_1.fastq.gz,SAMPLE_PE_RUN2_2.fastq.gz - SAMPLE_SE,SAMPLE_SE_RUN1_1.fastq.gz, - - .. _rnavar test samplesheet: - https://raw.githubusercontent.com/nf-core/test-datasets/rnavar/samplesheet/v1.0/samplesheet.csv - - """ - required_columns = {"sample", "fastq_1", "fastq_2"} - # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. - with file_in.open(newline="") as in_handle: - reader = csv.DictReader(in_handle, dialect=sniff_format(in_handle)) - # Validate the existence of the expected header columns. - if not required_columns.issubset(reader.fieldnames): - req_cols = ", ".join(required_columns) - logger.critical(f"The sample sheet **must** contain these column headers: {req_cols}.") - sys.exit(1) - # Validate each row. - checker = RowChecker() - for i, row in enumerate(reader): - try: - checker.validate_and_transform(row) - except AssertionError as error: - logger.critical(f"{str(error)} On line {i + 2}.") - sys.exit(1) - checker.validate_unique_samples() - header = list(reader.fieldnames) - header.insert(1, "single_end") - # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. - with file_out.open(mode="w", newline="") as out_handle: - writer = csv.DictWriter(out_handle, header, delimiter=",") - writer.writeheader() - for row in checker.modified: - writer.writerow(row) - - -def parse_args(argv=None): - """Define and immediately parse command line arguments.""" - parser = argparse.ArgumentParser( - description="Validate and transform a tabular samplesheet.", - epilog="Example: python check_samplesheet.py samplesheet.csv samplesheet.valid.csv", - ) - parser.add_argument( - "file_in", - metavar="FILE_IN", - type=Path, - help="Tabular input samplesheet in CSV or TSV format.", - ) - parser.add_argument( - "file_out", - metavar="FILE_OUT", - type=Path, - help="Transformed output samplesheet in CSV format.", - ) - parser.add_argument( - "-l", - "--log-level", - help="The desired log level (default WARNING).", - choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"), - default="WARNING", - ) - return parser.parse_args(argv) - - -def main(argv=None): - """Coordinate argument parsing and program execution.""" - args = parse_args(argv) - logging.basicConfig(level=args.log_level, format="[%(levelname)s] %(message)s") - if not args.file_in.is_file(): - logger.error(f"The given input file {args.file_in} was not found!") - sys.exit(2) - args.file_out.parent.mkdir(parents=True, exist_ok=True) - check_samplesheet(args.file_in, args.file_out) - - -if __name__ == "__main__": - sys.exit(main()) From a23a895f4af79c528c4815f949d2d5f5ad6501ca Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 21 Sep 2023 08:59:27 +0200 Subject: [PATCH 34/34] update schema --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 7ce1ee3d..36d0136e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -14,7 +14,7 @@ "properties": { "input": { "description": "Path to comma-separated file containing information about the samples in the experiment.", - "help_text": "A design file with information about the samples in your experiment. Use this parameter to specify the location of the input files. It has to be a comma-separated file with a header row. See [usage docs](https://nf-co.re/sarek/usage#input).\n\nIf no input file is specified, sarek will attempt to locate one in the `{outdir}` directory. If no input should be supplied, i.e. when --step is supplied or --build_from_index, then set --input false", + "help_text": "A design file with information about the samples in your experiment. Use this parameter to specify the location of the input files. It has to be a comma-separated file with a header row. See [usage docs](https://nf-co.re/rnavar/usage#input).", "fa_icon": "fas fa-file-csv", "schema": "assets/schema_input.json", "type": "string",