diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4ecfbfe3..b290e090 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,15 +10,7 @@ "vscode": { // Set *default* container specific settings.json values on container create. "settings": { - "python.defaultInterpreterPath": "/opt/conda/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", - "python.formatting.yapfPath": "/opt/conda/bin/yapf", - "python.linting.flake8Path": "/opt/conda/bin/flake8", - "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", - "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" + "python.defaultInterpreterPath": "/opt/conda/bin/python" }, // Add the IDs of extensions you want installed when the container is created. diff --git a/.editorconfig b/.editorconfig index 15309aea..2f879321 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,7 +18,12 @@ end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset -indent_size = unset +[/subworkflows/nf-core/**] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset [/assets/email*] indent_size = unset @@ -32,5 +37,5 @@ indent_style = unset indent_style = unset # ignore python -[*.{py}] +[*.{py,md}] indent_style = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 320b2087..72d2067c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -9,9 +9,8 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -:::info -If you need help using or modifying nf-core/viralrecon then the best place to ask is on the nf-core Slack [#viralrecon](https://nfcore.slack.com/channels/viralrecon) channel ([join our Slack here](https://nf-co.re/join/slack)). -::: +> [!NOTE] +> If you need help using or modifying nf-core/viralrecon then the best place to ask is on the nf-core Slack [#viralrecon](https://nfcore.slack.com/channels/viralrecon) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Contribution workflow @@ -27,8 +26,11 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests -You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to -receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir `. +You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command: + +```bash +nf-test test --profile debug,test,docker --verbose +``` When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. @@ -90,7 +92,7 @@ Once there, use `nf-core schema build` to add to `nextflow_schema.json`. Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. -The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block. +The process resources can be passed on to the tool dynamically within the process with the `${task.cpus}` and `${task.memory}` variables in the `script:` block. ### Naming schemes diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c7a4cea8..fc403ae6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,7 +18,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/vira - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/viralrecon/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the nf-core/viralrecon _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 52b1b1a3..ce44d6b2 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -19,7 +19,7 @@ jobs: # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v2 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 647c4827..c82142d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Free some space run: | @@ -44,6 +44,9 @@ jobs: with: version: "${{ matrix.NXF_VER }}" + - name: Disk space cleanup + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + - name: Run pipeline with test data run: | nextflow run ${GITHUB_WORKSPACE} -profile test,docker --publish_dir_mode link --outdir ./results diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index e37cfda5..0b6b1f27 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v9 + - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index 8a330045..08622fd5 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -6,6 +6,11 @@ name: Test successful pipeline download with 'nf-core download' # - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. on: workflow_dispatch: + inputs: + testbranch: + description: "The specific branch you wish to utilize for the test execution of nf-core download." + required: true + default: "dev" pull_request: types: - opened @@ -25,11 +30,11 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" architecture: "x64" - - uses: eWaterCycle/setup-singularity@v7 + - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: singularity-version: 3.8.3 @@ -42,13 +47,13 @@ jobs: run: | echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} - echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} - name: Download the pipeline env: NXF_SINGULARITY_CACHEDIR: ./ run: | - nf-core download ${{ env.REPO_LOWERCASE }} \ + nf-core download ${{ env.REPO_LOWERCASE }} \ --revision ${{ env.REPO_BRANCH }} \ --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ --compress "none" \ diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 81cd098e..073e1876 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -14,10 +14,10 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 cache: "pip" @@ -32,12 +32,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" architecture: "x64" @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 147bcd10..b706875f 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@v3 + uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 with: workflow: linting.yml workflow_conclusion: completed @@ -21,7 +21,7 @@ jobs: run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} number: ${{ steps.pr_number.outputs.pr_number }} diff --git a/.github/workflows/release-announcements.yml b/.github/workflows/release-announcements.yml index 21ac3f06..d468aeaa 100644 --- a/.github/workflows/release-announcements.yml +++ b/.github/workflows/release-announcements.yml @@ -9,6 +9,11 @@ jobs: toot: runs-on: ubuntu-latest steps: + - name: get topics and convert to hashtags + id: get_topics + run: | + curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' >> $GITHUB_OUTPUT + - uses: rzr/fediverse-action@master with: access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }} @@ -20,11 +25,13 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} + ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience #nextflow #bioinformatics + send-tweet: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.10" - name: Install dependencies @@ -56,7 +63,7 @@ jobs: bsky-post: runs-on: ubuntu-latest steps: - - uses: zentered/bluesky-post-action@v0.1.0 + - uses: zentered/bluesky-post-action@80dbe0a7697de18c15ad22f4619919ceb5ccf597 # v0.1.0 with: post: | Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! diff --git a/.gitpod.yml b/.gitpod.yml index 3a146b7e..bf09d883 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -13,10 +13,10 @@ vscode: - codezombiech.gitignore # Language support for .gitignore files # - cssho.vscode-svgviewer # SVG viewer - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code + - charliermarsh.ruff # Code linter Ruff diff --git a/README.md b/README.md index 3d8e4d0b..f4b3dd9d 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,15 @@ -[![GitHub Actions CI Status](https://github.com/nf-core/viralrecon/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/viralrecon/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/nf-core/viralrecon/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/viralrecon/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/viralrecon/results) -[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.3901628-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.3901628) +[![GitHub Actions CI Status](https://github.com/nf-core/viralrecon/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/viralrecon/actions/workflows/ci.yml) +[![GitHub Actions Linting Status](https://github.com/nf-core/viralrecon/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/viralrecon/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/viralrecon/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/viralrecon) +[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/viralrecon) [![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23viralrecon-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/viralrecon)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) diff --git a/assets/schema_input.json b/assets/schema_input.json index 3d255e41..a8e3c33b 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -10,25 +10,22 @@ "sample": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample name must be provided and cannot contain spaces" + "errorMessage": "Sample name must be provided and cannot contain spaces", + "meta": ["id"] }, "fastq_1": { "type": "string", + "format": "file-path", + "exists": true, "pattern": "^\\S+\\.f(ast)?q\\.gz$", "errorMessage": "FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" }, "fastq_2": { - "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", - "anyOf": [ - { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$" - }, - { - "type": "string", - "maxLength": 0 - } - ] + "type": "string", + "format": "file-path", + "exists": true, + "pattern": "^\\S+\\.f(ast)?q\\.gz$", + "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" }, "barcode": { "type": "integer", diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py deleted file mode 100755 index f866fd86..00000000 --- a/bin/check_samplesheet.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import errno -import argparse - - -def parse_args(args=None): - Description = "Reformat nf-core/viralrecon samplesheet file and check its contents." - Epilog = "Example usage: python check_samplesheet.py " - - parser = argparse.ArgumentParser(description=Description, epilog=Epilog) - parser.add_argument("FILE_IN", help="Input samplesheet file.") - parser.add_argument("FILE_OUT", help="Output file.") - parser.add_argument( - "-pl", - "--platform", - type=str, - dest="PLATFORM", - default="illumina", - help="Sequencing platform for input data. Accepted values = 'illumina' or 'nanopore' (default: 'illumina').", - ) - return parser.parse_args(args) - - -def make_dir(path): - if len(path) > 0: - try: - os.makedirs(path) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise exception - - -def print_error(error, context="Line", context_str=""): - error_str = "ERROR: Please check samplesheet -> {}".format(error) - if context != "" and context_str != "": - error_str = "ERROR: Please check samplesheet -> {}\n{}: '{}'".format( - error, context.strip(), context_str.strip() - ) - print(error_str) - sys.exit(1) - - -def check_illumina_samplesheet(file_in, file_out): - """ - This function checks that the samplesheet follows the following structure: - - sample,fastq_1,fastq_2 - SAMPLE_PE,SAMPLE_PE_RUN1_1.fastq.gz,SAMPLE_PE_RUN1_2.fastq.gz - SAMPLE_PE,SAMPLE_PE_RUN2_1.fastq.gz,SAMPLE_PE_RUN2_2.fastq.gz - SAMPLE_SE,SAMPLE_SE_RUN1_1.fastq.gz, - - For an example see: - https://github.com/nf-core/test-datasets/blob/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv - """ - - sample_mapping_dict = {} - with open(file_in, "r") as fin: - ## Check header - MIN_COLS = 2 - HEADER = ["sample", "fastq_1", "fastq_2"] - header = [x.strip('"') for x in fin.readline().strip().split(",")] - if header[: len(HEADER)] != HEADER: - print("ERROR: Please check samplesheet header -> {} != {}".format(",".join(header), ",".join(HEADER))) - sys.exit(1) - - ## Check sample entries - for line in fin: - lspl = [x.strip().strip('"') for x in line.strip().split(",")] - - # Check valid number of columns per row - if len(lspl) < len(HEADER): - print_error( - "Invalid number of columns (minimum = {})!".format(len(HEADER)), - "Line", - line, - ) - num_cols = len([x for x in lspl if x]) - if num_cols < MIN_COLS: - print_error( - "Invalid number of populated columns (minimum = {})!".format(MIN_COLS), - "Line", - line, - ) - - ## Check sample name entries - sample, fastq_1, fastq_2 = lspl[: len(HEADER)] - if sample.find(" ") != -1: - print(f"WARNING: Spaces have been replaced by underscores for sample: {sample}") - sample = sample.replace(" ", "_") - if not sample: - print_error("Sample entry has not been specified!", "Line", line) - - ## Check FastQ file extension - for fastq in [fastq_1, fastq_2]: - if fastq: - if fastq.find(" ") != -1: - print_error("FastQ file contains spaces!", "Line", line) - if not fastq.endswith(".fastq.gz") and not fastq.endswith(".fq.gz"): - print_error( - "FastQ file does not have extension '.fastq.gz' or '.fq.gz'!", - "Line", - line, - ) - - ## Auto-detect paired-end/single-end - sample_info = [] ## [single_end, fastq_1, fastq_2] - if sample and fastq_1 and fastq_2: ## Paired-end short reads - sample_info = ["0", fastq_1, fastq_2] - elif sample and fastq_1 and not fastq_2: ## Single-end short reads - sample_info = ["1", fastq_1, fastq_2] - else: - print_error("Invalid combination of columns provided!", "Line", line) - - ## Create sample mapping dictionary = { sample: [ single_end, fastq_1, fastq_2 ] } - if sample not in sample_mapping_dict: - sample_mapping_dict[sample] = [sample_info] - else: - if sample_info in sample_mapping_dict[sample]: - print_error("Samplesheet contains duplicate rows!", "Line", line) - else: - sample_mapping_dict[sample].append(sample_info) - - ## Write validated samplesheet with appropriate columns - if len(sample_mapping_dict) > 0: - out_dir = os.path.dirname(file_out) - make_dir(out_dir) - with open(file_out, "w") as fout: - fout.write(",".join(["sample", "single_end", "fastq_1", "fastq_2"]) + "\n") - for sample in sorted(sample_mapping_dict.keys()): - ## Check that multiple runs of the same sample are of the same datatype - if not all(x[0] == sample_mapping_dict[sample][0][0] for x in sample_mapping_dict[sample]): - print_error( - "Multiple runs of a sample must be of the same datatype!", - "Sample: {}".format(sample), - ) - - for idx, val in enumerate(sample_mapping_dict[sample]): - fout.write(",".join(["{}_T{}".format(sample, idx + 1)] + val) + "\n") - else: - print_error("No entries to process!", "Samplesheet: {}".format(file_in)) - - -def check_nanopore_samplesheet(file_in, file_out): - """ - This function checks that the samplesheet follows the following structure: - - sample,barcode - SAMPLE_N,1 - SAMPLE_X,2 - SAMPLE_Z,3 - - For an example see: - https://github.com/nf-core/test-datasets/blob/viralrecon/samplesheet/samplesheet_test_nanopore.csv - """ - - sample_mapping_dict = {} - with open(file_in, "r") as fin: - ## Check header - MIN_COLS = 2 - HEADER = ["sample", "barcode"] - header = [x.strip('"') for x in fin.readline().strip().split(",")] - if header[: len(HEADER)] != HEADER: - print("ERROR: Please check samplesheet header -> {} != {}".format(",".join(header), ",".join(HEADER))) - sys.exit(1) - - ## Check sample entries - for line in fin: - lspl = [x.strip().strip('"') for x in line.strip().split(",")] - - # Check valid number of columns per row - if len(lspl) < len(HEADER): - print_error( - "Invalid number of columns (minimum = {})!".format(len(HEADER)), - "Line", - line, - ) - num_cols = len([x for x in lspl if x]) - if num_cols < MIN_COLS: - print_error( - "Invalid number of populated columns (minimum = {})!".format(MIN_COLS), - "Line", - line, - ) - - ## Check sample entry - sample, barcode = lspl[: len(HEADER)] - if sample.find(" ") != -1: - print(f"WARNING: Spaces have been replaced by underscores for sample: {sample}") - sample = sample.replace(" ", "_") - if sample.find("-") != -1: - print(f"WARNING: Dashes have been replaced by underscores for sample: {sample}") - sample = sample.replace("-", "_") - if not sample: - print_error("Sample entry has not been specified!", "Line", line) - - ## Check barcode entry - if barcode: - if not barcode.isdigit(): - print_error("Barcode entry is not an integer!", "Line", line) - else: - barcode = "barcode%s" % (barcode.zfill(2)) - - ## Create sample mapping dictionary = { sample: barcode } - if barcode in sample_mapping_dict.values(): - print_error( - "Samplesheet contains duplicate entries in the 'barcode' column!", - "Line", - line, - ) - if sample not in sample_mapping_dict: - sample_mapping_dict[sample] = barcode - else: - print_error( - "Samplesheet contains duplicate entries in the 'sample' column!", - "Line", - line, - ) - - ## Write validated samplesheet with appropriate columns - if len(sample_mapping_dict) > 0: - out_dir = os.path.dirname(file_out) - make_dir(out_dir) - with open(file_out, "w") as fout: - fout.write(",".join(["sample", "barcode"]) + "\n") - for sample in sorted(sample_mapping_dict.keys()): - fout.write(",".join([sample, sample_mapping_dict[sample]]) + "\n") - else: - print_error("No entries to process!", "Samplesheet: {}".format(file_in)) - - -def main(args=None): - args = parse_args(args) - - if args.PLATFORM == "illumina": - check_illumina_samplesheet(args.FILE_IN, args.FILE_OUT) - elif args.PLATFORM == "nanopore": - check_nanopore_samplesheet(args.FILE_IN, args.FILE_OUT) - else: - print( - "Unrecognised option passed to --platform: {}. Accepted values = 'illumina' or 'nanopore'".format( - args.PLATFORM - ) - ) - sys.exit(1) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy deleted file mode 100755 index 2b5156b8..00000000 --- a/lib/NfcoreTemplate.groovy +++ /dev/null @@ -1,397 +0,0 @@ -// -// This file holds several functions used within the nf-core pipeline template. -// - -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput -import nextflow.extension.FilesEx - -class NfcoreTemplate { - - // - // Check AWS Batch related parameters have been specified correctly - // - public static void awsBatch(workflow, params) { - if (workflow.profile.contains('awsbatch')) { - // Check params.awsqueue and params.awsregion have been set if running on AWSBatch - assert (params.awsqueue && params.awsregion) : "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" - // Check outdir paths to be S3 buckets if running on AWSBatch - assert params.outdir.startsWith('s3:') : "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" - } - } - - // - // Warn if a -profile or Nextflow config has not been provided to run the pipeline - // - public static void checkConfigProvided(workflow, log) { - if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { - log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + - "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + - " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + - " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + - " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + - "Please refer to the quick start section and usage docs for the pipeline.\n " - } - } - - // - // Warn if using custom configs to provide pipeline parameters - // - public static void warnParamsProvidedInConfig(workflow, log) { - if (workflow.configFiles.size() > 1) { - log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " Multiple config files detected!\n" + - " Please provide pipeline parameters via the CLI or Nextflow '-params-file' option.\n" + - " Custom config files including those provided by the '-c' Nextflow option can be\n" + - " used to provide any configuration except for parameters.\n\n" + - " Docs: https://nf-co.re/usage/configuration#custom-configuration-files\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - } - } - - // - // Generate version string - // - public static String version(workflow) { - String version_string = "" - - if (workflow.manifest.version) { - def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' - version_string += "${prefix_v}${workflow.manifest.version}" - } - - if (workflow.commitId) { - def git_shortsha = workflow.commitId.substring(0, 7) - version_string += "-g${git_shortsha}" - } - - return version_string - } - - // - // Construct and send completion email - // - public static void email(workflow, params, summary_params, projectDir, log, multiqc_report=[], fail_mapped_reads=[:]) { - - // Set up the e-mail variables - def subject = "[$workflow.manifest.name] Successful: $workflow.runName" - if (fail_mapped_reads.size() > 0) { - subject = "[$workflow.manifest.name] Partially successful (${fail_mapped_reads.size()} skipped): $workflow.runName" - } - if (!workflow.success) { - subject = "[$workflow.manifest.name] FAILED: $workflow.runName" - } - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['Date Started'] = workflow.start - misc_fields['Date Completed'] = workflow.complete - misc_fields['Pipeline script file path'] = workflow.scriptFile - misc_fields['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository - if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId - if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision - misc_fields['Nextflow Version'] = workflow.nextflow.version - misc_fields['Nextflow Build'] = workflow.nextflow.build - misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - - def email_fields = [:] - email_fields['version'] = NfcoreTemplate.version(workflow) - email_fields['runName'] = workflow.runName - email_fields['success'] = workflow.success - email_fields['dateComplete'] = workflow.complete - email_fields['duration'] = workflow.duration - email_fields['exitStatus'] = workflow.exitStatus - email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - email_fields['errorReport'] = (workflow.errorReport ?: 'None') - email_fields['commandLine'] = workflow.commandLine - email_fields['projectDir'] = workflow.projectDir - email_fields['summary'] = summary << misc_fields - - // On success try attach the multiqc report - def mqc_report = null - try { - if (workflow.success && !params.skip_multiqc) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] - } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" - } - } - - // Check if we are only sending emails on failure - def email_address = params.email - if (!params.email && params.email_on_fail && !workflow.success) { - email_address = params.email_on_fail - } - - // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("$projectDir/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() - - // Render the HTML template - def hf = new File("$projectDir/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() - - // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] - def sf = new File("$projectDir/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() - - // Send the HTML e-mail - Map colors = logColours(params.monochrome_logs) - if (email_address) { - try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") - sendmail_tf.withWriter { w -> w << sendmail_html } - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" - } catch (all) { - // Catch failures and try with plaintext - def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report != null && mqc_report.size() <= max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] - } - mail_cmd.execute() << email_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" - } - } - - // Write summary e-mail HTML to a file - def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - FilesEx.copyTo(output_hf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.html"); - output_hf.delete() - - // Write summary e-mail TXT to a file - def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - FilesEx.copyTo(output_tf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.txt"); - output_tf.delete() - } - - // - // Construct and send a notification to a web server as JSON - // e.g. Microsoft Teams and Slack - // - public static void IM_notification(workflow, params, summary_params, projectDir, log) { - def hook_url = params.hook_url - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) misc_fields['repository'] = workflow.repository - if (workflow.commitId) misc_fields['commitid'] = workflow.commitId - if (workflow.revision) misc_fields['revision'] = workflow.revision - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = NfcoreTemplate.version(workflow) - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("$projectDir/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection(); - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")); - def postRC = post.getResponseCode(); - if (! postRC.equals(200)) { - log.warn(post.getErrorStream().getText()); - } - } - - // - // Dump pipeline parameters in a json file - // - public static void dump_parameters(workflow, params) { - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def filename = "params_${timestamp}.json" - def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = JsonOutput.toJson(params) - temp_pf.text = JsonOutput.prettyPrint(jsonStr) - - FilesEx.copyTo(temp_pf.toPath(), "${params.outdir}/pipeline_info/params_${timestamp}.json") - temp_pf.delete() - } - - // - // Print pipeline summary on completion - // - public static void summary(workflow, params, log, fail_mapped_reads=[:], pass_mapped_reads=[:]) { - Map colors = logColours(params.monochrome_logs) - - if (pass_mapped_reads.size() > 0) { - def idx = 0 - def samp_aln = '' - def total_aln_count = pass_mapped_reads.size() + fail_mapped_reads.size() - for (samp in pass_mapped_reads) { - samp_aln += " ${samp.value}: ${samp.key}\n" - idx += 1 - if (idx > 5) { - samp_aln += " ..see pipeline reports for full list\n" - break; - } - } - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} ${pass_mapped_reads.size()}/$total_aln_count samples passed Bowtie2 ${params.min_mapped_reads} mapped read threshold:\n${samp_aln}${colors.reset}-" - } - if (fail_mapped_reads.size() > 0) { - def samp_aln = '' - for (samp in fail_mapped_reads) { - samp_aln += " ${samp.value}: ${samp.key}\n" - } - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} ${fail_mapped_reads.size()} samples skipped since they failed Bowtie2 ${params.min_mapped_reads} mapped read threshold:\n${samp_aln}${colors.reset}-" - } - - if (workflow.success) { - if (workflow.stats.ignoredCount == 0) { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" - } - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" - } - } - - // - // ANSII Colours used for terminal logging - // - public static Map logColours(Boolean monochrome_logs) { - Map colorcodes = [:] - - // Reset / Meta - colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" - colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" - colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" - colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" - colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" - colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" - colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" - - // Regular Colors - colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" - colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" - colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" - colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" - colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" - colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" - colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" - colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" - - // Bold - colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" - colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" - colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" - colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" - colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" - colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" - colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" - colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" - - // Underline - colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" - colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" - colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" - colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" - colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" - colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" - colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" - colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" - - // High Intensity - colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" - colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" - colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" - colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" - colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" - colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" - colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" - colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" - - // Bold High Intensity - colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" - colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" - colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" - colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" - colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" - colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" - colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" - colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" - - return colorcodes - } - - // - // Does what is says on the tin - // - public static String dashedLine(monochrome_logs) { - Map colors = logColours(monochrome_logs) - return "-${colors.dim}----------------------------------------------------${colors.reset}-" - } - - // - // nf-core logo - // - public static String logo(workflow, monochrome_logs) { - Map colors = logColours(monochrome_logs) - String workflow_version = NfcoreTemplate.version(workflow) - String.format( - """\n - ${dashedLine(monochrome_logs)} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${workflow_version}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) - } -} diff --git a/lib/Utils.groovy b/lib/Utils.groovy deleted file mode 100644 index fd6095c7..00000000 --- a/lib/Utils.groovy +++ /dev/null @@ -1,47 +0,0 @@ -// -// This file holds several Groovy functions that could be useful for any Nextflow pipeline -// - -import org.yaml.snakeyaml.Yaml - -class Utils { - - // - // When running with -profile conda, warn if channels have not been set-up appropriately - // - public static void checkCondaChannels(log) { - Yaml parser = new Yaml() - def channels = [] - try { - def config = parser.load("conda config --show channels".execute().text) - channels = config.channels - } catch(NullPointerException | IOException e) { - log.warn "Could not verify conda channel configuration." - return - } - - // Check that all channels are present - // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] - def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean - - // Check that they are in the right order - def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) - } - - if (channels_missing | channel_priority_violation) { - log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " There is a problem with your Conda configuration!\n\n" + - " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/\n" + - " The observed channel order is \n" + - " ${channels}\n" + - " but the following channel order is required:\n" + - " ${required_channels_in_order}\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - } - } -} diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy deleted file mode 100755 index d6fb74e2..00000000 --- a/lib/WorkflowMain.groovy +++ /dev/null @@ -1,132 +0,0 @@ -// -// This file holds several functions specific to the main.nf workflow in the nf-core/viralrecon pipeline -// - -import nextflow.Nextflow - -class WorkflowMain { - - // - // Citation string for pipeline - // - public static String citation(workflow) { - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + - "* The pipeline\n" + - " https://doi.org/10.5281/zenodo.3901628\n\n" + - "* The nf-core framework\n" + - " https://doi.org/10.1038/s41587-020-0439-x\n\n" + - "* Software dependencies\n" + - " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" - } - - // - // Validate parameters and print summary to screen - // - public static void initialise(workflow, params, log, args) { - - // Print workflow version and exit on --version - if (params.version) { - String workflow_version = NfcoreTemplate.version(workflow) - log.info "${workflow.manifest.name} ${workflow_version}" - System.exit(0) - } - - // Warn about using custom configs to provide pipeline parameters - NfcoreTemplate.warnParamsProvidedInConfig(workflow, log) - - // Check that a -profile or Nextflow config has been provided to run the pipeline - NfcoreTemplate.checkConfigProvided(workflow, log) - // Check that the profile doesn't contain spaces and doesn't end with a trailing comma - checkProfile(workflow.profile, args, log) - - // Check that conda channels are set-up correctly - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - Utils.checkCondaChannels(log) - } - - // Check AWS batch settings - NfcoreTemplate.awsBatch(workflow, params) - - // Check sequencing platform - def platformList = ['illumina', 'nanopore'] - if (!params.platform) { - Nextflow.error("Platform not specified with e.g. '--platform illumina'. Valid options: ${platformList.join(', ')}.") - } else if (!platformList.contains(params.platform)) { - Nextflow.error("Invalid platform option: '${params.platform}'. Valid options: ${platformList.join(', ')}.") - } - - // Check Nextclade dataset parameters - if (!params.skip_consensus && !params.skip_nextclade) { - if (!params.nextclade_dataset && !params.nextclade_dataset_name) { - Nextflow.error("Nextclade dataset not specified with '--nextclade_dataset' or '--nextclade_dataset_name'. A list of available datasets can be obtained using the Nextclade 'nextclade dataset list' command.") - } - } - } - // - // Get attribute from genome config file e.g. fasta - // - public static String getGenomeAttribute(params, attribute, log, primer_set='', primer_set_version=0) { - def val = '' - def support_link = " The default genome config used by the pipeline can be found here:\n" + - " - https://github.com/nf-core/configs/blob/master/conf/pipeline/viralrecon/genomes.config\n\n" + - " If you would still like to blame us please come and find us on nf-core Slack:\n" + - " - https://nf-co.re/viralrecon#contributions-and-support\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { - def genome_map = params.genomes[ params.genome ] - if (primer_set) { - if (genome_map.containsKey('primer_sets')) { - genome_map = genome_map[ 'primer_sets' ] - if (genome_map.containsKey(primer_set)) { - genome_map = genome_map[ primer_set ] - primer_set_version = primer_set_version.toString() - if (genome_map.containsKey(primer_set_version)) { - genome_map = genome_map[ primer_set_version ] - } else { - Nextflow.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " --primer_set_version '${primer_set_version}' not found!\n\n" + - " Currently, the available primer set version keys are: ${genome_map.keySet().join(", ")}\n\n" + - " Please check:\n" + - " - The value provided to --primer_set_version (currently '${primer_set_version}')\n" + - " - The value provided to --primer_set (currently '${primer_set}')\n" + - " - The value provided to --genome (currently '${params.genome}')\n" + - " - Any custom config files provided to the pipeline.\n\n" + support_link) - } - } else { - Nextflow.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " --primer_set '${primer_set}' not found!\n\n" + - " Currently, the available primer set keys are: ${genome_map.keySet().join(", ")}\n\n" + - " Please check:\n" + - " - The value provided to --primer_set (currently '${primer_set}')\n" + - " - The value provided to --genome (currently '${params.genome}')\n" + - " - Any custom config files provided to the pipeline.\n\n" + support_link) - } - } else { - Nextflow.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " Genome '${params.genome}' does not contain any primer sets!\n\n" + - " Please check:\n" + - " - The value provided to --genome (currently '${params.genome}')\n" + - " - Any custom config files provided to the pipeline.\n\n" + support_link) - } - } - if (genome_map.containsKey(attribute)) { - val = genome_map[ attribute ] - } else if (params.genomes[ params.genome ].containsKey(attribute)) { - val = params.genomes[ params.genome ][ attribute ] - } - } - return val - } - - // - // Exit pipeline if --profile contains spaces - // - private static void checkProfile(profile, args, log) { - if (profile.endsWith(',')) { - Nextflow.error "Profile cannot end with a trailing comma. Please remove the comma from the end of the profile string.\nHint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." - } - if (args[0]) { - log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${args[0]}` has been detected.\n Hint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." - } - } -} diff --git a/main.nf b/main.nf index c13fc921..44bf00b6 100644 --- a/main.nf +++ b/main.nf @@ -11,6 +11,24 @@ nextflow.enable.dsl = 2 +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +if (params.platform == 'illumina') { + include { ILLUMINA } from './workflows/illumina' +} else if (params.platform == 'nanopore') { + include { NANOPORE } from './workflows/nanopore' +} + +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_viralrecon_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_viralrecon_pipeline' + +include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_viralrecon_pipeline' + + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GENOME PARAMETER VALUES @@ -25,74 +43,82 @@ if (params.platform == 'illumina' && params.protocol == 'amplicon') { } else if (params.platform == 'nanopore') { primer_set = params.primer_set primer_set_version = params.primer_set_version - params.artic_scheme = WorkflowMain.getGenomeAttribute(params, 'scheme', log, primer_set, primer_set_version) + params.artic_scheme = getGenomeAttribute('scheme', primer_set, primer_set_version) } -params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta' , log, primer_set, primer_set_version) -params.gff = WorkflowMain.getGenomeAttribute(params, 'gff' , log, primer_set, primer_set_version) -params.bowtie2_index = WorkflowMain.getGenomeAttribute(params, 'bowtie2' , log, primer_set, primer_set_version) -params.primer_bed = WorkflowMain.getGenomeAttribute(params, 'primer_bed', log, primer_set, primer_set_version) +params.fasta = getGenomeAttribute('fasta') +params.gff = getGenomeAttribute('gff') +params.bowtie2_index = getGenomeAttribute('bowtie2') +params.primer_bed = getGenomeAttribute('primer_bed', primer_set, primer_set_version) -params.nextclade_dataset = WorkflowMain.getGenomeAttribute(params, 'nextclade_dataset' , log, primer_set, primer_set_version) -params.nextclade_dataset_name = WorkflowMain.getGenomeAttribute(params, 'nextclade_dataset_name' , log, primer_set, primer_set_version) -params.nextclade_dataset_reference = WorkflowMain.getGenomeAttribute(params, 'nextclade_dataset_reference', log, primer_set, primer_set_version) -params.nextclade_dataset_tag = WorkflowMain.getGenomeAttribute(params, 'nextclade_dataset_tag' , log, primer_set, primer_set_version) +params.nextclade_dataset = getGenomeAttribute('nextclade_dataset') +params.nextclade_dataset_name = getGenomeAttribute('nextclade_dataset_name') +params.nextclade_dataset_reference = getGenomeAttribute('nextclade_dataset_reference') +params.nextclade_dataset_tag = getGenomeAttribute('nextclade_dataset_tag') /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - VALIDATE & PRINT PARAMETER SUMMARY + NAMED WORKFLOWS FOR PIPELINE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { validateParameters; paramsHelp } from 'plugin/nf-validation' - -// Print help message if needed -if (params.help) { - def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) - def citation = '\n' + WorkflowMain.citation(workflow) + '\n' - def String command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" - log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) - System.exit(0) -} - -// Validate input parameters -if (params.validate_params) { - validateParameters() -} -WorkflowMain.initialise(workflow, params, log, args) -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - NAMED WORKFLOW FOR PIPELINE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ +// +// WORKFLOW: Run main nf-core/viralrecon analysis pipeline depending on type of input +// +workflow NFCORE_VIRALRECON { -if (params.platform == 'illumina') { - include { ILLUMINA } from './workflows/illumina' -} else if (params.platform == 'nanopore') { - include { NANOPORE } from './workflows/nanopore' -} + take: + samplesheet // channel: samplesheet read in from --input -workflow NFCORE_VIRALRECON { + main: // - // WORKFLOW: Variant and de novo assembly analysis for Illumina data + // WORKFLOW: Run pipeline // + multiqc_report = Channel.empty() + if (params.platform == 'illumina') { - ILLUMINA () + ILLUMINA ( + samplesheet, + params.fasta, + params.gff, + params.primer_bed, + params.bowtie2_index, + params.nextclade_dataset, + params.nextclade_dataset_name, + params.nextclade_dataset_reference, + params.nextclade_dataset_tag + ) + + multiqc_report = ILLUMINA.out.multiqc_report - // - // WORKFLOW: Variant analysis for Nanopore data - // } else if (params.platform == 'nanopore') { - NANOPORE () + NANOPORE ( + samplesheet, + params.fasta, + params.gff, + params.primer_bed, + params.artic_scheme, + params.bowtie2_index, + params.nextclade_dataset, + params.nextclade_dataset_name, + params.nextclade_dataset_reference, + params.nextclade_dataset_tag + ) + + multiqc_report = NANOPORE.out.multiqc_report } + + emit: + multiqc_report // channel: /path/to/multiqc_report.html + } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN ALL WORKFLOWS + RUN MAIN WORKFLOW ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -102,7 +128,41 @@ workflow NFCORE_VIRALRECON { // workflow { - NFCORE_VIRALRECON () + + main: + + // + // SUBWORKFLOW: Run initialisation tasks + // + PIPELINE_INITIALISATION ( + params.version, + params.help, + params.validate_params, + params.monochrome_logs, + args, + params.outdir, + params.input + ) + + // + // WORKFLOW: Run main workflow + // + NFCORE_VIRALRECON ( + PIPELINE_INITIALISATION.out.samplesheet + ) + + // + // SUBWORKFLOW: Run completion tasks + // + PIPELINE_COMPLETION ( + params.email, + params.email_on_fail, + params.plaintext_email, + params.outdir, + params.monochrome_logs, + params.hook_url, + NFCORE_VIRALRECON.out.multiqc_report + ) } /* diff --git a/modules.json b/modules.json index e367b51c..52526931 100644 --- a/modules.json +++ b/modules.json @@ -32,7 +32,7 @@ }, "bcftools/filter": { "branch": "master", - "git_sha": "44096c08ffdbc694f5f92ae174ea0f7ba0f37e09", + "git_sha": "a3893076a76e91b3ff152faddf872f00778fb224", "installed_by": ["modules"] }, "bcftools/mpileup": { @@ -52,12 +52,12 @@ }, "bcftools/sort": { "branch": "master", - "git_sha": "44096c08ffdbc694f5f92ae174ea0f7ba0f37e09", + "git_sha": "487d92367b4d7bb9f1ca694bf72736be90720b15", "installed_by": ["modules"] }, "bcftools/stats": { "branch": "master", - "git_sha": "44096c08ffdbc694f5f92ae174ea0f7ba0f37e09", + "git_sha": "618364f55cb88f6c283f6c6c45c24d5f9f08f998", "installed_by": ["modules"] }, "bedtools/getfasta": { @@ -72,7 +72,7 @@ }, "bedtools/merge": { "branch": "master", - "git_sha": "575e1bc54b083fb15e7dd8b5fcc40bea60e8ce83", + "git_sha": "a5377837fe9013bde89de8689829e83e84086536", "installed_by": ["modules"] }, "blast/blastn": { @@ -87,7 +87,7 @@ }, "bowtie2/align": { "branch": "master", - "git_sha": "3c77ca9aac783e76c3614a06db3bfe4fef619bde", + "git_sha": "0fe30831abbc2ed115e46e92330edf38f56edc3d", "installed_by": ["fastq_align_bowtie2"] }, "bowtie2/build": { @@ -97,12 +97,7 @@ }, "cat/fastq": { "branch": "master", - "git_sha": "02fd5bd7275abad27aad32d5c852e0a9b1b98882", - "installed_by": ["modules"] - }, - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", + "git_sha": "0997b47c93c06b49aa7b3fefda87e728312cf2ca", "installed_by": ["modules"] }, "custom/getchromsizes": { @@ -112,12 +107,12 @@ }, "fastp": { "branch": "master", - "git_sha": "c9488585ce7bd35ccd2a30faa2371454c8112fb9", + "git_sha": "95cf5fe0194c7bf5cb0e3027a2eb7e7c89385080", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", + "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["modules"] }, "freyja/boot": { @@ -162,7 +157,7 @@ }, "kraken2/kraken2": { "branch": "master", - "git_sha": "8fae4ee738f645812384d6aba9d0dc651ad79ff9", + "git_sha": "ca87ad032a62f025f0c373facacef2df0c5411b2", "installed_by": ["modules"] }, "minia": { @@ -172,12 +167,12 @@ }, "mosdepth": { "branch": "master", - "git_sha": "69e3eb17fb31b772b18f134d6e8f8b93ee980e65", + "git_sha": "30d3ca4346ae38f0de821c57a9c517b8b0b135d6", "installed_by": ["modules"] }, "nanoplot": { "branch": "master", - "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", + "git_sha": "a31407dfaf0cb0d04768d5cb439fc6f4523a6981", "installed_by": ["modules"] }, "nextclade/datasetget": { @@ -202,7 +197,7 @@ }, "picard/markduplicates": { "branch": "master", - "git_sha": "ec833ac4c29db6005d18baccf3306f557c46b006", + "git_sha": "1943aa60f7490c3d6740e8872e6e69122ccc8087", "installed_by": ["bam_markduplicates_picard"] }, "plasmidid": { @@ -222,32 +217,32 @@ }, "samtools/flagstat": { "branch": "master", - "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c", + "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", "installed_by": ["bam_stats_samtools"] }, "samtools/idxstats": { "branch": "master", - "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c", + "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", "installed_by": ["bam_stats_samtools"] }, "samtools/index": { "branch": "master", - "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c", + "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", "installed_by": ["bam_markduplicates_picard", "bam_sort_stats_samtools"] }, "samtools/sort": { "branch": "master", - "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c", + "git_sha": "4352dbdb09ec40db71e9b172b97a01dcf5622c26", "installed_by": ["bam_sort_stats_samtools"] }, "samtools/stats": { "branch": "master", - "git_sha": "ec833ac4c29db6005d18baccf3306f557c46b006", + "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", "installed_by": ["bam_stats_samtools"] }, "samtools/view": { "branch": "master", - "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c", + "git_sha": "0bd7d2333a88483aa0476acea172e9f5f6dd83bb", "installed_by": ["modules"] }, "spades": { @@ -257,12 +252,12 @@ }, "tabix/bgzip": { "branch": "master", - "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", + "git_sha": "09d3c8c29b31a2dfd610305b10550f0e1dbcd4a9", "installed_by": ["modules"] }, "tabix/tabix": { "branch": "master", - "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", + "git_sha": "9502adb23c0b97ed8e616bbbdfa73b4585aec9a1", "installed_by": ["modules"] }, "unicycler": { @@ -272,7 +267,7 @@ }, "untar": { "branch": "master", - "git_sha": "e719354ba77df0a1bd310836aa2039b45c29d620", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["modules"] }, "vcflib/vcfuniq": { @@ -286,17 +281,17 @@ "nf-core": { "bam_markduplicates_picard": { "branch": "master", - "git_sha": "0c38be7e652a0b2f3a37681ee4c0dbdf85677647", + "git_sha": "1943aa60f7490c3d6740e8872e6e69122ccc8087", "installed_by": ["subworkflows"] }, "bam_sort_stats_samtools": { "branch": "master", - "git_sha": "0c38be7e652a0b2f3a37681ee4c0dbdf85677647", + "git_sha": "4352dbdb09ec40db71e9b172b97a01dcf5622c26", "installed_by": ["fastq_align_bowtie2"] }, "bam_stats_samtools": { "branch": "master", - "git_sha": "0c38be7e652a0b2f3a37681ee4c0dbdf85677647", + "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", "installed_by": ["bam_markduplicates_picard", "bam_sort_stats_samtools"] }, "bam_variant_demix_boot_freyja": { @@ -306,7 +301,22 @@ }, "fastq_align_bowtie2": { "branch": "master", - "git_sha": "cfd937a668919d948f6fcbf4218e79de50c2f36f", + "git_sha": "55e7bb6d5279ec21f254bbab0943117cb3d2fc35", + "installed_by": ["subworkflows"] + }, + "utils_nextflow_pipeline": { + "branch": "master", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "installed_by": ["subworkflows"] + }, + "utils_nfcore_pipeline": { + "branch": "master", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "installed_by": ["subworkflows"] + }, + "utils_nfvalidation_plugin": { + "branch": "master", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] } } diff --git a/modules/local/multiqc_illumina.nf b/modules/local/multiqc_illumina.nf index e87b45a7..8130bf92 100644 --- a/modules/local/multiqc_illumina.nf +++ b/modules/local/multiqc_illumina.nf @@ -7,9 +7,10 @@ process MULTIQC { 'quay.io/biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: - path 'multiqc_config.yaml' - path multiqc_custom_config - path software_versions + path multiqc_files, stageAs: "?/*" + path(multiqc_config) + path(extra_multiqc_config) + path(multiqc_logo) path workflow_summary path fail_reads_summary path fail_mapping_summary @@ -47,10 +48,13 @@ process MULTIQC { script: def args = task.ext.args ?: '' - def custom_config = multiqc_custom_config ? "--config $multiqc_custom_config" : '' + def config = multiqc_config ? "--config $multiqc_config" : '' + def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' + """ ## Run MultiQC once to parse tool logs - multiqc -f $args $custom_config . + multiqc -f $args $config $extra_config $logo. ## Parse YAML files dumped by MultiQC to obtain metrics multiqc_to_custom_csv.py --platform illumina @@ -67,7 +71,7 @@ process MULTIQC { rm -f variants/report.tsv ## Run MultiQC a second time - multiqc -f $args -e general_stats --ignore nextclade_clade_mqc.tsv $custom_config . + multiqc -f $args -e general_stats --ignore nextclade_clade_mqc.tsv $config $extra_config $logo . cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/multiqc_nanopore.nf b/modules/local/multiqc_nanopore.nf index 92ec1f8a..112dd216 100644 --- a/modules/local/multiqc_nanopore.nf +++ b/modules/local/multiqc_nanopore.nf @@ -7,9 +7,10 @@ process MULTIQC { 'quay.io/biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: - path 'multiqc_config.yaml' - path multiqc_custom_config - path software_versions + path multiqc_files, stageAs: "?/*" + path(multiqc_config) + path(extra_multiqc_config) + path(multiqc_logo) path workflow_summary path fail_barcodes_no_sample path fail_no_barcode_samples @@ -39,10 +40,13 @@ process MULTIQC { script: def args = task.ext.args ?: '' - def custom_config = multiqc_custom_config ? "--config $multiqc_custom_config" : '' + def config = multiqc_config ? "--config $multiqc_config" : '' + def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' + """ ## Run MultiQC once to parse tool logs - multiqc -f $args $custom_config . + multiqc -f $args $config $extra_config $logo . ## Parse YAML files dumped by MultiQC to obtain metrics multiqc_to_custom_csv.py --platform nanopore @@ -51,7 +55,7 @@ process MULTIQC { rm -rf quast ## Run MultiQC a second time - multiqc -f $args -e general_stats --ignore *nextclade_clade_mqc.tsv $custom_config . + multiqc -f $args -e general_stats --ignore *nextclade_clade_mqc.tsv $config $extra_config $logo . cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf deleted file mode 100644 index 0da5f587..00000000 --- a/modules/local/samplesheet_check.nf +++ /dev/null @@ -1,33 +0,0 @@ -process SAMPLESHEET_CHECK { - tag "$samplesheet" - label 'process_single' - - conda "conda-forge::python=3.9.5" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/python:3.9--1' : - 'quay.io/biocontainers/python:3.9--1' }" - - input: - path samplesheet - val platform - - output: - path '*.csv' , emit: csv - path "versions.yml", emit: versions - - when: - task.ext.when == null || task.ext.when - - script: // This script is bundled with the pipeline, in nf-core/viralrecon/bin/ - """ - check_samplesheet.py \\ - $samplesheet \\ - samplesheet.valid.csv \\ - --platform $platform - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS - """ -} diff --git a/modules/nf-core/bcftools/filter/main.nf b/modules/nf-core/bcftools/filter/main.nf index 1a40cb93..cc9a2361 100644 --- a/modules/nf-core/bcftools/filter/main.nf +++ b/modules/nf-core/bcftools/filter/main.nf @@ -43,6 +43,7 @@ process BCFTOOLS_FILTER { """ stub: + def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" extension = args.contains("--output-type b") || args.contains("-Ob") ? "bcf.gz" : @@ -53,8 +54,10 @@ process BCFTOOLS_FILTER { if ("$vcf" == "${prefix}.${extension}") error "Input and output names are the same, set prefix in module configuration to disambiguate!" + def create_file = extension.endsWith(".gz") ? "echo '' | gzip > ${prefix}.${extension}" : "touch ${prefix}.${extension}" + """ - touch ${prefix}.${extension} + ${create_file} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/bcftools/filter/meta.yml b/modules/nf-core/bcftools/filter/meta.yml index 5b111fc3..deb93b13 100644 --- a/modules/nf-core/bcftools/filter/meta.yml +++ b/modules/nf-core/bcftools/filter/meta.yml @@ -21,7 +21,7 @@ input: - vcf: type: file description: VCF input file - pattern: "*.{vcf}" + pattern: "*.{vcf,bcf,vcf.gz,bcf.gz}" output: - meta: type: map @@ -31,7 +31,7 @@ output: - vcf: type: file description: VCF filtered output file - pattern: "*.{vcf}" + pattern: "*.{vcf,bcf,vcf.gz,bcf.gz}" - versions: type: file description: File containing software versions diff --git a/modules/nf-core/bcftools/filter/tests/main.nf.test b/modules/nf-core/bcftools/filter/tests/main.nf.test new file mode 100644 index 00000000..eaf100e8 --- /dev/null +++ b/modules/nf-core/bcftools/filter/tests/main.nf.test @@ -0,0 +1,82 @@ +nextflow_process { + + name "Test Process BCFTOOLS_FILTER" + script "../main.nf" + process "BCFTOOLS_FILTER" + + tag "modules" + tag "modules_nfcore" + tag "bcftools" + tag "bcftools/filter" + + config "./nextflow.config" + + test("sarscov2 - vcf") { + + when { + process { + """ + input[0] = [ + [id:"vcf_test"], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match("vcf") } + ) + } + + } + + test("sarscov2 - vcf - bcf output") { + + when { + process { + """ + input[0] = [ + [id:"bcf_test"], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match("bcf output") } + ) + } + + } + + test("sarscov2 - vcf - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [id:"vcf_test"], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match("vcf - stub") } + ) + } + + } + +} diff --git a/modules/nf-core/bcftools/filter/tests/main.nf.test.snap b/modules/nf-core/bcftools/filter/tests/main.nf.test.snap new file mode 100644 index 00000000..f8e17aa0 --- /dev/null +++ b/modules/nf-core/bcftools/filter/tests/main.nf.test.snap @@ -0,0 +1,101 @@ +{ + "vcf": { + "content": [ + { + "0": [ + [ + { + "id": "vcf_test" + }, + "vcf_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "1": [ + "versions.yml:md5,7dc77043f9afb848d942d47a7bc19f67" + ], + "vcf": [ + [ + { + "id": "vcf_test" + }, + "vcf_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "versions": [ + "versions.yml:md5,7dc77043f9afb848d942d47a7bc19f67" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.02.0" + }, + "timestamp": "2024-03-27T16:57:32.940161987" + }, + "bcf output": { + "content": [ + { + "0": [ + [ + { + "id": "bcf_test" + }, + "bcf_test.bcf.gz:md5,c8a304c8d2892039201154153c8cd536" + ] + ], + "1": [ + "versions.yml:md5,7dc77043f9afb848d942d47a7bc19f67" + ], + "vcf": [ + [ + { + "id": "bcf_test" + }, + "bcf_test.bcf.gz:md5,c8a304c8d2892039201154153c8cd536" + ] + ], + "versions": [ + "versions.yml:md5,7dc77043f9afb848d942d47a7bc19f67" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.02.0" + }, + "timestamp": "2024-03-27T16:45:14.586866398" + }, + "vcf - stub": { + "content": [ + { + "0": [ + [ + { + "id": "vcf_test" + }, + "vcf_test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "1": [ + "versions.yml:md5,7dc77043f9afb848d942d47a7bc19f67" + ], + "vcf": [ + [ + { + "id": "vcf_test" + }, + "vcf_test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,7dc77043f9afb848d942d47a7bc19f67" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.02.0" + }, + "timestamp": "2024-03-27T17:05:52.80837892" + } +} \ No newline at end of file diff --git a/modules/nf-core/bcftools/filter/tests/nextflow.config b/modules/nf-core/bcftools/filter/tests/nextflow.config new file mode 100644 index 00000000..4e960c8d --- /dev/null +++ b/modules/nf-core/bcftools/filter/tests/nextflow.config @@ -0,0 +1,3 @@ +process { + ext.args = {"--no-version${meta.id == 'bcf_test' ? ' --output-type b' : ' --output-type z'}"} +} diff --git a/modules/nf-core/bcftools/filter/tests/tags.yml b/modules/nf-core/bcftools/filter/tests/tags.yml new file mode 100644 index 00000000..d5e01080 --- /dev/null +++ b/modules/nf-core/bcftools/filter/tests/tags.yml @@ -0,0 +1,2 @@ +bcftools/filter: + - "modules/nf-core/bcftools/filter/**" diff --git a/modules/nf-core/bcftools/sort/tests/main.nf.test b/modules/nf-core/bcftools/sort/tests/main.nf.test new file mode 100644 index 00000000..fec59cfe --- /dev/null +++ b/modules/nf-core/bcftools/sort/tests/main.nf.test @@ -0,0 +1,32 @@ +nextflow_process { + + name "Test Process BCFTOOLS_SORT" + script "../main.nf" + process "BCFTOOLS_SORT" + + tag "modules" + tag "modules_nfcore" + tag "bcftools" + tag "bcftools/sort" + + test("SarsCov2 VCF") { + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } +} diff --git a/modules/nf-core/bcftools/sort/tests/main.nf.test.snap b/modules/nf-core/bcftools/sort/tests/main.nf.test.snap new file mode 100644 index 00000000..7f599559 --- /dev/null +++ b/modules/nf-core/bcftools/sort/tests/main.nf.test.snap @@ -0,0 +1,35 @@ +{ + "SarsCov2 VCF": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "1": [ + "versions.yml:md5,622bd32d4ff0fac3360cd534ae0f0168" + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "versions": [ + "versions.yml:md5,622bd32d4ff0fac3360cd534ae0f0168" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.2" + }, + "timestamp": "2024-03-18T12:50:10.340362246" + } +} \ No newline at end of file diff --git a/modules/nf-core/bcftools/sort/tests/tags.yml b/modules/nf-core/bcftools/sort/tests/tags.yml new file mode 100644 index 00000000..6e9520dd --- /dev/null +++ b/modules/nf-core/bcftools/sort/tests/tags.yml @@ -0,0 +1,2 @@ +bcftools/sort: + - "modules/nf-core/bcftools/sort/**" diff --git a/modules/nf-core/bcftools/stats/environment.yml b/modules/nf-core/bcftools/stats/environment.yml index 1a969528..7bb40dc0 100644 --- a/modules/nf-core/bcftools/stats/environment.yml +++ b/modules/nf-core/bcftools/stats/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::bcftools=1.18 + - bioconda::htslib=1.18 diff --git a/modules/nf-core/bcftools/stats/tests/main.nf.test b/modules/nf-core/bcftools/stats/tests/main.nf.test new file mode 100644 index 00000000..f027f6b1 --- /dev/null +++ b/modules/nf-core/bcftools/stats/tests/main.nf.test @@ -0,0 +1,182 @@ +nextflow_process { + + name "Test Process BCFTOOLS_STATS" + script "../main.nf" + process "BCFTOOLS_STATS" + + tag "modules" + tag "modules_nfcore" + tag "bcftools" + tag "bcftools/stats" + + test("sarscov2 - vcf_gz") { + + when { + process { + """ + input[0] = [ [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + []] + input[1] = [ [], [] ] + input[2] = [ [], [] ] + input[3] = [ [], [] ] + input[4] = [ [], [] ] + input[5] = [ [], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.version).match("version") }, + { assert snapshot(file(process.out.stats.get(0).get(1)).readLines()[0..5]).match() }, + ) + } + + } + + test("sarscov2 - vcf_gz - regions") { + + when { + process { + """ + input[0] = [ [ id:'regions_test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz.tbi', checkIfExists: true)] + input[1] = [ [id:'regions_test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test3.vcf.gz', checkIfExists: true) ] + input[2] = [ [], [] ] + input[3] = [ [], [] ] + input[4] = [ [], [] ] + input[5] = [ [], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.version).match("regions_version") }, + { assert snapshot(file(process.out.stats.get(0).get(1)).readLines()[0..5]).match() }, + ) + } + + } + + test("sarscov2 - vcf_gz - targets") { + + when { + process { + """ + input[0] = [ [ id:'targets_test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + [] ] + input[1] = [ [], [] ] + input[2] = [ [id:'targets_test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.targets.tsv.gz', checkIfExists: true) + ] + input[3] = [ [], [] ] + input[4] = [ [], [] ] + input[5] = [ [], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.version).match("targets_version") }, + { assert snapshot(file(process.out.stats.get(0).get(1)).readLines()[0..5]).match() }, + ) + } + + } + + test("sarscov2 - vcf_gz - exons") { + + when { + process { + """ + input[0] = [ [ id:'exon_test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + [] ] + input[1] = [ [], [] ] + input[2] = [ [], [] ] + input[3] = [ [], [] ] + input[4] = [ [id: "exon_test"], + file(params.modules_testdata_base_path + 'delete_me/bcftools/stats/exons.tsv.gz', checkIfExists: true) ] + input[5] = [ [], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.version).match("exon_version") }, + { assert snapshot(file(process.out.stats.get(0).get(1)).readLines()[0..5]).match() }, + ) + } + + } + + test("sarscov2 - vcf_gz - reference") { + + when { + process { + """ + input[0] = [ [ id:'ref_test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + [] ] + input[1] = [ [], [] ] + input[2] = [ [], [] ] + input[3] = [ [], [] ] + input[4] = [ [], [] ] + input[5] = [ [id: 'ref_test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.version).match("ref_version") }, + { assert snapshot(file(process.out.stats.get(0).get(1)).readLines()[0..5]).match() }, + ) + } + + } + + + test("sarscov2 - vcf_gz - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + []] + input[1] = [ [], [] ] + input[2] = [ [], [] ] + input[3] = [ [], [] ] + input[4] = [ [], [] ] + input[5] = [ [], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} \ No newline at end of file diff --git a/modules/nf-core/bcftools/stats/tests/main.nf.test.snap b/modules/nf-core/bcftools/stats/tests/main.nf.test.snap new file mode 100644 index 00000000..30691c32 --- /dev/null +++ b/modules/nf-core/bcftools/stats/tests/main.nf.test.snap @@ -0,0 +1,160 @@ +{ + "sarscov2 - vcf_gz - reference": { + "content": [ + [ + "# This file was produced by bcftools stats (1.18+htslib-1.18) and can be plotted using plot-vcfstats.", + "# The command line was:\tbcftools stats --fasta-ref genome.fasta test.vcf.gz", + "#", + "# Definition of sets:", + "# ID\t[2]id\t[3]tab-separated file names", + "ID\t0\ttest.vcf.gz" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-20T11:46:24.34147" + }, + "sarscov2 - vcf_gz - exons": { + "content": [ + [ + "# This file was produced by bcftools stats (1.18+htslib-1.18) and can be plotted using plot-vcfstats.", + "# The command line was:\tbcftools stats --exons exons.tsv.gz test.vcf.gz", + "#", + "# Definition of sets:", + "# ID\t[2]id\t[3]tab-separated file names", + "ID\t0\ttest.vcf.gz" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-20T11:46:18.378716" + }, + "exon_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T10:02:02.530551189" + }, + "ref_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T10:02:06.885381764" + }, + "sarscov2 - vcf_gz - targets": { + "content": [ + [ + "# This file was produced by bcftools stats (1.18+htslib-1.18) and can be plotted using plot-vcfstats.", + "# The command line was:\tbcftools stats --targets-file test2.targets.tsv.gz test.vcf.gz", + "#", + "# Definition of sets:", + "# ID\t[2]id\t[3]tab-separated file names", + "ID\t0\ttest.vcf.gz" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-20T11:46:12.48194" + }, + "targets_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T10:01:58.412147664" + }, + "sarscov2 - vcf_gz - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.bcftools_stats.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,5909d472a49b0aa2bfbbb1094c129e48" + ], + "stats": [ + [ + { + "id": "test" + }, + "test.bcftools_stats.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,5909d472a49b0aa2bfbbb1094c129e48" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T16:26:21.450513562" + }, + "version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T09:57:04.317347424" + }, + "regions_version": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T10:01:54.349855366" + }, + "sarscov2 - vcf_gz": { + "content": [ + [ + "# This file was produced by bcftools stats (1.18+htslib-1.18) and can be plotted using plot-vcfstats.", + "# The command line was:\tbcftools stats test.vcf.gz", + "#", + "# Definition of sets:", + "# ID\t[2]id\t[3]tab-separated file names", + "ID\t0\ttest.vcf.gz" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-20T11:46:01.862297" + }, + "sarscov2 - vcf_gz - regions": { + "content": [ + [ + "# This file was produced by bcftools stats (1.18+htslib-1.18) and can be plotted using plot-vcfstats.", + "# The command line was:\tbcftools stats --regions-file test3.vcf.gz test.vcf.gz", + "#", + "# Definition of sets:", + "# ID\t[2]id\t[3]tab-separated file names", + "ID\t0\ttest.vcf.gz" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-20T11:46:07.296109" + } +} \ No newline at end of file diff --git a/modules/nf-core/bcftools/stats/tests/tags.yml b/modules/nf-core/bcftools/stats/tests/tags.yml new file mode 100644 index 00000000..53c12d92 --- /dev/null +++ b/modules/nf-core/bcftools/stats/tests/tags.yml @@ -0,0 +1,2 @@ +bcftools/stats: + - "modules/nf-core/bcftools/stats/**" diff --git a/modules/nf-core/bedtools/merge/tests/main.nf.test b/modules/nf-core/bedtools/merge/tests/main.nf.test new file mode 100644 index 00000000..95dba8e5 --- /dev/null +++ b/modules/nf-core/bedtools/merge/tests/main.nf.test @@ -0,0 +1,34 @@ +nextflow_process { + + name "Test Process BEDTOOLS_MERGE" + script "../main.nf" + config "./nextflow.config" + process "BEDTOOLS_MERGE" + + tag "modules" + tag "modules_nfcore" + tag "bedtools" + tag "bedtools/merge" + + test("test_bedtools_merge") { + + when { + process { + """ + input[0] = [ [ id:'test'], + file(params.test_data['sarscov2']['genome']['test_bed'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} \ No newline at end of file diff --git a/modules/nf-core/bedtools/merge/tests/main.nf.test.snap b/modules/nf-core/bedtools/merge/tests/main.nf.test.snap new file mode 100644 index 00000000..ee6c4e63 --- /dev/null +++ b/modules/nf-core/bedtools/merge/tests/main.nf.test.snap @@ -0,0 +1,35 @@ +{ + "test_bedtools_merge": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test_out.bed:md5,0cf6ed2b6f470cd44a247da74ca4fe4e" + ] + ], + "1": [ + "versions.yml:md5,2d134badb4cd1e4e903696c7967f28d6" + ], + "bed": [ + [ + { + "id": "test" + }, + "test_out.bed:md5,0cf6ed2b6f470cd44a247da74ca4fe4e" + ] + ], + "versions": [ + "versions.yml:md5,2d134badb4cd1e4e903696c7967f28d6" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-18T17:07:09.721153" + } +} \ No newline at end of file diff --git a/modules/nf-core/bedtools/merge/tests/nextflow.config b/modules/nf-core/bedtools/merge/tests/nextflow.config new file mode 100644 index 00000000..16444e98 --- /dev/null +++ b/modules/nf-core/bedtools/merge/tests/nextflow.config @@ -0,0 +1,7 @@ +process { + + withName: BEDTOOLS_MERGE { + ext.prefix = { "${meta.id}_out" } + } + +} diff --git a/modules/nf-core/bedtools/merge/tests/tags.yml b/modules/nf-core/bedtools/merge/tests/tags.yml new file mode 100644 index 00000000..60c8cad1 --- /dev/null +++ b/modules/nf-core/bedtools/merge/tests/tags.yml @@ -0,0 +1,2 @@ +bedtools/merge: + - "modules/nf-core/bedtools/merge/**" diff --git a/modules/nf-core/bowtie2/align/main.nf b/modules/nf-core/bowtie2/align/main.nf index 8c405ee3..96a7027d 100644 --- a/modules/nf-core/bowtie2/align/main.nf +++ b/modules/nf-core/bowtie2/align/main.nf @@ -10,13 +10,18 @@ process BOWTIE2_ALIGN { input: tuple val(meta) , path(reads) tuple val(meta2), path(index) + tuple val(meta3), path(fasta) val save_unaligned val sort_bam output: - tuple val(meta), path("*.{bam,sam}"), emit: aligned + tuple val(meta), path("*.sam") , emit: sam , optional:true + tuple val(meta), path("*.bam") , emit: bam , optional:true + tuple val(meta), path("*.cram") , emit: cram , optional:true + tuple val(meta), path("*.csi") , emit: csi , optional:true + tuple val(meta), path("*.crai") , emit: crai , optional:true tuple val(meta), path("*.log") , emit: log - tuple val(meta), path("*fastq.gz") , emit: fastq, optional:true + tuple val(meta), path("*fastq.gz") , emit: fastq , optional:true path "versions.yml" , emit: versions when: @@ -39,7 +44,10 @@ process BOWTIE2_ALIGN { def samtools_command = sort_bam ? 'sort' : 'view' def extension_pattern = /(--output-fmt|-O)+\s+(\S+)/ - def extension = (args2 ==~ extension_pattern) ? (args2 =~ extension_pattern)[0][2].toLowerCase() : "bam" + def extension_matcher = (args2 =~ extension_pattern) + def extension = extension_matcher.getCount() > 0 ? extension_matcher[0][2].toLowerCase() : "bam" + def reference = fasta && extension=="cram" ? "--reference ${fasta}" : "" + if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed "s/\\.rev.1.bt2\$//"` @@ -53,7 +61,7 @@ process BOWTIE2_ALIGN { $unaligned \\ $args \\ 2> >(tee ${prefix}.bowtie2.log >&2) \\ - | samtools $samtools_command $args2 --threads $task.cpus -o ${prefix}.${extension} - + | samtools $samtools_command $args2 --threads $task.cpus ${reference} -o ${prefix}.${extension} - if [ -f ${prefix}.unmapped.fastq.1.gz ]; then mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz @@ -82,9 +90,19 @@ process BOWTIE2_ALIGN { } else { create_unmapped = save_unaligned ? "touch ${prefix}.unmapped_1.fastq.gz && touch ${prefix}.unmapped_2.fastq.gz" : "" } + def reference = fasta && extension=="cram" ? "--reference ${fasta}" : "" + if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output" + + def create_index = "" + if (extension == "cram") { + create_index = "touch ${prefix}.crai" + } else if (extension == "bam") { + create_index = "touch ${prefix}.csi" + } """ touch ${prefix}.${extension} + ${create_index} touch ${prefix}.bowtie2.log ${create_unmapped} diff --git a/modules/nf-core/bowtie2/align/tests/cram_crai.config b/modules/nf-core/bowtie2/align/tests/cram_crai.config new file mode 100644 index 00000000..03f1d5e5 --- /dev/null +++ b/modules/nf-core/bowtie2/align/tests/cram_crai.config @@ -0,0 +1,5 @@ +process { + withName: BOWTIE2_ALIGN { + ext.args2 = '--output-fmt cram --write-index' + } +} diff --git a/modules/nf-core/bowtie2/align/tests/main.nf.test b/modules/nf-core/bowtie2/align/tests/main.nf.test index a478d17b..03aeaf9e 100644 --- a/modules/nf-core/bowtie2/align/tests/main.nf.test +++ b/modules/nf-core/bowtie2/align/tests/main.nf.test @@ -6,9 +6,10 @@ nextflow_process { tag "modules" tag "modules_nfcore" tag "bowtie2" + tag "bowtie2/build" tag "bowtie2/align" - test("sarscov2 - fastq, index, false, false - bam") { + test("sarscov2 - fastq, index, fasta, false, false - bam") { setup { run("BOWTIE2_BUILD") { @@ -32,8 +33,9 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -42,7 +44,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -52,7 +54,7 @@ nextflow_process { } - test("sarscov2 - fastq, index, false, false - sam") { + test("sarscov2 - fastq, index, fasta, false, false - sam") { config "./sam.config" setup { @@ -77,8 +79,9 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -87,7 +90,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).readLines()[0..4], + file(process.out.sam[0][1]).readLines()[0..4], process.out.log, process.out.fastq, process.out.versions @@ -97,7 +100,7 @@ nextflow_process { } - test("sarscov2 - fastq, index, false, false - sam2") { + test("sarscov2 - fastq, index, fasta, false, false - sam2") { config "./sam2.config" setup { @@ -122,8 +125,9 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -132,7 +136,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).readLines()[0..4], + file(process.out.sam[0][1]).readLines()[0..4], process.out.log, process.out.fastq, process.out.versions @@ -142,7 +146,7 @@ nextflow_process { } - test("sarscov2 - fastq, index, false, true - bam") { + test("sarscov2 - fastq, index, fasta, false, true - bam") { setup { run("BOWTIE2_BUILD") { @@ -166,8 +170,9 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = true //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -176,7 +181,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -186,7 +191,7 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], index, false, false - bam") { + test("sarscov2 - [fastq1, fastq2], index, fasta, false, false - bam") { setup { run("BOWTIE2_BUILD") { @@ -213,8 +218,9 @@ nextflow_process { ] ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -223,7 +229,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -233,7 +239,7 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], index, false, true - bam") { + test("sarscov2 - [fastq1, fastq2], index, fasta, false, true - bam") { setup { run("BOWTIE2_BUILD") { @@ -260,8 +266,9 @@ nextflow_process { ] ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = true //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -270,7 +277,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -280,7 +287,7 @@ nextflow_process { } - test("sarscov2 - fastq, large_index, false, false - bam") { + test("sarscov2 - fastq, large_index, fasta, false, false - bam") { config "./large_index.config" setup { @@ -305,8 +312,9 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -315,7 +323,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -325,7 +333,7 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], large_index, false, false - bam") { + test("sarscov2 - [fastq1, fastq2], large_index, fasta, false, false - bam") { config "./large_index.config" setup { @@ -353,8 +361,9 @@ nextflow_process { ] ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -363,7 +372,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -373,7 +382,7 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], index, true, false - bam") { + test("sarscov2 - [fastq1, fastq2], index, fasta, true, false - bam") { setup { run("BOWTIE2_BUILD") { @@ -400,8 +409,9 @@ nextflow_process { ] ] input[1] = BOWTIE2_BUILD.out.index - input[2] = true //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -410,7 +420,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -420,7 +430,7 @@ nextflow_process { } - test("sarscov2 - fastq, index, true, false - bam") { + test("sarscov2 - fastq, index, fasta, true, false - bam") { setup { run("BOWTIE2_BUILD") { @@ -444,8 +454,9 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] input[1] = BOWTIE2_BUILD.out.index - input[2] = true //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -454,7 +465,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, process.out.log, process.out.fastq, process.out.versions @@ -465,7 +476,54 @@ nextflow_process { } - test("sarscov2 - [fastq1, fastq2], index, false, false - stub") { + test("sarscov2 - [fastq1, fastq2], index, fasta, true, true - cram") { + + config "./cram_crai.config" + setup { + run("BOWTIE2_BUILD") { + script "../../build/main.nf" + process { + """ + input[0] = [ + [ id:'test'], + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + ] + """ + } + } + } + + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) + ] + ] + input[1] = BOWTIE2_BUILD.out.index + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = true //sort + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot( + file(process.out.cram[0][1]).name, + file(process.out.crai[0][1]).name + ).match() } + ) + } + + } + + test("sarscov2 - [fastq1, fastq2], index, fasta, false, false - stub") { options "-stub" setup { @@ -493,8 +551,9 @@ nextflow_process { ] ] input[1] = BOWTIE2_BUILD.out.index - input[2] = false //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -503,7 +562,8 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, + file(process.out.csi[0][1]).name, file(process.out.log[0][1]).name, process.out.fastq, process.out.versions @@ -513,7 +573,7 @@ nextflow_process { } - test("sarscov2 - fastq, index, true, false - stub") { + test("sarscov2 - fastq, index, fasta, true, false - stub") { options "-stub" setup { @@ -538,8 +598,9 @@ nextflow_process { file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] input[1] = BOWTIE2_BUILD.out.index - input[2] = true //save_unaligned - input[3] = false //sort + input[2] = [[ id:'test'], file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = false //save_unaligned + input[4] = false //sort """ } } @@ -548,14 +609,15 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot( - file(process.out.aligned[0][1]).name, + file(process.out.bam[0][1]).name, + file(process.out.csi[0][1]).name, file(process.out.log[0][1]).name, - file(process.out.fastq[0][1]).name, + process.out.fastq, process.out.versions ).match() } ) } } - + } diff --git a/modules/nf-core/bowtie2/align/tests/main.nf.test.snap b/modules/nf-core/bowtie2/align/tests/main.nf.test.snap index 883dc7ec..028e7da6 100644 --- a/modules/nf-core/bowtie2/align/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie2/align/tests/main.nf.test.snap @@ -1,34 +1,38 @@ { - "sarscov2 - fastq, index, false, false - sam2": { + "sarscov2 - [fastq1, fastq2], large_index, fasta, false, false - bam": { "content": [ - [ - "ERR5069949.2151832\t16\tMT192765.1\t17453\t42\t150M\t*\t0\t0\tACGCACATTGCTAACTAAGGGCACACTAGAACCAGAATATTTCAATTCAGTGTGTAGACTTATGAAAACTATAGGTCCAGACATGTTCCTCGGAACTTGTCGGCGTTGTCCTGCTGAAATTGTTGACACTGTGAGTGCTTTGGTTTATGA\tAAAA - #nf-core-versions tbody:nth-child(even) { - background-color: #f2f2f2; - } - - - - - - - - - - """ - ) - ] - for process, tmp_versions in sorted(versions.items()): - html.append("") - for i, (tool, version) in enumerate(sorted(tmp_versions.items())): - html.append( - dedent( - f"""\\ - - - - - - """ - ) - ) - html.append("") - html.append("
Process Name Software Version
{process if (i == 0) else ''}{tool}{version}
") - return "\\n".join(html) - - -def main(): - """Load all version files and generate merged output.""" - versions_this_module = {} - versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, - } - - with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - - # aggregate versions by the module name (derived from fully-qualified process name) - versions_by_module = {} - for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - - versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", - } - - versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), - } - - with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) - with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - - with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) - - -if __name__ == "__main__": - main() diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test deleted file mode 100644 index b1e1630b..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test +++ /dev/null @@ -1,43 +0,0 @@ -nextflow_process { - - name "Test Process CUSTOM_DUMPSOFTWAREVERSIONS" - script "../main.nf" - process "CUSTOM_DUMPSOFTWAREVERSIONS" - tag "modules" - tag "modules_nfcore" - tag "custom" - tag "dumpsoftwareversions" - tag "custom/dumpsoftwareversions" - - test("Should run without failures") { - when { - process { - """ - def tool1_version = ''' - TOOL1: - tool1: 0.11.9 - '''.stripIndent() - - def tool2_version = ''' - TOOL2: - tool2: 1.9 - '''.stripIndent() - - input[0] = Channel.of(tool1_version, tool2_version).collectFile() - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot( - process.out.versions, - file(process.out.mqc_yml[0]).readLines()[0..10], - file(process.out.yml[0]).readLines()[0..7] - ).match() - } - ) - } - } -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap deleted file mode 100644 index 5f59a936..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap +++ /dev/null @@ -1,33 +0,0 @@ -{ - "Should run without failures": { - "content": [ - [ - "versions.yml:md5,76d454d92244589d32455833f7c1ba6d" - ], - [ - "data: \"\\n\\n \\n \\n \\n \\n \\n \\n \\n\\", - " \\n\\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n \\n \\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n\\n\\n \\n\\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\" - ], - [ - "CUSTOM_DUMPSOFTWAREVERSIONS:", - " python: 3.11.7", - " yaml: 5.4.1", - "TOOL1:", - " tool1: 0.11.9", - "TOOL2:", - " tool2: '1.9'", - "Workflow:" - ] - ], - "timestamp": "2024-01-09T23:01:18.710682" - } -} \ No newline at end of file diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml b/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml deleted file mode 100644 index 405aa24a..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -custom/dumpsoftwareversions: - - modules/nf-core/custom/dumpsoftwareversions/** diff --git a/modules/nf-core/fastp/main.nf b/modules/nf-core/fastp/main.nf index 2a3b679e..4fc19b74 100644 --- a/modules/nf-core/fastp/main.nf +++ b/modules/nf-core/fastp/main.nf @@ -29,7 +29,7 @@ process FASTP { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def adapter_list = adapter_fasta ? "--adapter_fasta ${adapter_fasta}" : "" - def fail_fastq = save_trimmed_fail && meta.single_end ? "--failed_out ${prefix}.fail.fastq.gz" : save_trimmed_fail && !meta.single_end ? "--unpaired1 ${prefix}_1.fail.fastq.gz --unpaired2 ${prefix}_2.fail.fastq.gz" : '' + def fail_fastq = save_trimmed_fail && meta.single_end ? "--failed_out ${prefix}.fail.fastq.gz" : save_trimmed_fail && !meta.single_end ? "--failed_out ${prefix}.paired.fail.fastq.gz --unpaired1 ${prefix}_1.fail.fastq.gz --unpaired2 ${prefix}_2.fail.fastq.gz" : '' // Added soft-links to original fastqs for consistent naming in MultiQC // Use single ended for interleaved. Add --interleaved_in in config. if ( task.ext.args?.contains('--interleaved_in') ) { diff --git a/modules/nf-core/fastp/tests/main.nf.test b/modules/nf-core/fastp/tests/main.nf.test index fa7e5b4d..6f1f4897 100644 --- a/modules/nf-core/fastp/tests/main.nf.test +++ b/modules/nf-core/fastp/tests/main.nf.test @@ -67,9 +67,9 @@ nextflow_process { process.out.reads_fail.collect { file(it[1]).getName() } + process.out.reads_merged.collect { file(it[1]).getName() } ).sort() - ).match("test_fastp_single_end-for_stub_match") + ).match("test_fastp_single_end-_match") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_single_end") } ) } } @@ -116,7 +116,7 @@ nextflow_process { ).sort() ).match("test_fastp_single_end-for_stub_match") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_single_end_stub") } ) } } @@ -196,9 +196,9 @@ nextflow_process { process.out.reads_fail.collect { file(it[1]).getName() } + process.out.reads_merged.collect { file(it[1]).getName() } ).sort() - ).match("test_fastp_paired_end-for_stub_match") + ).match("test_fastp_paired_end_match") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_paired_end") } ) } } @@ -245,13 +245,14 @@ nextflow_process { ).sort() ).match("test_fastp_paired_end-for_stub_match") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_paired_end-stub") } ) } } test("fastp test_fastp_interleaved") { - config './nextflow.config' + + config './nextflow.interleaved.config' when { params { outdir = "$outputDir" @@ -277,7 +278,7 @@ nextflow_process { def html_text = [ "Q20 bases:
Process Name \\", - " \\ Software Version
CUSTOM_DUMPSOFTWAREVERSIONSpython3.11.7
yaml5.4.1
TOOL1tool10.11.9
TOOL2tool21.9
WorkflowNextflow25.719000 K (93.033098%)", "paired end (151 cycles + 151 cycles)"] def log_text = [ "Q20 bases: 12922(92.9841%)", - "reads passed filter: 198"] + "reads passed filter: 162"] def read_lines = [ "@ERR5069949.2151832 NS500628:121:HK3MMAFX2:2:21208:10793:15304/1", "TCATAAACCAAAGCACTCACAGTGTCAACAATTTCAGCAGGACAACGCCGACAAGTTCCGAGGAACATGTCTGGACCTATAGTTTTCATAAGTCTACACACTGAATTGAAATATTCTGGTTCTAGTGTGCCCTTAGTTAGCAATGTGCGT", "AAAAAAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAAEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAAEEEEE - { assert path(process.out.reads_fail.get(0).get(1).get(1)).linesGzip.contains(failed_read2_line) } + { assert path(process.out.reads_fail.get(0).get(1).get(2)).linesGzip.contains(failed_read2_line) } } }, { html_text.each { html_part -> @@ -498,7 +500,7 @@ nextflow_process { { assert path(process.out.log.get(0).get(1)).getText().contains(log_part) } } }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_paired_end_trim_fail") } ) } } @@ -587,9 +589,9 @@ nextflow_process { process.out.reads_fail.collect { file(it[1]).getName() } + process.out.reads_merged.collect { file(it[1]).getName() } ).sort() - ).match("test_fastp_paired_end_merged-for_stub_match") + ).match("test_fastp_paired_end_merged_match") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_paired_end_merged") } ) } } @@ -636,7 +638,7 @@ nextflow_process { ).sort() ).match("test_fastp_paired_end_merged-for_stub_match") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_paired_end_merged_stub") } ) } } @@ -716,7 +718,7 @@ nextflow_process { { assert path(process.out.log.get(0).get(1)).getText().contains(log_part) } } }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("versions_paired_end_merged_adapterlist") } ) } } diff --git a/modules/nf-core/fastp/tests/main.nf.test.snap b/modules/nf-core/fastp/tests/main.nf.test.snap index 6a71b680..3e876288 100644 --- a/modules/nf-core/fastp/tests/main.nf.test.snap +++ b/modules/nf-core/fastp/tests/main.nf.test.snap @@ -1,5 +1,23 @@ { - "test_fastp_paired_end-for_stub_match": { + "fastp test_fastp_interleaved_json": { + "content": [ + [ + [ + { + "id": "test", + "single_end": true + }, + "test.fastp.json:md5,b24e0624df5cc0b11cd5ba21b726fb22" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-18T16:19:15.063001" + }, + "test_fastp_paired_end_merged-for_stub_match": { "content": [ [ [ @@ -9,12 +27,29 @@ "test.fastp.html", "test.fastp.json", "test.fastp.log", + "test.merged.fastq.gz", "{id=test, single_end=false}" ] ], - "timestamp": "2024-01-17T18:07:15.398827" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-17T18:10:13.467574" }, - "fastp test_fastp_interleaved_json": { + "versions_interleaved": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:56:24.615634793" + }, + "test_fastp_single_end_json": { "content": [ [ [ @@ -22,13 +57,64 @@ "id": "test", "single_end": true }, - "test.fastp.json:md5,168f516f7bd4b7b6c32da7cba87299a4" + "test.fastp.json:md5,c852d7a6dba5819e4ac8d9673bedcacc" ] ] ], - "timestamp": "2024-01-17T18:08:06.123035" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-18T16:18:43.526412" }, - "test_fastp_paired_end_merged-for_stub_match": { + "versions_paired_end": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:55:42.333545689" + }, + "test_fastp_paired_end_match": { + "content": [ + [ + [ + "test_1.fastp.fastq.gz", + "test_2.fastp.fastq.gz" + ], + "test.fastp.html", + "test.fastp.json", + "test.fastp.log", + "{id=test, single_end=false}" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T12:03:06.431833729" + }, + "test_fastp_interleaved-_match": { + "content": [ + [ + "test.fastp.fastq.gz", + "test.fastp.html", + "test.fastp.json", + "test.fastp.log", + "{id=test, single_end=true}" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-18T16:19:15.111894" + }, + "test_fastp_paired_end_merged_match": { "content": [ [ [ @@ -42,29 +128,102 @@ "{id=test, single_end=false}" ] ], - "timestamp": "2024-01-17T18:10:13.467574" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T12:08:44.496251446" }, - "test_fastp_single_end_json": { + "versions_single_end_stub": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:55:27.354051299" + }, + "versions_interleaved-stub": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:56:46.535528418" + }, + "versions_single_end_trim_fail": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:59:03.724591407" + }, + "test_fastp_paired_end-for_stub_match": { "content": [ [ [ - { - "id": "test", - "single_end": true - }, - "test.fastp.json:md5,c852d7a6dba5819e4ac8d9673bedcacc" - ] + "test_1.fastp.fastq.gz", + "test_2.fastp.fastq.gz" + ], + "test.fastp.html", + "test.fastp.json", + "test.fastp.log", + "{id=test, single_end=false}" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-17T18:07:15.398827" + }, + "versions_paired_end-stub": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" ] ], - "timestamp": "2024-01-17T18:06:00.223817" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:56:06.50017282" }, - "versions": { + "versions_single_end": { "content": [ [ "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" ] ], - "timestamp": "2024-01-17T18:06:00.248422" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:55:07.67921647" + }, + "versions_paired_end_merged_stub": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:59:47.350653154" }, "test_fastp_interleaved-for_stub_match": { "content": [ @@ -76,8 +235,24 @@ "{id=test, single_end=true}" ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, "timestamp": "2024-01-17T18:08:06.127974" }, + "versions_paired_end_trim_fail": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:59:18.140484878" + }, "test_fastp_single_end-for_stub_match": { "content": [ [ @@ -88,8 +263,52 @@ "{id=test, single_end=true}" ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, "timestamp": "2024-01-17T18:06:00.244202" }, + "test_fastp_single_end-_match": { + "content": [ + [ + "test.fastp.fastq.gz", + "test.fastp.html", + "test.fastp.json", + "test.fastp.log", + "{id=test, single_end=true}" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-18T16:18:43.580336" + }, + "versions_paired_end_merged_adapterlist": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T12:05:37.845370554" + }, + "versions_paired_end_merged": { + "content": [ + [ + "versions.yml:md5,48ffc994212fb1fc9f83a74fa69c9f02" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-01T11:59:32.860543858" + }, "test_fastp_single_end_trim_fail_json": { "content": [ [ @@ -102,6 +321,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, "timestamp": "2024-01-17T18:08:41.942317" } } \ No newline at end of file diff --git a/modules/nf-core/fastp/tests/nextflow.interleaved.config b/modules/nf-core/fastp/tests/nextflow.interleaved.config new file mode 100644 index 00000000..4be8dbd2 --- /dev/null +++ b/modules/nf-core/fastp/tests/nextflow.interleaved.config @@ -0,0 +1,5 @@ +process { + withName: FASTP { + ext.args = "--interleaved_in -e 30" + } +} diff --git a/modules/nf-core/fastp/tests/nextflow.config b/modules/nf-core/fastp/tests/nextflow.save_failed.config similarity index 50% rename from modules/nf-core/fastp/tests/nextflow.config rename to modules/nf-core/fastp/tests/nextflow.save_failed.config index 0f7849ad..53b61b0c 100644 --- a/modules/nf-core/fastp/tests/nextflow.config +++ b/modules/nf-core/fastp/tests/nextflow.save_failed.config @@ -1,6 +1,5 @@ process { - withName: FASTP { - ext.args = "--interleaved_in" + ext.args = "-e 30" } } diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 9e19a74c..d79f1c86 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -25,6 +25,11 @@ process FASTQC { def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // FastQC memory value allowed range (100 - 10000) + def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) + """ printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name @@ -33,6 +38,7 @@ process FASTQC { fastqc \\ $args \\ --threads $task.cpus \\ + --memory $fastqc_memory \\ $renamed_files cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/kraken2/kraken2/environment.yml b/modules/nf-core/kraken2/kraken2/environment.yml index 63be419b..0c067fee 100644 --- a/modules/nf-core/kraken2/kraken2/environment.yml +++ b/modules/nf-core/kraken2/kraken2/environment.yml @@ -4,5 +4,6 @@ channels: - bioconda - defaults dependencies: - - bioconda::kraken2=2.1.2 - - conda-forge::pigz=2.6 + - "bioconda::kraken2=2.1.3" + - "coreutils=9.4" + - "pigz=2.8" diff --git a/modules/nf-core/kraken2/kraken2/main.nf b/modules/nf-core/kraken2/kraken2/main.nf index cdf02d0f..364a6fe2 100644 --- a/modules/nf-core/kraken2/kraken2/main.nf +++ b/modules/nf-core/kraken2/kraken2/main.nf @@ -4,8 +4,8 @@ process KRAKEN2_KRAKEN2 { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-5799ab18b5fc681e75923b2450abaa969907ec98:87fc08d11968d081f3e8a37131c1f1f6715b6542-0' : - 'biocontainers/mulled-v2-5799ab18b5fc681e75923b2450abaa969907ec98:87fc08d11968d081f3e8a37131c1f1f6715b6542-0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-8706a1dd73c6cc426e12dd4dd33a5e917b3989ae:c8cbdc8ff4101e6745f8ede6eb5261ef98bdaff4-0' : + 'biocontainers/mulled-v2-8706a1dd73c6cc426e12dd4dd33a5e917b3989ae:c8cbdc8ff4101e6745f8ede6eb5261ef98bdaff4-0' }" input: tuple val(meta), path(reads) @@ -55,4 +55,31 @@ process KRAKEN2_KRAKEN2 { pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) END_VERSIONS """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def paired = meta.single_end ? "" : "--paired" + def classified = meta.single_end ? "${prefix}.classified.fastq.gz" : "${prefix}.classified_1.fastq.gz ${prefix}.classified_2.fastq.gz" + def unclassified = meta.single_end ? "${prefix}.unclassified.fastq.gz" : "${prefix}.unclassified_1.fastq.gz ${prefix}.unclassified_2.fastq.gz" + def readclassification_option = save_reads_assignment ? "--output ${prefix}.kraken2.classifiedreads.txt" : "--output /dev/null" + def compress_reads_command = save_output_fastqs ? "pigz -p $task.cpus *.fastq" : "" + + """ + touch ${prefix}.kraken2.report.txt + if [ "$save_output_fastqs" == "true" ]; then + touch $classified + touch $unclassified + fi + if [ "$save_reads_assignment" == "true" ]; then + touch ${prefix}.kraken2.classifiedreads.txt + fi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + kraken2: \$(echo \$(kraken2 --version 2>&1) | sed 's/^.*Kraken version //; s/ .*\$//') + pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) + END_VERSIONS + """ + } diff --git a/modules/nf-core/kraken2/kraken2/tests/main.nf.test.snap b/modules/nf-core/kraken2/kraken2/tests/main.nf.test.snap index c1bdd0c6..b432f878 100644 --- a/modules/nf-core/kraken2/kraken2/tests/main.nf.test.snap +++ b/modules/nf-core/kraken2/kraken2/tests/main.nf.test.snap @@ -11,10 +11,14 @@ ] ], [ - "versions.yml:md5,bcb3e2520685846df02bb27cc6b1794b" + "versions.yml:md5,79adf2ca1cfc625cb77e391b27142c43" ] ], - "timestamp": "2023-10-25T09:01:29.775797" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-04-04T18:47:03.745692" }, "sarscov2 illumina paired end [fastq]": { "content": [ @@ -28,10 +32,14 @@ ] ], [ - "versions.yml:md5,bcb3e2520685846df02bb27cc6b1794b" + "versions.yml:md5,79adf2ca1cfc625cb77e391b27142c43" ] ], - "timestamp": "2023-10-25T09:01:37.025389" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-04-04T18:47:13.75649" }, "sarscov2 illumina single end [fastq] + save_reads_assignment": { "content": [ @@ -54,9 +62,13 @@ ] ], [ - "versions.yml:md5,bcb3e2520685846df02bb27cc6b1794b" + "versions.yml:md5,79adf2ca1cfc625cb77e391b27142c43" ] ], - "timestamp": "2023-10-25T09:01:45.775262" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-04-04T18:47:22.459465" } } \ No newline at end of file diff --git a/modules/nf-core/mosdepth/main.nf b/modules/nf-core/mosdepth/main.nf index c7e24303..0190a09d 100644 --- a/modules/nf-core/mosdepth/main.nf +++ b/modules/nf-core/mosdepth/main.nf @@ -63,13 +63,13 @@ process MOSDEPTH { touch ${prefix}.region.dist.txt touch ${prefix}.summary.txt touch ${prefix}.per-base.d4 - touch ${prefix}.per-base.bed.gz + echo "" | gzip > ${prefix}.per-base.bed.gz touch ${prefix}.per-base.bed.gz.csi - touch ${prefix}.regions.bed.gz + echo "" | gzip > ${prefix}.regions.bed.gz touch ${prefix}.regions.bed.gz.csi - touch ${prefix}.quantized.bed.gz + echo "" | gzip > ${prefix}.quantized.bed.gz touch ${prefix}.quantized.bed.gz.csi - touch ${prefix}.thresholds.bed.gz + echo "" | gzip > ${prefix}.thresholds.bed.gz touch ${prefix}.thresholds.bed.gz.csi cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/mosdepth/tests/main.nf.test b/modules/nf-core/mosdepth/tests/main.nf.test index d991f819..21eebc1f 100644 --- a/modules/nf-core/mosdepth/tests/main.nf.test +++ b/modules/nf-core/mosdepth/tests/main.nf.test @@ -237,21 +237,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.global_txt[0][1]).name, - file(process.out.summary_txt[0][1]).name, - file(process.out.regions_txt[0][1]).name, - file(process.out.per_base_d4[0][1]).name, - file(process.out.per_base_bed[0][1]).name, - file(process.out.per_base_csi[0][1]).name, - file(process.out.regions_bed[0][1]).name, - file(process.out.regions_csi[0][1]).name, - file(process.out.quantized_bed[0][1]).name, - file(process.out.quantized_csi[0][1]).name, - file(process.out.thresholds_bed[0][1]).name, - file(process.out.thresholds_csi[0][1]).name, - process.out.versions - ).match() } + { assert snapshot(process.out).match() } ) } diff --git a/modules/nf-core/mosdepth/tests/main.nf.test.snap b/modules/nf-core/mosdepth/tests/main.nf.test.snap index dc4d9508..00f4cd97 100644 --- a/modules/nf-core/mosdepth/tests/main.nf.test.snap +++ b/modules/nf-core/mosdepth/tests/main.nf.test.snap @@ -1,23 +1,236 @@ { "homo_sapiens - bam, bai, [] - stub": { "content": [ - "test.global.dist.txt", - "test.summary.txt", - "test.region.dist.txt", - "test.per-base.d4", - "test.per-base.bed.gz", - "test.per-base.bed.gz.csi", - "test.regions.bed.gz", - "test.regions.bed.gz.csi", - "test.quantized.bed.gz", - "test.quantized.bed.gz.csi", - "test.thresholds.bed.gz", - "test.thresholds.bed.gz.csi", - [ - "versions.yml:md5,f8b1896c9c6784181f1234e87225f0e8" - ] + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.global.dist.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "10": [ + [ + { + "id": "test", + "single_end": true + }, + "test.thresholds.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "11": [ + [ + { + "id": "test", + "single_end": true + }, + "test.thresholds.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "12": [ + "versions.yml:md5,f8b1896c9c6784181f1234e87225f0e8" + ], + "2": [ + [ + { + "id": "test", + "single_end": true + }, + "test.region.dist.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": true + }, + "test.per-base.d4:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + [ + { + "id": "test", + "single_end": true + }, + "test.per-base.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "5": [ + [ + { + "id": "test", + "single_end": true + }, + "test.per-base.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "6": [ + [ + { + "id": "test", + "single_end": true + }, + "test.regions.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "7": [ + [ + { + "id": "test", + "single_end": true + }, + "test.regions.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "8": [ + [ + { + "id": "test", + "single_end": true + }, + "test.quantized.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "9": [ + [ + { + "id": "test", + "single_end": true + }, + "test.quantized.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "global_txt": [ + [ + { + "id": "test", + "single_end": true + }, + "test.global.dist.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "per_base_bed": [ + [ + { + "id": "test", + "single_end": true + }, + "test.per-base.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "per_base_csi": [ + [ + { + "id": "test", + "single_end": true + }, + "test.per-base.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "per_base_d4": [ + [ + { + "id": "test", + "single_end": true + }, + "test.per-base.d4:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "quantized_bed": [ + [ + { + "id": "test", + "single_end": true + }, + "test.quantized.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "quantized_csi": [ + [ + { + "id": "test", + "single_end": true + }, + "test.quantized.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "regions_bed": [ + [ + { + "id": "test", + "single_end": true + }, + "test.regions.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "regions_csi": [ + [ + { + "id": "test", + "single_end": true + }, + "test.regions.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "regions_txt": [ + [ + { + "id": "test", + "single_end": true + }, + "test.region.dist.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "summary_txt": [ + [ + { + "id": "test", + "single_end": true + }, + "test.summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "thresholds_bed": [ + [ + { + "id": "test", + "single_end": true + }, + "test.thresholds.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "thresholds_csi": [ + [ + { + "id": "test", + "single_end": true + }, + "test.thresholds.bed.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,f8b1896c9c6784181f1234e87225f0e8" + ] + } ], - "timestamp": "2023-11-27T15:14:34.897155161" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:29:04.084192" }, "homo_sapiens - cram, crai, bed": { "content": [ @@ -67,7 +280,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "5": [ @@ -85,7 +298,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,5d398caf7171ec4406278e2add3009ae" + "test.regions.bed.gz:md5,9ded0397623fda26a6a3514d6a0e2a2c" ] ], "7": [ @@ -118,7 +331,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "per_base_csi": [ @@ -145,7 +358,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,5d398caf7171ec4406278e2add3009ae" + "test.regions.bed.gz:md5,9ded0397623fda26a6a3514d6a0e2a2c" ] ], "regions_csi": [ @@ -186,7 +399,11 @@ ] } ], - "timestamp": "2023-11-27T14:47:18.171150781" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:28:16.664319" }, "homo_sapiens - bam, bai, [] - quantized": { "content": [ @@ -230,7 +447,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "5": [ @@ -254,7 +471,7 @@ "id": "test", "single_end": true }, - "test.quantized.bed.gz:md5,3e434a8bafcf59a67841ae3d4d752838" + "test.quantized.bed.gz:md5,f037c215449d361112efc10108fcc17c" ] ], "9": [ @@ -281,7 +498,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "per_base_csi": [ @@ -302,7 +519,7 @@ "id": "test", "single_end": true }, - "test.quantized.bed.gz:md5,3e434a8bafcf59a67841ae3d4d752838" + "test.quantized.bed.gz:md5,f037c215449d361112efc10108fcc17c" ] ], "quantized_csi": [ @@ -343,7 +560,11 @@ ] } ], - "timestamp": "2023-11-27T14:47:29.228103864" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:28:35.978752" }, "homo_sapiens - bam, bai, bed": { "content": [ @@ -393,7 +614,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "5": [ @@ -411,7 +632,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,5d398caf7171ec4406278e2add3009ae" + "test.regions.bed.gz:md5,9ded0397623fda26a6a3514d6a0e2a2c" ] ], "7": [ @@ -444,7 +665,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "per_base_csi": [ @@ -471,7 +692,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,5d398caf7171ec4406278e2add3009ae" + "test.regions.bed.gz:md5,9ded0397623fda26a6a3514d6a0e2a2c" ] ], "regions_csi": [ @@ -512,7 +733,11 @@ ] } ], - "timestamp": "2023-11-27T14:47:04.537716314" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:27:55.598384" }, "homo_sapiens - bam, bai, [] - window": { "content": [ @@ -562,7 +787,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "5": [ @@ -580,7 +805,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,f02e2cb49cc050e13d76942d6960827a" + "test.regions.bed.gz:md5,34f48d16fcdd61e44d812e29e02c77b8" ] ], "7": [ @@ -613,7 +838,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "per_base_csi": [ @@ -640,7 +865,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,f02e2cb49cc050e13d76942d6960827a" + "test.regions.bed.gz:md5,34f48d16fcdd61e44d812e29e02c77b8" ] ], "regions_csi": [ @@ -681,7 +906,11 @@ ] } ], - "timestamp": "2023-11-27T14:47:23.708536171" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:28:26.37386" }, "homo_sapiens - bam, bai, []": { "content": [ @@ -725,7 +954,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "5": [ @@ -764,7 +993,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "per_base_csi": [ @@ -814,7 +1043,11 @@ ] } ], - "timestamp": "2023-11-27T14:46:56.975710077" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:27:44.658599" }, "homo_sapiens - cram, crai, []": { "content": [ @@ -858,7 +1091,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "5": [ @@ -897,7 +1130,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "per_base_csi": [ @@ -947,7 +1180,11 @@ ] } ], - "timestamp": "2023-11-27T14:47:12.09259995" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:28:06.482838" }, "homo_sapiens - bam, bai, bed - thresholds": { "content": [ @@ -976,7 +1213,7 @@ "id": "test", "single_end": true }, - "test.thresholds.bed.gz:md5,13101e326eea3cbfa1d569b69f494f4c" + "test.thresholds.bed.gz:md5,fe70ae728cd10726c42a2bcd44adfc9d" ] ], "11": [ @@ -1009,7 +1246,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "5": [ @@ -1027,7 +1264,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,5d398caf7171ec4406278e2add3009ae" + "test.regions.bed.gz:md5,9ded0397623fda26a6a3514d6a0e2a2c" ] ], "7": [ @@ -1060,7 +1297,7 @@ "id": "test", "single_end": true }, - "test.per-base.bed.gz:md5,bc1df47d46f818fee5275975925d769a" + "test.per-base.bed.gz:md5,da6db0fb375a3053a89db8c935eebbaa" ] ], "per_base_csi": [ @@ -1087,7 +1324,7 @@ "id": "test", "single_end": true }, - "test.regions.bed.gz:md5,5d398caf7171ec4406278e2add3009ae" + "test.regions.bed.gz:md5,9ded0397623fda26a6a3514d6a0e2a2c" ] ], "regions_csi": [ @@ -1123,7 +1360,7 @@ "id": "test", "single_end": true }, - "test.thresholds.bed.gz:md5,13101e326eea3cbfa1d569b69f494f4c" + "test.thresholds.bed.gz:md5,fe70ae728cd10726c42a2bcd44adfc9d" ] ], "thresholds_csi": [ @@ -1140,6 +1377,10 @@ ] } ], - "timestamp": "2023-11-27T14:49:44.311847326" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-28T12:28:45.918241" } } \ No newline at end of file diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index 7625b752..ca39fb67 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.19 + - bioconda::multiqc=1.21 diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index d0438eda..f1c4242e 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -3,6 +3,7 @@ nextflow_process { name "Test Process MULTIQC" script "../main.nf" process "MULTIQC" + tag "modules" tag "modules_nfcore" tag "multiqc" @@ -12,7 +13,7 @@ nextflow_process { when { process { """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) input[1] = [] input[2] = [] input[3] = [] @@ -25,7 +26,7 @@ nextflow_process { { assert process.success }, { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, { assert process.out.data[0] ==~ ".*/multiqc_data" }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("multiqc_versions_single") } ) } @@ -36,7 +37,7 @@ nextflow_process { when { process { """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] @@ -49,7 +50,7 @@ nextflow_process { { assert process.success }, { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, { assert process.out.data[0] ==~ ".*/multiqc_data" }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out.versions).match("multiqc_versions_config") } ) } } @@ -61,7 +62,7 @@ nextflow_process { when { process { """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) input[1] = [] input[2] = [] input[3] = [] @@ -75,7 +76,7 @@ nextflow_process { { assert snapshot(process.out.report.collect { file(it).getName() } + process.out.data.collect { file(it).getName() } + process.out.plots.collect { file(it).getName() } + - process.out.versions ).match() } + process.out.versions ).match("multiqc_stub") } ) } diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap index d37e7304..bfebd802 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -1,21 +1,41 @@ { - "versions": { + "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], - "timestamp": "2024-01-09T23:02:49.911994" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-29T08:48:55.657331" }, - "sarscov2 single-end [fastqc] - stub": { + "multiqc_stub": { "content": [ [ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], - "timestamp": "2024-01-09T23:03:14.524346" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-29T08:49:49.071937" + }, + "multiqc_versions_config": { + "content": [ + [ + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-29T08:49:25.457567" } } \ No newline at end of file diff --git a/modules/nf-core/nanoplot/main.nf b/modules/nf-core/nanoplot/main.nf index 6dfe1352..9fe43c0e 100644 --- a/modules/nf-core/nanoplot/main.nf +++ b/modules/nf-core/nanoplot/main.nf @@ -30,6 +30,26 @@ process NANOPLOT { -t $task.cpus \\ $input_file + cat <<-END_VERSIONS > versions.yml + "${task.process}": + nanoplot: \$(echo \$(NanoPlot --version 2>&1) | sed 's/^.*NanoPlot //; s/ .*\$//') + END_VERSIONS + """ + + stub: + """ + touch LengthvsQualityScatterPlot_dot.html + touch LengthvsQualityScatterPlot_kde.html + touch NanoPlot-report.html + touch NanoPlot_20240301_1130.log + touch NanoStats.txt + touch Non_weightedHistogramReadlength.html + touch Non_weightedLogTransformed_HistogramReadlength.html + touch WeightedHistogramReadlength.html + touch WeightedLogTransformed_HistogramReadlength.html + touch Yield_By_Length.html + + cat <<-END_VERSIONS > versions.yml "${task.process}": nanoplot: \$(echo \$(NanoPlot --version 2>&1) | sed 's/^.*NanoPlot //; s/ .*\$//') diff --git a/modules/nf-core/nanoplot/tests/main.nf.test b/modules/nf-core/nanoplot/tests/main.nf.test index d8dc6bc0..29b57c10 100644 --- a/modules/nf-core/nanoplot/tests/main.nf.test +++ b/modules/nf-core/nanoplot/tests/main.nf.test @@ -32,7 +32,7 @@ nextflow_process { with(process.out.html.get(0)) { assert get(1).collect { p -> file(p).getName() }.contains("NanoPlot-report.html") } - } + } ) } @@ -63,7 +63,30 @@ nextflow_process { with(process.out.html.get(0)) { assert get(1).collect { p -> file(p).getName() }.contains("NanoPlot-report.html") } - } + } + ) + } + + } + + test("NanoPlot - stub") { + + options "-stub" + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + [ file(params.test_data['sarscov2']['nanopore']['test_sequencing_summary'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } ) } diff --git a/modules/nf-core/nanoplot/tests/main.nf.test.snap b/modules/nf-core/nanoplot/tests/main.nf.test.snap index 1df28066..f7f8028a 100644 --- a/modules/nf-core/nanoplot/tests/main.nf.test.snap +++ b/modules/nf-core/nanoplot/tests/main.nf.test.snap @@ -1,4 +1,93 @@ { + "NanoPlot - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + [ + "LengthvsQualityScatterPlot_dot.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "LengthvsQualityScatterPlot_kde.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "NanoPlot-report.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Non_weightedHistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Non_weightedLogTransformed_HistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "WeightedHistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "WeightedLogTransformed_HistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Yield_By_Length.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + + ], + "2": [ + [ + { + "id": "test" + }, + "NanoStats.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + [ + { + "id": "test" + }, + "NanoPlot_20240301_1130.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + "versions.yml:md5,961cee64736aeb9e56b65d05ee3cd1a5" + ], + "html": [ + [ + { + "id": "test" + }, + [ + "LengthvsQualityScatterPlot_dot.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "LengthvsQualityScatterPlot_kde.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "NanoPlot-report.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Non_weightedHistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Non_weightedLogTransformed_HistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "WeightedHistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "WeightedLogTransformed_HistogramReadlength.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Yield_By_Length.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "log": [ + [ + { + "id": "test" + }, + "NanoPlot_20240301_1130.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "png": [ + + ], + "txt": [ + [ + { + "id": "test" + }, + "NanoStats.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,961cee64736aeb9e56b65d05ee3cd1a5" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, + "timestamp": "2024-03-01T14:54:18.083198" + }, "NanoPlot FASTQ": { "content": [ [ @@ -13,6 +102,10 @@ "versions.yml:md5,961cee64736aeb9e56b65d05ee3cd1a5" ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, "timestamp": "2023-10-17T16:18:44.848688965" }, "NanoPlot summary": { @@ -29,6 +122,10 @@ "versions.yml:md5,961cee64736aeb9e56b65d05ee3cd1a5" ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.0" + }, "timestamp": "2023-10-17T16:18:31.104601192" } } \ No newline at end of file diff --git a/modules/nf-core/picard/markduplicates/main.nf b/modules/nf-core/picard/markduplicates/main.nf index 80930cc4..ad0b2963 100644 --- a/modules/nf-core/picard/markduplicates/main.nf +++ b/modules/nf-core/picard/markduplicates/main.nf @@ -8,13 +8,14 @@ process PICARD_MARKDUPLICATES { 'biocontainers/picard:3.1.1--hdfd78af_0' }" input: - tuple val(meta), path(bam) + tuple val(meta), path(reads) tuple val(meta2), path(fasta) tuple val(meta3), path(fai) output: - tuple val(meta), path("*.bam") , emit: bam - tuple val(meta), path("*.bai") , optional:true, emit: bai + tuple val(meta), path("*.bam") , emit: bam, optional: true + tuple val(meta), path("*.bai") , emit: bai, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true tuple val(meta), path("*.metrics.txt"), emit: metrics path "versions.yml" , emit: versions @@ -24,6 +25,8 @@ process PICARD_MARKDUPLICATES { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def suffix = task.ext.suffix ?: "${reads.getExtension()}" + def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : "" def avail_mem = 3072 if (!task.memory) { log.info '[Picard MarkDuplicates] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' @@ -31,16 +34,16 @@ process PICARD_MARKDUPLICATES { avail_mem = (task.memory.mega*0.8).intValue() } - if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + if ("$reads" == "${prefix}.${suffix}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" """ picard \\ -Xmx${avail_mem}M \\ MarkDuplicates \\ $args \\ - --INPUT $bam \\ - --OUTPUT ${prefix}.bam \\ - --REFERENCE_SEQUENCE $fasta \\ + --INPUT $reads \\ + --OUTPUT ${prefix}.${suffix} \\ + $reference \\ --METRICS_FILE ${prefix}.MarkDuplicates.metrics.txt cat <<-END_VERSIONS > versions.yml @@ -51,10 +54,10 @@ process PICARD_MARKDUPLICATES { stub: def prefix = task.ext.prefix ?: "${meta.id}" - if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + def suffix = task.ext.suffix ?: "${reads.getExtension()}" + if ("$reads" == "${prefix}.${suffix}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" """ - touch ${prefix}.bam - touch ${prefix}.bam.bai + touch ${prefix}.${suffix} touch ${prefix}.MarkDuplicates.metrics.txt cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/picard/markduplicates/meta.yml b/modules/nf-core/picard/markduplicates/meta.yml index 1ab90c07..1f0ffe16 100644 --- a/modules/nf-core/picard/markduplicates/meta.yml +++ b/modules/nf-core/picard/markduplicates/meta.yml @@ -21,9 +21,9 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - bam: + - reads: type: file - description: BAM file + description: Sequence reads file, can be SAM/BAM/CRAM format pattern: "*.{bam,cram,sam}" - meta2: type: map @@ -32,7 +32,7 @@ input: e.g. [ id:'genome' ] - fasta: type: file - description: Reference genome fasta file + description: Reference genome fasta file, required for CRAM input pattern: "*.{fasta,fa}" - meta3: type: map @@ -57,6 +57,10 @@ output: type: file description: An optional BAM index file. If desired, --CREATE_INDEX must be passed as a flag pattern: "*.{bai}" + - cram: + type: file + description: Output CRAM file + pattern: "*.{cram}" - metrics: type: file description: Duplicate metrics file generated by picard diff --git a/modules/nf-core/picard/markduplicates/tests/main.nf.test b/modules/nf-core/picard/markduplicates/tests/main.nf.test index c5a29b4b..e3e97f6c 100644 --- a/modules/nf-core/picard/markduplicates/tests/main.nf.test +++ b/modules/nf-core/picard/markduplicates/tests/main.nf.test @@ -18,14 +18,8 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true) ]) - input[1] = Channel.of([ - [ id:'genome' ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ]) - input[2] = Channel.of([ - [ id:'genome' ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) - ]) + input[1] = [ [:], [] ] + input[2] = [ [:], [] ] """ } } @@ -49,14 +43,8 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) ]) - input[1] = Channel.of([ - [ id:'genome' ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ]) - input[2] = Channel.of([ - [ id:'genome' ], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta.fai', checkIfExists: true) - ]) + input[1] = [ [:], [] ] + input[2] = [ [:], [] ] """ } } @@ -95,7 +83,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(file(process.out.bam[0][1]).name).match("cram_name") }, + { assert snapshot(file(process.out.cram[0][1]).name).match("cram_name") }, { assert snapshot(path(process.out.metrics.get(0).get(1)).readLines()[0..2]).match("cram_metrics") }, { assert snapshot(process.out.versions).match("cram_versions") } ) diff --git a/modules/nf-core/picard/markduplicates/tests/main.nf.test.snap b/modules/nf-core/picard/markduplicates/tests/main.nf.test.snap index 31c9130d..eb17111e 100644 --- a/modules/nf-core/picard/markduplicates/tests/main.nf.test.snap +++ b/modules/nf-core/picard/markduplicates/tests/main.nf.test.snap @@ -5,39 +5,59 @@ "versions.yml:md5,b699af51b1956f3810f8a7c066e0ab17" ] ], - "timestamp": "2024-01-19T10:26:45.092349" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T15:31:50.928021" }, "unsorted_bam_name": { "content": [ "test.marked.bam" ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, "timestamp": "2024-01-19T10:26:28.100755" }, "cram_metrics": { "content": [ [ "## htsjdk.samtools.metrics.StringHeader", - "# MarkDuplicates --INPUT test.paired_end.sorted.cram --OUTPUT test.marked.bam --METRICS_FILE test.marked.MarkDuplicates.metrics.txt --ASSUME_SORT_ORDER queryname --REFERENCE_SEQUENCE genome.fasta --MAX_SEQUENCES_FOR_DISK_READ_ENDS_MAP 50000 --MAX_FILE_HANDLES_FOR_READ_ENDS_MAP 8000 --SORTING_COLLECTION_SIZE_RATIO 0.25 --TAG_DUPLICATE_SET_MEMBERS false --REMOVE_SEQUENCING_DUPLICATES false --TAGGING_POLICY DontTag --CLEAR_DT true --DUPLEX_UMI false --FLOW_MODE false --FLOW_QUALITY_SUM_STRATEGY false --USE_END_IN_UNPAIRED_READS false --USE_UNPAIRED_CLIPPED_END false --UNPAIRED_END_UNCERTAINTY 0 --FLOW_SKIP_FIRST_N_FLOWS 0 --FLOW_Q_IS_KNOWN_END false --FLOW_EFFECTIVE_QUALITY_THRESHOLD 15 --ADD_PG_TAG_TO_READS true --REMOVE_DUPLICATES false --ASSUME_SORTED false --DUPLICATE_SCORING_STRATEGY SUM_OF_BASE_QUALITIES --PROGRAM_RECORD_ID MarkDuplicates --PROGRAM_GROUP_NAME MarkDuplicates --READ_NAME_REGEX --OPTICAL_DUPLICATE_PIXEL_DISTANCE 100 --MAX_OPTICAL_DUPLICATE_SET_SIZE 300000 --VERBOSITY INFO --QUIET false --VALIDATION_STRINGENCY STRICT --COMPRESSION_LEVEL 5 --MAX_RECORDS_IN_RAM 500000 --CREATE_INDEX false --CREATE_MD5_FILE false --help false --version false --showHidden false --USE_JDK_DEFLATER false --USE_JDK_INFLATER false", + "# MarkDuplicates --INPUT test.paired_end.sorted.cram --OUTPUT test.marked.cram --METRICS_FILE test.marked.MarkDuplicates.metrics.txt --ASSUME_SORT_ORDER queryname --REFERENCE_SEQUENCE genome.fasta --MAX_SEQUENCES_FOR_DISK_READ_ENDS_MAP 50000 --MAX_FILE_HANDLES_FOR_READ_ENDS_MAP 8000 --SORTING_COLLECTION_SIZE_RATIO 0.25 --TAG_DUPLICATE_SET_MEMBERS false --REMOVE_SEQUENCING_DUPLICATES false --TAGGING_POLICY DontTag --CLEAR_DT true --DUPLEX_UMI false --FLOW_MODE false --FLOW_QUALITY_SUM_STRATEGY false --USE_END_IN_UNPAIRED_READS false --USE_UNPAIRED_CLIPPED_END false --UNPAIRED_END_UNCERTAINTY 0 --FLOW_SKIP_FIRST_N_FLOWS 0 --FLOW_Q_IS_KNOWN_END false --FLOW_EFFECTIVE_QUALITY_THRESHOLD 15 --ADD_PG_TAG_TO_READS true --REMOVE_DUPLICATES false --ASSUME_SORTED false --DUPLICATE_SCORING_STRATEGY SUM_OF_BASE_QUALITIES --PROGRAM_RECORD_ID MarkDuplicates --PROGRAM_GROUP_NAME MarkDuplicates --READ_NAME_REGEX --OPTICAL_DUPLICATE_PIXEL_DISTANCE 100 --MAX_OPTICAL_DUPLICATE_SET_SIZE 300000 --VERBOSITY INFO --QUIET false --VALIDATION_STRINGENCY STRICT --COMPRESSION_LEVEL 5 --MAX_RECORDS_IN_RAM 500000 --CREATE_INDEX false --CREATE_MD5_FILE false --help false --version false --showHidden false --USE_JDK_DEFLATER false --USE_JDK_INFLATER false", "## htsjdk.samtools.metrics.StringHeader" ] ], - "timestamp": "2024-01-19T10:27:03.253071" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T15:25:47.518152" }, "sorted_bam_metrics": { "content": [ [ "## htsjdk.samtools.metrics.StringHeader", - "# MarkDuplicates --INPUT test.paired_end.sorted.bam --OUTPUT test.marked.bam --METRICS_FILE test.marked.MarkDuplicates.metrics.txt --ASSUME_SORT_ORDER queryname --REFERENCE_SEQUENCE genome.fasta --MAX_SEQUENCES_FOR_DISK_READ_ENDS_MAP 50000 --MAX_FILE_HANDLES_FOR_READ_ENDS_MAP 8000 --SORTING_COLLECTION_SIZE_RATIO 0.25 --TAG_DUPLICATE_SET_MEMBERS false --REMOVE_SEQUENCING_DUPLICATES false --TAGGING_POLICY DontTag --CLEAR_DT true --DUPLEX_UMI false --FLOW_MODE false --FLOW_QUALITY_SUM_STRATEGY false --USE_END_IN_UNPAIRED_READS false --USE_UNPAIRED_CLIPPED_END false --UNPAIRED_END_UNCERTAINTY 0 --FLOW_SKIP_FIRST_N_FLOWS 0 --FLOW_Q_IS_KNOWN_END false --FLOW_EFFECTIVE_QUALITY_THRESHOLD 15 --ADD_PG_TAG_TO_READS true --REMOVE_DUPLICATES false --ASSUME_SORTED false --DUPLICATE_SCORING_STRATEGY SUM_OF_BASE_QUALITIES --PROGRAM_RECORD_ID MarkDuplicates --PROGRAM_GROUP_NAME MarkDuplicates --READ_NAME_REGEX --OPTICAL_DUPLICATE_PIXEL_DISTANCE 100 --MAX_OPTICAL_DUPLICATE_SET_SIZE 300000 --VERBOSITY INFO --QUIET false --VALIDATION_STRINGENCY STRICT --COMPRESSION_LEVEL 5 --MAX_RECORDS_IN_RAM 500000 --CREATE_INDEX false --CREATE_MD5_FILE false --help false --version false --showHidden false --USE_JDK_DEFLATER false --USE_JDK_INFLATER false", + "# MarkDuplicates --INPUT test.paired_end.sorted.bam --OUTPUT test.marked.bam --METRICS_FILE test.marked.MarkDuplicates.metrics.txt --ASSUME_SORT_ORDER queryname --MAX_SEQUENCES_FOR_DISK_READ_ENDS_MAP 50000 --MAX_FILE_HANDLES_FOR_READ_ENDS_MAP 8000 --SORTING_COLLECTION_SIZE_RATIO 0.25 --TAG_DUPLICATE_SET_MEMBERS false --REMOVE_SEQUENCING_DUPLICATES false --TAGGING_POLICY DontTag --CLEAR_DT true --DUPLEX_UMI false --FLOW_MODE false --FLOW_QUALITY_SUM_STRATEGY false --USE_END_IN_UNPAIRED_READS false --USE_UNPAIRED_CLIPPED_END false --UNPAIRED_END_UNCERTAINTY 0 --FLOW_SKIP_FIRST_N_FLOWS 0 --FLOW_Q_IS_KNOWN_END false --FLOW_EFFECTIVE_QUALITY_THRESHOLD 15 --ADD_PG_TAG_TO_READS true --REMOVE_DUPLICATES false --ASSUME_SORTED false --DUPLICATE_SCORING_STRATEGY SUM_OF_BASE_QUALITIES --PROGRAM_RECORD_ID MarkDuplicates --PROGRAM_GROUP_NAME MarkDuplicates --READ_NAME_REGEX --OPTICAL_DUPLICATE_PIXEL_DISTANCE 100 --MAX_OPTICAL_DUPLICATE_SET_SIZE 300000 --VERBOSITY INFO --QUIET false --VALIDATION_STRINGENCY STRICT --COMPRESSION_LEVEL 5 --MAX_RECORDS_IN_RAM 500000 --CREATE_INDEX false --CREATE_MD5_FILE false --help false --version false --showHidden false --USE_JDK_DEFLATER false --USE_JDK_INFLATER false", "## htsjdk.samtools.metrics.StringHeader" ] ], - "timestamp": "2024-01-19T10:26:45.086503" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-21T11:39:10.318331" }, "cram_name": { "content": [ - "test.marked.bam" + "test.marked.cram" ], - "timestamp": "2024-01-19T10:27:03.241617" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T15:25:47.459663" }, "cram_versions": { "content": [ @@ -45,6 +65,10 @@ "versions.yml:md5,b699af51b1956f3810f8a7c066e0ab17" ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, "timestamp": "2024-01-19T10:27:03.26989" }, "unsorted_bam_versions": { @@ -53,22 +77,34 @@ "versions.yml:md5,b699af51b1956f3810f8a7c066e0ab17" ] ], - "timestamp": "2024-01-19T10:26:28.159071" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T15:31:24.040403" }, "unsorted_bam_metrics": { "content": [ [ "## htsjdk.samtools.metrics.StringHeader", - "# MarkDuplicates --INPUT test.paired_end.bam --OUTPUT test.marked.bam --METRICS_FILE test.marked.MarkDuplicates.metrics.txt --ASSUME_SORT_ORDER queryname --REFERENCE_SEQUENCE genome.fasta --MAX_SEQUENCES_FOR_DISK_READ_ENDS_MAP 50000 --MAX_FILE_HANDLES_FOR_READ_ENDS_MAP 8000 --SORTING_COLLECTION_SIZE_RATIO 0.25 --TAG_DUPLICATE_SET_MEMBERS false --REMOVE_SEQUENCING_DUPLICATES false --TAGGING_POLICY DontTag --CLEAR_DT true --DUPLEX_UMI false --FLOW_MODE false --FLOW_QUALITY_SUM_STRATEGY false --USE_END_IN_UNPAIRED_READS false --USE_UNPAIRED_CLIPPED_END false --UNPAIRED_END_UNCERTAINTY 0 --FLOW_SKIP_FIRST_N_FLOWS 0 --FLOW_Q_IS_KNOWN_END false --FLOW_EFFECTIVE_QUALITY_THRESHOLD 15 --ADD_PG_TAG_TO_READS true --REMOVE_DUPLICATES false --ASSUME_SORTED false --DUPLICATE_SCORING_STRATEGY SUM_OF_BASE_QUALITIES --PROGRAM_RECORD_ID MarkDuplicates --PROGRAM_GROUP_NAME MarkDuplicates --READ_NAME_REGEX --OPTICAL_DUPLICATE_PIXEL_DISTANCE 100 --MAX_OPTICAL_DUPLICATE_SET_SIZE 300000 --VERBOSITY INFO --QUIET false --VALIDATION_STRINGENCY STRICT --COMPRESSION_LEVEL 5 --MAX_RECORDS_IN_RAM 500000 --CREATE_INDEX false --CREATE_MD5_FILE false --help false --version false --showHidden false --USE_JDK_DEFLATER false --USE_JDK_INFLATER false", + "# MarkDuplicates --INPUT test.paired_end.bam --OUTPUT test.marked.bam --METRICS_FILE test.marked.MarkDuplicates.metrics.txt --ASSUME_SORT_ORDER queryname --MAX_SEQUENCES_FOR_DISK_READ_ENDS_MAP 50000 --MAX_FILE_HANDLES_FOR_READ_ENDS_MAP 8000 --SORTING_COLLECTION_SIZE_RATIO 0.25 --TAG_DUPLICATE_SET_MEMBERS false --REMOVE_SEQUENCING_DUPLICATES false --TAGGING_POLICY DontTag --CLEAR_DT true --DUPLEX_UMI false --FLOW_MODE false --FLOW_QUALITY_SUM_STRATEGY false --USE_END_IN_UNPAIRED_READS false --USE_UNPAIRED_CLIPPED_END false --UNPAIRED_END_UNCERTAINTY 0 --FLOW_SKIP_FIRST_N_FLOWS 0 --FLOW_Q_IS_KNOWN_END false --FLOW_EFFECTIVE_QUALITY_THRESHOLD 15 --ADD_PG_TAG_TO_READS true --REMOVE_DUPLICATES false --ASSUME_SORTED false --DUPLICATE_SCORING_STRATEGY SUM_OF_BASE_QUALITIES --PROGRAM_RECORD_ID MarkDuplicates --PROGRAM_GROUP_NAME MarkDuplicates --READ_NAME_REGEX --OPTICAL_DUPLICATE_PIXEL_DISTANCE 100 --MAX_OPTICAL_DUPLICATE_SET_SIZE 300000 --VERBOSITY INFO --QUIET false --VALIDATION_STRINGENCY STRICT --COMPRESSION_LEVEL 5 --MAX_RECORDS_IN_RAM 500000 --CREATE_INDEX false --CREATE_MD5_FILE false --help false --version false --showHidden false --USE_JDK_DEFLATER false --USE_JDK_INFLATER false", "## htsjdk.samtools.metrics.StringHeader" ] ], - "timestamp": "2024-01-19T10:26:28.143979" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-21T10:51:12.831787" }, "sorted_bam_name": { "content": [ "test.marked.bam" ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, "timestamp": "2024-01-19T10:26:45.080116" } } \ No newline at end of file diff --git a/modules/nf-core/samtools/flagstat/environment.yml b/modules/nf-core/samtools/flagstat/environment.yml index dd0b5c19..bd57cb54 100644 --- a/modules/nf-core/samtools/flagstat/environment.yml +++ b/modules/nf-core/samtools/flagstat/environment.yml @@ -4,5 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::samtools=1.18 - - bioconda::htslib=1.18 + - bioconda::samtools=1.19.2 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/samtools/flagstat/main.nf b/modules/nf-core/samtools/flagstat/main.nf index f1893d7c..eb5f5252 100644 --- a/modules/nf-core/samtools/flagstat/main.nf +++ b/modules/nf-core/samtools/flagstat/main.nf @@ -4,8 +4,8 @@ process SAMTOOLS_FLAGSTAT { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1' : - 'biocontainers/samtools:1.18--h50ea8bc_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' : + 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/samtools/flagstat/tests/main.nf.test b/modules/nf-core/samtools/flagstat/tests/main.nf.test index c8dd8dc9..24c3c04b 100644 --- a/modules/nf-core/samtools/flagstat/tests/main.nf.test +++ b/modules/nf-core/samtools/flagstat/tests/main.nf.test @@ -16,11 +16,11 @@ nextflow_process { } process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) - ] + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true) + ]) """ } } @@ -28,8 +28,8 @@ nextflow_process { then { assertAll ( { assert process.success }, - { assert snapshot(process.out.flagstat).match() }, - { assert path(process.out.versions.get(0)).getText().contains("samtools") } + { assert snapshot(process.out.flagstat).match("flagstat") }, + { assert snapshot(process.out.versions).match("versions") } ) } } diff --git a/modules/nf-core/samtools/flagstat/tests/main.nf.test.snap b/modules/nf-core/samtools/flagstat/tests/main.nf.test.snap index 880019f2..a76fc27e 100644 --- a/modules/nf-core/samtools/flagstat/tests/main.nf.test.snap +++ b/modules/nf-core/samtools/flagstat/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "BAM": { + "flagstat": { "content": [ [ [ @@ -11,6 +11,22 @@ ] ] ], - "timestamp": "2023-11-14T15:49:22.577133" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T18:31:37.783927" + }, + "versions": { + "content": [ + [ + "versions.yml:md5,fd0030ce49ab3a92091ad80260226452" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:11:44.299617452" } } \ No newline at end of file diff --git a/modules/nf-core/samtools/idxstats/environment.yml b/modules/nf-core/samtools/idxstats/environment.yml index de3ed47e..174973b8 100644 --- a/modules/nf-core/samtools/idxstats/environment.yml +++ b/modules/nf-core/samtools/idxstats/environment.yml @@ -4,5 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::samtools=1.18 - - bioconda::htslib=1.18 + - bioconda::samtools=1.19.2 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/samtools/idxstats/main.nf b/modules/nf-core/samtools/idxstats/main.nf index 00d916bb..a544026f 100644 --- a/modules/nf-core/samtools/idxstats/main.nf +++ b/modules/nf-core/samtools/idxstats/main.nf @@ -4,8 +4,8 @@ process SAMTOOLS_IDXSTATS { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1' : - 'biocontainers/samtools:1.18--h50ea8bc_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' : + 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/samtools/idxstats/tests/main.nf.test b/modules/nf-core/samtools/idxstats/tests/main.nf.test index f6c92150..a2dcb27c 100644 --- a/modules/nf-core/samtools/idxstats/tests/main.nf.test +++ b/modules/nf-core/samtools/idxstats/tests/main.nf.test @@ -8,7 +8,7 @@ nextflow_process { tag "samtools" tag "samtools/idxstats" - test("BAM") { + test("bam") { when { params { @@ -16,11 +16,11 @@ nextflow_process { } process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) - ] + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true) + ]) """ } } @@ -28,8 +28,8 @@ nextflow_process { then { assertAll ( { assert process.success }, - { assert snapshot(process.out.idxstats).match() }, - { assert path(process.out.versions.get(0)).getText().contains("samtools") } + { assert snapshot(process.out.idxstats).match("idxstats") }, + { assert snapshot(process.out.versions).match("versions") } ) } } diff --git a/modules/nf-core/samtools/idxstats/tests/main.nf.test.snap b/modules/nf-core/samtools/idxstats/tests/main.nf.test.snap index 4c6c12bd..a7050bdc 100644 --- a/modules/nf-core/samtools/idxstats/tests/main.nf.test.snap +++ b/modules/nf-core/samtools/idxstats/tests/main.nf.test.snap @@ -1,5 +1,17 @@ { - "BAM": { + "versions": { + "content": [ + [ + "versions.yml:md5,613dde56f108418039ffcdeeddba397a" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:16:50.147462763" + }, + "idxstats": { "content": [ [ [ @@ -11,6 +23,10 @@ ] ] ], - "timestamp": "2023-11-14T15:52:19.875194" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T18:36:41.561026" } } \ No newline at end of file diff --git a/modules/nf-core/samtools/index/environment.yml b/modules/nf-core/samtools/index/environment.yml index 81f09391..a5e50649 100644 --- a/modules/nf-core/samtools/index/environment.yml +++ b/modules/nf-core/samtools/index/environment.yml @@ -4,5 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::samtools=1.18 - - bioconda::htslib=1.18 + - bioconda::samtools=1.19.2 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf index 8ad18fdc..dc14f98d 100644 --- a/modules/nf-core/samtools/index/main.nf +++ b/modules/nf-core/samtools/index/main.nf @@ -4,8 +4,8 @@ process SAMTOOLS_INDEX { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1' : - 'biocontainers/samtools:1.18--h50ea8bc_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' : + 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: tuple val(meta), path(input) diff --git a/modules/nf-core/samtools/index/tests/main.nf.test b/modules/nf-core/samtools/index/tests/main.nf.test index c76a9169..bb7756d1 100644 --- a/modules/nf-core/samtools/index/tests/main.nf.test +++ b/modules/nf-core/samtools/index/tests/main.nf.test @@ -8,7 +8,7 @@ nextflow_process { tag "samtools" tag "samtools/index" - test("sarscov2 [BAI]") { + test("bai") { when { params { @@ -16,10 +16,10 @@ nextflow_process { } process { """ - input[0] = [ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ]) """ } } @@ -28,12 +28,12 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot(process.out.bai).match("bai") }, - { assert path(process.out.versions.get(0)).getText().contains("samtools") } + { assert snapshot(process.out.versions).match("bai_versions") } ) } } - test("homo_sapiens [CRAI]") { + test("crai") { when { params { @@ -41,10 +41,10 @@ nextflow_process { } process { """ - input[0] = [ - [ id:'test' ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_recalibrated_sorted_cram'], checkIfExists: true) - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram', checkIfExists: true) + ]) """ } } @@ -53,12 +53,12 @@ nextflow_process { assertAll ( { assert process.success }, { assert snapshot(process.out.crai).match("crai") }, - { assert path(process.out.versions.get(0)).getText().contains("samtools") } + { assert snapshot(process.out.versions).match("crai_versions") } ) } } - test("homo_sapiens [CSI]") { + test("csi") { config "./csi.nextflow.config" @@ -68,10 +68,10 @@ nextflow_process { } process { """ - input[0] = [ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ]) """ } } @@ -80,7 +80,7 @@ nextflow_process { assertAll ( { assert process.success }, { assert path(process.out.csi.get(0).get(1)).exists() }, - { assert path(process.out.versions.get(0)).getText().contains("samtools") } + { assert snapshot(process.out.versions).match("csi_versions") } ) } } diff --git a/modules/nf-core/samtools/index/tests/main.nf.test.snap b/modules/nf-core/samtools/index/tests/main.nf.test.snap index b3baee7f..3dc8e7de 100644 --- a/modules/nf-core/samtools/index/tests/main.nf.test.snap +++ b/modules/nf-core/samtools/index/tests/main.nf.test.snap @@ -1,28 +1,74 @@ { + "crai_versions": { + "content": [ + [ + "versions.yml:md5,cc4370091670b64bba7c7206403ffb3e" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:12:00.324667957" + }, + "csi_versions": { + "content": [ + [ + "versions.yml:md5,cc4370091670b64bba7c7206403ffb3e" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:12:07.885103162" + }, "crai": { "content": [ [ [ { - "id": "test" + "id": "test", + "single_end": false }, "test.paired_end.recalibrated.sorted.cram.crai:md5,14bc3bd5c89cacc8f4541f9062429029" ] ] ], - "timestamp": "2023-11-15T15:17:37.30801" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T18:41:38.446424" }, "bai": { "content": [ [ [ { - "id": "test" + "id": "test", + "single_end": false }, "test.paired_end.sorted.bam.bai:md5,704c10dd1326482448ca3073fdebc2f4" ] ] ], - "timestamp": "2023-11-15T15:17:30.869234" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T18:40:46.579747" + }, + "bai_versions": { + "content": [ + [ + "versions.yml:md5,cc4370091670b64bba7c7206403ffb3e" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:11:51.641425452" } } \ No newline at end of file diff --git a/modules/nf-core/samtools/sort/environment.yml b/modules/nf-core/samtools/sort/environment.yml index f4064b72..4d898e48 100644 --- a/modules/nf-core/samtools/sort/environment.yml +++ b/modules/nf-core/samtools/sort/environment.yml @@ -4,5 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::samtools=1.18 - - bioconda::htslib=1.18 + - bioconda::samtools=1.19.2 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf index 4a666d42..fc374f98 100644 --- a/modules/nf-core/samtools/sort/main.nf +++ b/modules/nf-core/samtools/sort/main.nf @@ -4,15 +4,18 @@ process SAMTOOLS_SORT { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1' : - 'biocontainers/samtools:1.18--h50ea8bc_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' : + 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: - tuple val(meta), path(bam) + tuple val(meta) , path(bam) + tuple val(meta2), path(fasta) output: - tuple val(meta), path("*.bam"), emit: bam - tuple val(meta), path("*.csi"), emit: csi, optional: true + tuple val(meta), path("*.bam"), emit: bam, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true + tuple val(meta), path("*.crai"), emit: crai, optional: true + tuple val(meta), path("*.csi"), emit: csi, optional: true path "versions.yml" , emit: versions when: @@ -21,14 +24,24 @@ process SAMTOOLS_SORT { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def extension = args.contains("--output-fmt sam") ? "sam" : + args.contains("--output-fmt cram") ? "cram" : + "bam" + def reference = fasta ? "--reference ${fasta}" : "" if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + """ + samtools cat \\ + --threads $task.cpus \\ + ${bam} \\ + | \\ samtools sort \\ $args \\ - -@ $task.cpus \\ - -o ${prefix}.bam \\ - -T $prefix \\ - $bam + -T ${prefix} \\ + --threads $task.cpus \\ + ${reference} \\ + -o ${prefix}.${extension} \\ + - cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -40,6 +53,7 @@ process SAMTOOLS_SORT { def prefix = task.ext.prefix ?: "${meta.id}" """ touch ${prefix}.bam + touch ${prefix}.bam.csi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/samtools/sort/meta.yml b/modules/nf-core/samtools/sort/meta.yml index 2200de72..341a7d0e 100644 --- a/modules/nf-core/samtools/sort/meta.yml +++ b/modules/nf-core/samtools/sort/meta.yml @@ -23,8 +23,18 @@ input: e.g. [ id:'test', single_end:false ] - bam: type: file - description: BAM/CRAM/SAM file + description: BAM/CRAM/SAM file(s) pattern: "*.{bam,cram,sam}" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] + - fasta: + type: file + description: Reference genome FASTA file + pattern: "*.{fa,fasta,fna}" + optional: true output: - meta: type: map @@ -33,19 +43,29 @@ output: e.g. [ id:'test', single_end:false ] - bam: type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - - versions: + description: Sorted BAM file + pattern: "*.{bam}" + - cram: type: file - description: File containing software versions - pattern: "versions.yml" + description: Sorted CRAM file + pattern: "*.{cram}" + - crai: + type: file + description: CRAM index file (optional) + pattern: "*.crai" - csi: type: file description: BAM index file (optional) pattern: "*.csi" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@ewels" + - "@matthdsm" maintainers: - "@drpatelh" - "@ewels" + - "@matthdsm" diff --git a/modules/nf-core/samtools/sort/tests/main.nf.test b/modules/nf-core/samtools/sort/tests/main.nf.test index abb80978..8360e2b1 100644 --- a/modules/nf-core/samtools/sort/tests/main.nf.test +++ b/modules/nf-core/samtools/sort/tests/main.nf.test @@ -8,22 +8,21 @@ nextflow_process { tag "samtools" tag "samtools/sort" - test("test_samtools_sort") { + test("bam") { config "./nextflow.config" when { - params { - outdir = "$outputDir" - } process { """ - input[0] = [ - [ id:'test', single_end:false ], - [ - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true) + ]) + input[1] = Channel.of([ + [ id:'fasta' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } @@ -34,13 +33,39 @@ nextflow_process { { assert snapshot(process.out).match() } ) } + } + + test("cram") { + + config "./nextflow.config" + + when { + process { + """ + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true) + ]) + input[1] = Channel.of([ + [ id:'fasta' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + """ + } + } + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } } - test("test_samtools_sort_stub") { + test("bam_stub") { config "./nextflow.config" - options "-stub-run" + options "-stub" when { params { @@ -48,12 +73,14 @@ nextflow_process { } process { """ - input[0] = [ - [ id:'test', single_end:false ], - [ - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true) + ]) + input[1] = Channel.of([ + [ id:'fasta' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } @@ -61,13 +88,9 @@ nextflow_process { then { assertAll ( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - process.out.versions - ).match() } + { assert snapshot(file(process.out.bam[0][1]).name).match("bam_stub_bam") }, + { assert snapshot(process.out.versions).match("bam_stub_versions") } ) } - } - } diff --git a/modules/nf-core/samtools/sort/tests/main.nf.test.snap b/modules/nf-core/samtools/sort/tests/main.nf.test.snap index ff722259..38477656 100644 --- a/modules/nf-core/samtools/sort/tests/main.nf.test.snap +++ b/modules/nf-core/samtools/sort/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "test_samtools_sort": { + "cram": { "content": [ { "0": [ @@ -8,14 +8,26 @@ "id": "test", "single_end": false }, - "test.sorted.bam:md5,ea6a0fef94eb534e901f107a05a33a06" + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" ] ], "1": [ ], "2": [ - "versions.yml:md5,33b6a403dc19a0d28e4219ccab0a1d80" + + ], + "3": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] + ], + "4": [ + "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" ], "bam": [ [ @@ -23,26 +35,120 @@ "id": "test", "single_end": false }, - "test.sorted.bam:md5,ea6a0fef94eb534e901f107a05a33a06" + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" ] ], - "csi": [ + "crai": [ + + ], + "cram": [ + ], + "csi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] ], "versions": [ - "versions.yml:md5,33b6a403dc19a0d28e4219ccab0a1d80" + "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" ] } ], - "timestamp": "2023-12-04T11:11:22.005628301" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-04T15:08:00.830294" + }, + "bam_stub_bam": { + "content": [ + "test.sorted.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:21:04.364044" }, - "test_samtools_sort_stub": { + "bam_stub_versions": { "content": [ - "test.sorted.bam", [ - "versions.yml:md5,33b6a403dc19a0d28e4219ccab0a1d80" + "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" ] ], - "timestamp": "2023-12-04T17:47:22.314445935" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:15:00.20800281" + }, + "bam": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" + ] + ], + "1": [ + + ], + "2": [ + + ], + "3": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] + ], + "4": [ + "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" + ], + "bam": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" + ] + ], + "crai": [ + + ], + "cram": [ + + ], + "csi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] + ], + "versions": [ + "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-04T15:07:48.773803" } } \ No newline at end of file diff --git a/modules/nf-core/samtools/sort/tests/nextflow.config b/modules/nf-core/samtools/sort/tests/nextflow.config index d0f35086..f642771f 100644 --- a/modules/nf-core/samtools/sort/tests/nextflow.config +++ b/modules/nf-core/samtools/sort/tests/nextflow.config @@ -1,7 +1,8 @@ process { withName: SAMTOOLS_SORT { - ext.prefix = { "${meta.id}.sorted" } + ext.prefix = { "${meta.id}.sorted" } + ext.args = "--write-index" } } diff --git a/modules/nf-core/samtools/stats/environment.yml b/modules/nf-core/samtools/stats/environment.yml index b45ba90c..67bb0ca4 100644 --- a/modules/nf-core/samtools/stats/environment.yml +++ b/modules/nf-core/samtools/stats/environment.yml @@ -4,5 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::samtools=1.18 - - bioconda::htslib=1.18 + - bioconda::samtools=1.19.2 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/samtools/stats/main.nf b/modules/nf-core/samtools/stats/main.nf index 7539140a..52b00f4b 100644 --- a/modules/nf-core/samtools/stats/main.nf +++ b/modules/nf-core/samtools/stats/main.nf @@ -4,8 +4,8 @@ process SAMTOOLS_STATS { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1' : - 'biocontainers/samtools:1.18--h50ea8bc_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' : + 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: tuple val(meta), path(input), path(input_index) diff --git a/modules/nf-core/samtools/stats/tests/main.nf.test b/modules/nf-core/samtools/stats/tests/main.nf.test index 20c3efe1..e3d5cb14 100644 --- a/modules/nf-core/samtools/stats/tests/main.nf.test +++ b/modules/nf-core/samtools/stats/tests/main.nf.test @@ -8,71 +8,58 @@ nextflow_process { tag "samtools" tag "samtools/stats" - test("SAMTOOLS STATS Should run without failures") { + test("bam") { when { params { - outdir = "$outputDir" } process { """ - // define inputs of the process here. - input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) - - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true) + ]) input[1] = [[],[]] """ - } } then { assertAll( - {assert process.success}, - {assert snapshot(process.out).match()} + {assert process.success}, + {assert snapshot(process.out).match()} ) } - } - test("SAMTOOLS CRAM Should run without failures") { + test("cram") { when { params { - outdir = "$outputDir" } process { """ - // define inputs of the process here - input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_recalibrated_sorted_cram'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_recalibrated_sorted_cram_crai'], checkIfExists: true) - - ] - input[1] = [ - [ id:'genome' ], - file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram.crai', checkIfExists: true) + ]) + input[1] = Channel.of([ + [ id:'genome' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) """ } - - } then { assertAll( - {assert process.success}, - {assert snapshot(process.out).match()} + {assert process.success}, + {assert snapshot(process.out).match()} ) } - } - - } diff --git a/modules/nf-core/samtools/stats/tests/main.nf.test.snap b/modules/nf-core/samtools/stats/tests/main.nf.test.snap index 025c83a5..1b7c9ba4 100644 --- a/modules/nf-core/samtools/stats/tests/main.nf.test.snap +++ b/modules/nf-core/samtools/stats/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "SAMTOOLS STATS Should run without failures": { + "cram": { "content": [ { "0": [ @@ -8,11 +8,11 @@ "id": "test", "single_end": false }, - "test.stats:md5,045a48208b1c6f5b8af4347fe31f4def" + "test.stats:md5,01812900aa4027532906c5d431114233" ] ], "1": [ - "versions.yml:md5,650a365c6635001436008350ae83337c" + "versions.yml:md5,0514ceb1769b2a88843e08c1f82624a9" ], "stats": [ [ @@ -20,17 +20,21 @@ "id": "test", "single_end": false }, - "test.stats:md5,045a48208b1c6f5b8af4347fe31f4def" + "test.stats:md5,01812900aa4027532906c5d431114233" ] ], "versions": [ - "versions.yml:md5,650a365c6635001436008350ae83337c" + "versions.yml:md5,0514ceb1769b2a88843e08c1f82624a9" ] } ], - "timestamp": "2023-12-04T11:07:28.26821485" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:15:25.562429714" }, - "SAMTOOLS CRAM Should run without failures": { + "bam": { "content": [ { "0": [ @@ -39,11 +43,11 @@ "id": "test", "single_end": false }, - "test.stats:md5,dfbfa130d4a6925ddd1931dcd8354a43" + "test.stats:md5,5d8681bf541199898c042bf400391d59" ] ], "1": [ - "versions.yml:md5,650a365c6635001436008350ae83337c" + "versions.yml:md5,0514ceb1769b2a88843e08c1f82624a9" ], "stats": [ [ @@ -51,14 +55,18 @@ "id": "test", "single_end": false }, - "test.stats:md5,dfbfa130d4a6925ddd1931dcd8354a43" + "test.stats:md5,5d8681bf541199898c042bf400391d59" ] ], "versions": [ - "versions.yml:md5,650a365c6635001436008350ae83337c" + "versions.yml:md5,0514ceb1769b2a88843e08c1f82624a9" ] } ], - "timestamp": "2023-12-04T11:07:50.356233402" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:15:07.857611509" } } \ No newline at end of file diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml index 73ce7999..b0676f33 100644 --- a/modules/nf-core/samtools/view/environment.yml +++ b/modules/nf-core/samtools/view/environment.yml @@ -4,5 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::samtools=1.18 - - bioconda::htslib=1.18 + - bioconda::samtools=1.19.2 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf index 0b5a2912..5a8989d6 100644 --- a/modules/nf-core/samtools/view/main.nf +++ b/modules/nf-core/samtools/view/main.nf @@ -4,8 +4,8 @@ process SAMTOOLS_VIEW { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1' : - 'biocontainers/samtools:1.18--h50ea8bc_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.19.2--h50ea8bc_0' : + 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: tuple val(meta), path(input), path(index) diff --git a/modules/nf-core/samtools/view/tests/main.nf.test b/modules/nf-core/samtools/view/tests/main.nf.test index 89ed3555..45a0defb 100644 --- a/modules/nf-core/samtools/view/tests/main.nf.test +++ b/modules/nf-core/samtools/view/tests/main.nf.test @@ -9,16 +9,16 @@ nextflow_process { tag "samtools" tag "samtools/view" - test("sarscov2 - [bam, []], [], []") { + test("bam") { when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true), [] - ] + ]) input[1] = [[],[]] input[2] = [] """ @@ -28,34 +28,31 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - process.out.cram, - process.out.sam, - process.out.bai, - process.out.crai, - process.out.csi, - process.out.versions - ).match() } + { assert snapshot(file(process.out.bam[0][1]).name).match("bam_bam") }, + { assert snapshot(process.out.bai).match("bam_bai") }, + { assert snapshot(process.out.crai).match("bam_crai") }, + { assert snapshot(process.out.cram).match("bam_cram") }, + { assert snapshot(process.out.csi).match("bam_csi") }, + { assert snapshot(process.out.sam).match("bam_sam") }, + { assert snapshot(process.out.versions).match("bam_versions") } ) } - } - test("homo_sapiens - [cram, crai], fasta, []") { + test("cram") { when { process { """ - input[0] = [ - [ id: 'test' ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram_crai'], checkIfExists: true) - ] - input[1] = [ - [ id:'genome' ], - file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - ] + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram.crai', checkIfExists: true) + ]) + input[1] = Channel.of([ + [ id:'genome' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) input[2] = [] """ } @@ -64,36 +61,33 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.cram[0][1]).name, - process.out.bam, - process.out.sam, - process.out.bai, - process.out.crai, - process.out.csi, - process.out.versions - ).match() } + { assert snapshot(file(process.out.cram[0][1]).name).match("cram_cram") }, + { assert snapshot(process.out.bai).match("cram_bai") }, + { assert snapshot(process.out.bam).match("cram_bam") }, + { assert snapshot(process.out.crai).match("cram_crai") }, + { assert snapshot(process.out.csi).match("cram_csi") }, + { assert snapshot(process.out.sam).match("cram_sam") }, + { assert snapshot(process.out.versions).match("cram_versions") } ) } - } - test("homo_sapiens - [cram, []], fasta, [] - bam output") { + test("cram_to_bam") { config "./bam.config" when { process { """ - input[0] = [ - [ id: 'test' ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram'], checkIfExists: true), + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true), [] - ] - input[1] = [ - [ id:'genome' ], - file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - ] + ]) + input[1] = Channel.of([ + [ id:'genome' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) input[2] = [] """ } @@ -102,36 +96,33 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - process.out.cram, - process.out.sam, - process.out.bai, - process.out.crai, - process.out.csi, - process.out.versions - ).match() } + { assert snapshot(file(process.out.bam[0][1]).name).match("cram_to_bam_bam") }, + { assert snapshot(process.out.bai).match("cram_to_bam_bai") }, + { assert snapshot(process.out.crai).match("cram_to_bam_crai") }, + { assert snapshot(process.out.cram).match("cram_to_bam_cram") }, + { assert snapshot(process.out.csi).match("cram_to_bam_csi") }, + { assert snapshot(process.out.sam).match("cram_to_bam_sam") }, + { assert snapshot(process.out.versions).match("cram_to_bam_versions") } ) } - } - test("homo_sapiens - [cram, []], fasta, [] - bam & index output") { + test("cram_to_bam_index") { config "./bam_index.config" when { process { """ - input[0] = [ - [ id: 'test' ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram'], checkIfExists: true), + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true), [] - ] - input[1] = [ - [ id:'genome' ], - file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - ] + ]) + input[1] = Channel.of([ + [ id:'genome' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) input[2] = [] """ } @@ -140,36 +131,33 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - process.out.cram, - process.out.sam, - file(process.out.csi[0][1]).name, - process.out.crai, - process.out.bai, - process.out.versions - ).match() } + { assert snapshot(file(process.out.bam[0][1]).name).match("cram_to_bam_index_bam") }, + { assert snapshot(file(process.out.csi[0][1]).name).match("cram_to_bam_index_csi") }, + { assert snapshot(process.out.bai).match("cram_to_bam_index_bai") }, + { assert snapshot(process.out.crai).match("cram_to_bam_index_crai") }, + { assert snapshot(process.out.cram).match("cram_to_bam_index_cram") }, + { assert snapshot(process.out.sam).match("cram_to_bam_index_sam") }, + { assert snapshot(process.out.versions).match("cram_to_bam_index_versions") } ) } - } - test("homo_sapiens - [cram, []], fasta, qname - bam & index output") { + test("cram_to_bam_index_qname") { config "./bam_index.config" when { process { """ - input[0] = [ - [ id: 'test' ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram'], checkIfExists: true), + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true), [] - ] - input[1] = [ - [ id:'genome' ], - file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) - ] + ]) + input[1] = Channel.of([ + [ id:'genome' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) input[2] = Channel.of("testN:2817", "testN:2814").collectFile(name: "readnames.list", newLine: true) """ } @@ -178,21 +166,18 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - process.out.cram, - process.out.sam, - file(process.out.csi[0][1]).name, - process.out.crai, - process.out.bai, - process.out.versions - ).match() } + { assert snapshot(file(process.out.bam[0][1]).name).match("cram_to_bam_index_qname_bam") }, + { assert snapshot(file(process.out.csi[0][1]).name).match("cram_to_bam_index_qname_csi") }, + { assert snapshot(process.out.bai).match("cram_to_bam_index_qname_bai") }, + { assert snapshot(process.out.crai).match("cram_to_bam_index_qname_crai") }, + { assert snapshot(process.out.cram).match("cram_to_bam_index_qname_cram") }, + { assert snapshot(process.out.sam).match("cram_to_bam_index_qname_sam") }, + { assert snapshot(process.out.versions).match("cram_to_bam_index_qname_versions") } ) } - } - test("sarscov2 - [bam, []], [], [] - stub") { + test("bam_stub") { options "-stub" config "./bam_index.config" @@ -200,11 +185,11 @@ nextflow_process { when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true), [] - ] + ]) input[1] = [[],[]] input[2] = [] """ @@ -214,18 +199,14 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - process.out.cram, - process.out.sam, - file(process.out.csi[0][1]).name, - process.out.crai, - process.out.bai, - process.out.versions - ).match() } + { assert snapshot(file(process.out.bam[0][1]).name).match("bam_stub_bam") }, + { assert snapshot(file(process.out.csi[0][1]).name).match("bam_stub_csi") }, + { assert snapshot(process.out.bai).match("bam_stub_bai") }, + { assert snapshot(process.out.crai).match("bam_stub_crai") }, + { assert snapshot(process.out.cram).match("bam_stub_cram") }, + { assert snapshot(process.out.sam).match("bam_stub_sam") }, + { assert snapshot(process.out.versions).match("bam_stub_versions") } ) } - } - } diff --git a/modules/nf-core/samtools/view/tests/main.nf.test.snap b/modules/nf-core/samtools/view/tests/main.nf.test.snap index 83427491..f55943a7 100644 --- a/modules/nf-core/samtools/view/tests/main.nf.test.snap +++ b/modules/nf-core/samtools/view/tests/main.nf.test.snap @@ -1,140 +1,488 @@ { - "homo_sapiens - [cram, []], fasta, [] - bam output": { + "bam_bam": { + "content": [ + "test.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:51.256068" + }, + "cram_to_bam_index_csi": { + "content": [ + "test.bam.csi" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:12.958617" + }, + "bam_stub_bam": { + "content": [ + "test.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:32.065301" + }, + "bam_bai": { "content": [ - "test.bam", [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:51.258578" + }, + "bam_stub_bai": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:32.071284" + }, + "bam_stub_versions": { + "content": [ [ - - ], + "versions.yml:md5,4ea32c57d546102a1b32d9693ada7cf1" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:13:09.713353823" + }, + "cram_to_bam_index_cram": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:12.972288" + }, + "cram_to_bam_sam": { + "content": [ [ - ], - [ - "versions.yml:md5,06b9049228b111e7bed5c52fe8a98d9b" ] ], - "timestamp": "2023-12-04T17:41:17.563069206" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:04.999247" }, - "sarscov2 - [bam, []], [], []": { + "cram_to_bam_index_sam": { "content": [ - "test.bam", [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:12.976457" + }, + "cram_crai": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:56.497581" + }, + "cram_csi": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:56.50038" + }, + "cram_to_bam_cram": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:04.992239" + }, + "cram_to_bam_index_qname_csi": { + "content": [ + "test.bam.csi" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:23.325496" + }, + "bam_stub_sam": { + "content": [ [ - ], - [ - "versions.yml:md5,06b9049228b111e7bed5c52fe8a98d9b" ] ], - "timestamp": "2023-12-04T17:41:03.206994564" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:32.079529" }, - "homo_sapiens - [cram, []], fasta, qname - bam & index output": { + "cram_cram": { + "content": [ + "test.cram" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:56.490286" + }, + "bam_csi": { "content": [ - "test.bam", [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:51.262882" + }, + "cram_to_bam_crai": { + "content": [ [ - ], - "test.bam.csi", + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:04.989247" + }, + "cram_to_bam_index_crai": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:12.967681" + }, + "cram_to_bam_index_qname_versions": { + "content": [ [ - - ], + "versions.yml:md5,4ea32c57d546102a1b32d9693ada7cf1" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:13:03.935041046" + }, + "cram_to_bam_bam": { + "content": [ + "test.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:04.982361" + }, + "cram_to_bam_index_bam": { + "content": [ + "test.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:12.95456" + }, + "cram_to_bam_index_versions": { + "content": [ [ - "versions.yml:md5,06b9049228b111e7bed5c52fe8a98d9b" + "versions.yml:md5,4ea32c57d546102a1b32d9693ada7cf1" ] ], - "timestamp": "2023-12-04T17:44:39.165289759" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:12:55.910685496" }, - "homo_sapiens - [cram, []], fasta, [] - bam & index output": { + "cram_to_bam_bai": { "content": [ - "test.bam", [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:04.98601" + }, + "cram_to_bam_versions": { + "content": [ [ - - ], - "test.bam.csi", + "versions.yml:md5,4ea32c57d546102a1b32d9693ada7cf1" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:12:47.715221169" + }, + "cram_bam": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:56.495512" + }, + "bam_stub_cram": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:32.076908" + }, + "cram_to_bam_index_qname_bai": { + "content": [ [ - "versions.yml:md5,06b9049228b111e7bed5c52fe8a98d9b" + ] ], - "timestamp": "2023-12-04T17:44:32.25731224" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:23.328458" }, - "sarscov2 - [bam, []], [], [] - stub": { + "cram_to_bam_index_qname_crai": { "content": [ - "test.bam", [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:23.330789" + }, + "cram_bai": { + "content": [ [ - ], - "test.csi", + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:56.493129" + }, + "bam_stub_crai": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:32.074313" + }, + "cram_to_bam_index_qname_bam": { + "content": [ + "test.bam" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:23.322874" + }, + "bam_versions": { + "content": [ + [ + "versions.yml:md5,4ea32c57d546102a1b32d9693ada7cf1" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:12:31.692607421" + }, + "cram_to_bam_index_qname_cram": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:23.333248" + }, + "bam_crai": { + "content": [ [ - "versions.yml:md5,06b9049228b111e7bed5c52fe8a98d9b" + ] ], - "timestamp": "2023-12-04T17:44:45.81037195" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:51.259774" }, - "homo_sapiens - [cram, crai], fasta, []": { + "bam_cram": { "content": [ - "test.cram", [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:51.261287" + }, + "cram_to_bam_csi": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:04.995454" + }, + "cram_sam": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:56.502625" + }, + "cram_versions": { + "content": [ + [ + "versions.yml:md5,4ea32c57d546102a1b32d9693ada7cf1" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:12:39.913411036" + }, + "bam_sam": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:37:51.264651" + }, + "cram_to_bam_index_bai": { + "content": [ [ - ], + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:12.962863" + }, + "cram_to_bam_index_qname_sam": { + "content": [ [ - "versions.yml:md5,06b9049228b111e7bed5c52fe8a98d9b" + ] ], - "timestamp": "2023-12-04T17:41:10.730011823" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:23.337634" + }, + "bam_stub_csi": { + "content": [ + "test.csi" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.04.3" + }, + "timestamp": "2024-02-12T19:38:32.068596" } } \ No newline at end of file diff --git a/modules/nf-core/tabix/bgzip/environment.yml b/modules/nf-core/tabix/bgzip/environment.yml index 4fe40c56..361c078b 100644 --- a/modules/nf-core/tabix/bgzip/environment.yml +++ b/modules/nf-core/tabix/bgzip/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::tabix=1.11 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/tabix/bgzip/main.nf b/modules/nf-core/tabix/bgzip/main.nf index 7772e9ad..3065dab0 100644 --- a/modules/nf-core/tabix/bgzip/main.nf +++ b/modules/nf-core/tabix/bgzip/main.nf @@ -4,8 +4,8 @@ process TABIX_BGZIP { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0' : - 'biocontainers/tabix:1.11--hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/htslib:1.19.1--h81da01d_1' : + 'biocontainers/htslib:1.19.1--h81da01d_1' }" input: tuple val(meta), path(input) @@ -44,7 +44,8 @@ process TABIX_BGZIP { output = in_bgzip ? input.getBaseName() : "${prefix}.${input.getExtension()}.gz" """ - touch ${output} + echo "" | gzip > ${output} + touch ${output}.gzi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/bgzip/tests/bgzip_compress.config b/modules/nf-core/tabix/bgzip/tests/bgzip_compress.config new file mode 100644 index 00000000..6b6ff55f --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/bgzip_compress.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_BGZIP { + ext.args = ' -i' + } +} diff --git a/modules/nf-core/tabix/bgzip/tests/main.nf.test b/modules/nf-core/tabix/bgzip/tests/main.nf.test new file mode 100644 index 00000000..95fd4c50 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/main.nf.test @@ -0,0 +1,111 @@ +nextflow_process { + + name "Test Process TABIX_BGZIP" + script "modules/nf-core/tabix/bgzip/main.nf" + process "TABIX_BGZIP" + + tag "modules" + tag "modules_nfcore" + tag "tabix" + tag "tabix/bgzip" + + test("sarscov2_vcf_bgzip_compress") { + when { + process { + """ + input[0] = [ + [ id:'bgzip_test' ], + [ file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.output[0][1]).name + ).match("bgzip_test") + } + ) + } + } + + test("homo_genome_bedgz_compress") { + when { + process { + """ + input[0] = [ + [ id:'bedgz_test' ], + [ file(params.test_data['homo_sapiens']['genome']['genome_bed_gz'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.output[0][1]).name + ).match("bedgz_test") + } + ) + } + } + + test("sarscov2_vcf_bgzip_compress_stub") { + options '-stub' + config "./bgzip_compress.config" + + when { + process { + """ + input[0] = [ + [ id:"test_stub" ], + [ file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.output[0][1]).name + ).match("test_stub") + } + ) + } + } + + test("sarscov2_vcf_bgzip_compress_gzi") { + config "./bgzip_compress.config" + when { + process { + """ + input[0] = [ + [ id:"gzi_compress_test" ], + [ file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.gzi[0][1]).name + ).match("gzi_compress_test") + } + ) + } + } +} diff --git a/modules/nf-core/tabix/bgzip/tests/main.nf.test.snap b/modules/nf-core/tabix/bgzip/tests/main.nf.test.snap new file mode 100644 index 00000000..53d59932 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/main.nf.test.snap @@ -0,0 +1,186 @@ +{ + "gzi_compress_test": { + "content": [ + "gzi_compress_test.vcf.gz.gzi" + ], + "timestamp": "2024-02-19T14:52:29.328146" + }, + "homo_genome_bedgz_compress": { + "content": [ + { + "0": [ + [ + { + "id": "bedgz_test" + }, + "bedgz_test.bed:md5,87a15eb9c2ff20ccd5cd8735a28708f7" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ], + "gzi": [ + + ], + "output": [ + [ + { + "id": "bedgz_test" + }, + "bedgz_test.bed:md5,87a15eb9c2ff20ccd5cd8735a28708f7" + ] + ], + "versions": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ] + } + ], + "timestamp": "2024-02-19T14:52:12.422209" + }, + "test_stub": { + "content": [ + "test_stub.vcf.gz" + ], + "timestamp": "2024-02-19T14:52:20.811489" + }, + "sarscov2_vcf_bgzip_compress": { + "content": [ + { + "0": [ + [ + { + "id": "bgzip_test" + }, + "bgzip_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ], + "gzi": [ + + ], + "output": [ + [ + { + "id": "bgzip_test" + }, + "bgzip_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "versions": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ] + } + ], + "timestamp": "2024-02-19T14:52:03.706028" + }, + "sarscov2_vcf_bgzip_compress_gzi": { + "content": [ + { + "0": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "1": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz.gzi:md5,26fd00d4e26141cd11561f6e7d4a2ad0" + ] + ], + "2": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ], + "gzi": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz.gzi:md5,26fd00d4e26141cd11561f6e7d4a2ad0" + ] + ], + "output": [ + [ + { + "id": "gzi_compress_test" + }, + "gzi_compress_test.vcf.gz:md5,8e722884ffb75155212a3fc053918766" + ] + ], + "versions": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ] + } + ], + "timestamp": "2024-02-19T14:52:29.271494" + }, + "bgzip_test": { + "content": [ + "bgzip_test.vcf.gz" + ], + "timestamp": "2024-02-19T14:52:03.768295" + }, + "bedgz_test": { + "content": [ + "bedgz_test.bed" + ], + "timestamp": "2024-02-19T14:52:12.453855" + }, + "sarscov2_vcf_bgzip_compress_stub": { + "content": [ + { + "0": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "1": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz.gzi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ], + "gzi": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz.gzi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "output": [ + [ + { + "id": "test_stub" + }, + "test_stub.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,e023292de6ee109a44fc67475d658174" + ] + } + ], + "timestamp": "2024-02-19T14:52:20.769619" + } +} \ No newline at end of file diff --git a/modules/nf-core/tabix/bgzip/tests/tags.yml b/modules/nf-core/tabix/bgzip/tests/tags.yml new file mode 100644 index 00000000..de0eec86 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/tags.yml @@ -0,0 +1,2 @@ +tabix/bgzip: + - "modules/nf-core/tabix/bgzip/**" diff --git a/modules/nf-core/tabix/bgzip/tests/vcf_none.config b/modules/nf-core/tabix/bgzip/tests/vcf_none.config new file mode 100644 index 00000000..f3a3c467 --- /dev/null +++ b/modules/nf-core/tabix/bgzip/tests/vcf_none.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_BGZIP { + ext.args = '' + } +} diff --git a/modules/nf-core/tabix/tabix/environment.yml b/modules/nf-core/tabix/tabix/environment.yml index 7167fb87..76b45e16 100644 --- a/modules/nf-core/tabix/tabix/environment.yml +++ b/modules/nf-core/tabix/tabix/environment.yml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - bioconda::tabix=1.11 + - bioconda::htslib=1.19.1 diff --git a/modules/nf-core/tabix/tabix/main.nf b/modules/nf-core/tabix/tabix/main.nf index c304a8a3..1737141d 100644 --- a/modules/nf-core/tabix/tabix/main.nf +++ b/modules/nf-core/tabix/tabix/main.nf @@ -4,8 +4,8 @@ process TABIX_TABIX { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0' : - 'biocontainers/tabix:1.11--hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/htslib:1.19.1--h81da01d_1' : + 'biocontainers/htslib:1.19.1--h81da01d_1' }" input: tuple val(meta), path(tab) @@ -30,9 +30,9 @@ process TABIX_TABIX { """ stub: - def prefix = task.ext.prefix ?: "${meta.id}" """ touch ${tab}.tbi + touch ${tab}.csi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/tabix/tests/main.nf.test b/modules/nf-core/tabix/tabix/tests/main.nf.test new file mode 100644 index 00000000..3a150c70 --- /dev/null +++ b/modules/nf-core/tabix/tabix/tests/main.nf.test @@ -0,0 +1,142 @@ +nextflow_process { + + name "Test Process TABIX_TABIX" + script "modules/nf-core/tabix/tabix/main.nf" + process "TABIX_TABIX" + + tag "modules" + tag "modules_nfcore" + tag "tabix" + tag "tabix/tabix" + + test("sarscov2_bedgz_tbi") { + config "./tabix_bed.config" + when { + process { + """ + input[0] = [ + [ id:'tbi_bed' ], + [ file(params.test_data['sarscov2']['genome']['test_bed_gz'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.tbi[0][1]).name + ).match("tbi_bed") + } + ) + } + } + + test("sarscov2_gff_tbi") { + config "./tabix_gff.config" + when { + process { + """ + input[0] = [ + [ id:'tbi_gff' ], + [ file(params.test_data['sarscov2']['genome']['genome_gff3_gz'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.tbi[0][1]).name + ).match("tbi_gff") + } + ) + } + + } + + test("sarscov2_vcf_tbi") { + config "./tabix_vcf_tbi.config" + when { + process { + """ + input[0] = [ + [ id:'tbi_vcf' ], + [ file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.tbi[0][1]).name + ).match("tbi_vcf") + } + ) + } + + } + + test("sarscov2_vcf_csi") { + config "./tabix_vcf_csi.config" + when { + process { + """ + input[0] = [ + [ id:'vcf_csi' ], + [ file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.csi[0][1]).name + ).match("vcf_csi") + } + ) + } + + } + + test("sarscov2_vcf_csi_stub") { + config "./tabix_vcf_csi.config" + options "-stub" + when { + process { + """ + input[0] = [ + [ id:'vcf_csi_stub' ], + [ file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true) ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + { assert snapshot( + file(process.out.csi[0][1]).name + ).match("vcf_csi_stub") + } + ) + } + + } + +} diff --git a/modules/nf-core/tabix/tabix/tests/main.nf.test.snap b/modules/nf-core/tabix/tabix/tests/main.nf.test.snap new file mode 100644 index 00000000..034e38b6 --- /dev/null +++ b/modules/nf-core/tabix/tabix/tests/main.nf.test.snap @@ -0,0 +1,217 @@ +{ + "vcf_csi_stub": { + "content": [ + "test.vcf.gz.csi" + ], + "timestamp": "2024-03-04T14:51:59.788002" + }, + "tbi_gff": { + "content": [ + "genome.gff3.gz.tbi" + ], + "timestamp": "2024-02-19T14:53:37.420216" + }, + "sarscov2_gff_tbi": { + "content": [ + { + "0": [ + [ + { + "id": "tbi_gff" + }, + "genome.gff3.gz.tbi:md5,53fc683fd217aae47ef10d23c52a9178" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ], + "csi": [ + + ], + "tbi": [ + [ + { + "id": "tbi_gff" + }, + "genome.gff3.gz.tbi:md5,53fc683fd217aae47ef10d23c52a9178" + ] + ], + "versions": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ] + } + ], + "timestamp": "2024-02-19T14:53:37.388157" + }, + "sarscov2_bedgz_tbi": { + "content": [ + { + "0": [ + [ + { + "id": "tbi_bed" + }, + "test.bed.gz.tbi:md5,0f17d85e7f0a042b2aa367b70df224f8" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ], + "csi": [ + + ], + "tbi": [ + [ + { + "id": "tbi_bed" + }, + "test.bed.gz.tbi:md5,0f17d85e7f0a042b2aa367b70df224f8" + ] + ], + "versions": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ] + } + ], + "timestamp": "2024-02-19T14:53:28.879408" + }, + "tbi_vcf": { + "content": [ + "test.vcf.gz.tbi" + ], + "timestamp": "2024-02-19T14:53:46.402522" + }, + "vcf_csi": { + "content": [ + "test.vcf.gz.csi" + ], + "timestamp": "2024-02-19T14:53:54.921189" + }, + "sarscov2_vcf_tbi": { + "content": [ + { + "0": [ + [ + { + "id": "tbi_vcf" + }, + "test.vcf.gz.tbi:md5,897f3f378a811b90e6dee56ce08d2bcf" + ] + ], + "1": [ + + ], + "2": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ], + "csi": [ + + ], + "tbi": [ + [ + { + "id": "tbi_vcf" + }, + "test.vcf.gz.tbi:md5,897f3f378a811b90e6dee56ce08d2bcf" + ] + ], + "versions": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ] + } + ], + "timestamp": "2024-02-19T14:53:46.370358" + }, + "sarscov2_vcf_csi_stub": { + "content": [ + { + "0": [ + [ + { + "id": "vcf_csi_stub" + }, + "test.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "vcf_csi_stub" + }, + "test.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,3d45df6d80883bad358631069a2940fd" + ], + "csi": [ + [ + { + "id": "vcf_csi_stub" + }, + "test.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "tbi": [ + [ + { + "id": "vcf_csi_stub" + }, + "test.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,3d45df6d80883bad358631069a2940fd" + ] + } + ], + "timestamp": "2024-03-04T14:51:59.766184" + }, + "sarscov2_vcf_csi": { + "content": [ + { + "0": [ + + ], + "1": [ + [ + { + "id": "vcf_csi" + }, + "test.vcf.gz.csi:md5,0731ad6f40104d2bbb1a2cc478ef8f03" + ] + ], + "2": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ], + "csi": [ + [ + { + "id": "vcf_csi" + }, + "test.vcf.gz.csi:md5,0731ad6f40104d2bbb1a2cc478ef8f03" + ] + ], + "tbi": [ + + ], + "versions": [ + "versions.yml:md5,f4feeda7fdd4b567102f7f8e5d7037a3" + ] + } + ], + "timestamp": "2024-02-19T14:53:54.886876" + }, + "tbi_bed": { + "content": [ + "test.bed.gz.tbi" + ], + "timestamp": "2024-02-19T14:53:28.947628" + } +} \ No newline at end of file diff --git a/modules/nf-core/tabix/tabix/tests/tabix_bed.config b/modules/nf-core/tabix/tabix/tests/tabix_bed.config new file mode 100644 index 00000000..7ff05905 --- /dev/null +++ b/modules/nf-core/tabix/tabix/tests/tabix_bed.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_TABIX { + ext.args = '-p bed' + } +} \ No newline at end of file diff --git a/modules/nf-core/tabix/tabix/tests/tabix_gff.config b/modules/nf-core/tabix/tabix/tests/tabix_gff.config new file mode 100644 index 00000000..20c0a1e3 --- /dev/null +++ b/modules/nf-core/tabix/tabix/tests/tabix_gff.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_TABIX { + ext.args = '-p gff' + } +} \ No newline at end of file diff --git a/modules/nf-core/tabix/tabix/tests/tabix_vcf_csi.config b/modules/nf-core/tabix/tabix/tests/tabix_vcf_csi.config new file mode 100644 index 00000000..eb4f2d7e --- /dev/null +++ b/modules/nf-core/tabix/tabix/tests/tabix_vcf_csi.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_TABIX { + ext.args = '-p vcf --csi' + } +} diff --git a/modules/nf-core/tabix/tabix/tests/tabix_vcf_tbi.config b/modules/nf-core/tabix/tabix/tests/tabix_vcf_tbi.config new file mode 100644 index 00000000..2774c8a9 --- /dev/null +++ b/modules/nf-core/tabix/tabix/tests/tabix_vcf_tbi.config @@ -0,0 +1,5 @@ +process { + withName: TABIX_TABIX { + ext.args = '-p vcf' + } +} \ No newline at end of file diff --git a/modules/nf-core/tabix/tabix/tests/tags.yml b/modules/nf-core/tabix/tabix/tests/tags.yml new file mode 100644 index 00000000..6eda0653 --- /dev/null +++ b/modules/nf-core/tabix/tabix/tests/tags.yml @@ -0,0 +1,2 @@ +tabix/tabix: + - "modules/nf-core/tabix/tabix/**" diff --git a/modules/nf-core/untar/tests/main.nf.test b/modules/nf-core/untar/tests/main.nf.test index 679e83c7..2a7c97bf 100644 --- a/modules/nf-core/untar/tests/main.nf.test +++ b/modules/nf-core/untar/tests/main.nf.test @@ -3,17 +3,12 @@ nextflow_process { name "Test Process UNTAR" script "../main.nf" process "UNTAR" - tag "modules" tag "modules_nfcore" tag "untar" - test("test_untar") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = [ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) ] @@ -33,9 +28,6 @@ nextflow_process { test("test_untar_onlyfiles") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = [ [], file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) ] diff --git a/modules/nf-core/untar/tests/main.nf.test.snap b/modules/nf-core/untar/tests/main.nf.test.snap index ace42576..64550292 100644 --- a/modules/nf-core/untar/tests/main.nf.test.snap +++ b/modules/nf-core/untar/tests/main.nf.test.snap @@ -12,7 +12,11 @@ ] ] ], - "timestamp": "2023-10-18T11:56:46.878844" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:49:41.320643" }, "test_untar": { "content": [ @@ -29,6 +33,10 @@ ] ] ], - "timestamp": "2023-10-18T11:56:08.16574" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:49:33.795172" } } \ No newline at end of file diff --git a/nextflow_schema.json b/nextflow_schema.json index 62b78d6e..893346fd 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -15,9 +15,10 @@ "input": { "type": "string", "format": "file-path", + "exists": true, + "schema": "assets/schema_input.json", "mimetype": "text/csv", "pattern": "^\\S+\\.csv$", - "schema": "assets/schema_input.json", "fa_icon": "fas fa-file-csv", "help_text": "You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/viralrecon/docs/usage#introduction).", "description": "Path to comma-separated file containing information about the samples you would like to analyse." diff --git a/pyproject.toml b/pyproject.toml index 7d08e1c8..56110621 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,13 @@ [tool.ruff] line-length = 120 target-version = "py38" -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" -[tool.ruff.isort] +[tool.ruff.lint] +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] + +[tool.ruff.lint.isort] known-first-party = ["nf_core"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401"] diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf deleted file mode 100644 index 9d83d9f1..00000000 --- a/subworkflows/local/input_check.nf +++ /dev/null @@ -1,61 +0,0 @@ -// -// Check input samplesheet and get read channels -// - -include { SAMPLESHEET_CHECK } from '../../modules/local/samplesheet_check' - -workflow INPUT_CHECK { - take: - samplesheet // file : /path/to/samplesheet.csv - platform // string: sequencing platform. Accepted values: 'illumina', 'nanopore' - - main: - - SAMPLESHEET_CHECK ( - samplesheet, - platform - ) - - if (platform == 'illumina') { - SAMPLESHEET_CHECK - .out - .csv - .splitCsv ( header:true, sep:',' ) - .map { create_fastq_channel(it) } - .set { sample_info } - } else if (platform == 'nanopore') { - SAMPLESHEET_CHECK - .out - .csv - .splitCsv ( header:true, sep:',' ) - .map { row -> [ row.barcode, row.sample ] } - .set { sample_info } - } - - emit: - sample_info // channel: [ val(meta), [ reads ] ] - versions = SAMPLESHEET_CHECK.out.versions // channel: [ versions.yml ] -} - -// Function to get list of [ meta, [ fastq_1, fastq_2 ] ] -def create_fastq_channel(LinkedHashMap row) { - // create meta map - def meta = [:] - meta.id = row.sample - meta.single_end = row.single_end.toBoolean() - - // add path(s) of the fastq file(s) to the meta map - def fastq_meta = [] - if (!file(row.fastq_1).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 1 FastQ file does not exist!\n${row.fastq_1}" - } - if (meta.single_end) { - fastq_meta = [ meta, [ file(row.fastq_1) ] ] - } else { - if (!file(row.fastq_2).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 2 FastQ file does not exist!\n${row.fastq_2}" - } - fastq_meta = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] - } - return fastq_meta -} diff --git a/subworkflows/local/prepare_genome_illumina.nf b/subworkflows/local/prepare_genome_illumina.nf index f55aef83..8ebacb0f 100644 --- a/subworkflows/local/prepare_genome_illumina.nf +++ b/subworkflows/local/prepare_genome_illumina.nf @@ -20,6 +20,18 @@ include { KRAKEN2_BUILD } from '../../modules/local/kraken2_buil include { SNPEFF_BUILD } from '../../modules/local/snpeff_build' workflow PREPARE_GENOME { + + take: + fasta + gff + primer_bed + bowtie2_index + nextclade_dataset + nextclade_dataset_name + nextclade_dataset_reference + nextclade_dataset_tag + + main: ch_versions = Channel.empty() @@ -27,29 +39,29 @@ workflow PREPARE_GENOME { // // Uncompress genome fasta file if required // - if (params.fasta.endsWith('.gz')) { + if (fasta.endsWith('.gz')) { GUNZIP_FASTA ( - [ [:], params.fasta ] + [ [:], fasta ] ) ch_fasta = GUNZIP_FASTA.out.gunzip.map { it[1] } ch_versions = ch_versions.mix(GUNZIP_FASTA.out.versions) } else { - ch_fasta = Channel.value(file(params.fasta)) + ch_fasta = Channel.value(file(fasta)) } // // Uncompress GFF annotation file // ch_gff = Channel.empty() - if (params.gff) { - if (params.gff.endsWith('.gz')) { + if (gff) { + if (gff.endsWith('.gz')) { GUNZIP_GFF ( - [ [:], params.gff ] + [ [:], gff ] ) ch_gff = GUNZIP_GFF.out.gunzip.map { it[1] } ch_versions = ch_versions.mix(GUNZIP_GFF.out.versions) } else { - ch_gff = Channel.value(file(params.gff)) + ch_gff = Channel.value(file(gff)) } } @@ -94,15 +106,15 @@ workflow PREPARE_GENOME { ch_primer_fasta = Channel.empty() ch_primer_collapsed_bed = Channel.empty() if (params.protocol == 'amplicon') { - if (params.primer_bed) { - if (params.primer_bed.endsWith('.gz')) { + if (primer_bed) { + if (primer_bed.endsWith('.gz')) { GUNZIP_PRIMER_BED ( - [ [:], params.primer_bed ] + [ [:], primer_bed ] ) ch_primer_bed = GUNZIP_PRIMER_BED.out.gunzip.map { it[1] } ch_versions = ch_versions.mix(GUNZIP_PRIMER_BED.out.versions) } else { - ch_primer_bed = Channel.value(file(params.primer_bed)) + ch_primer_bed = Channel.value(file(primer_bed)) } } @@ -143,15 +155,15 @@ workflow PREPARE_GENOME { // ch_bowtie2_index = Channel.empty() if (!params.skip_variants) { - if (params.bowtie2_index) { - if (params.bowtie2_index.endsWith('.tar.gz')) { + if (bowtie2_index) { + if (bowtie2_index.endsWith('.tar.gz')) { UNTAR_BOWTIE2_INDEX ( - [ [:], params.bowtie2_index ] + [ [:], bowtie2_index ] ) ch_bowtie2_index = UNTAR_BOWTIE2_INDEX.out.untar.map { it[1] } ch_versions = ch_versions.mix(UNTAR_BOWTIE2_INDEX.out.versions) } else { - ch_bowtie2_index = Channel.value(file(params.bowtie2_index)) + ch_bowtie2_index = Channel.value(file(bowtie2_index)) } } else { BOWTIE2_BUILD ( @@ -167,21 +179,21 @@ workflow PREPARE_GENOME { // ch_nextclade_db = Channel.empty() if (!params.skip_consensus && !params.skip_nextclade) { - if (params.nextclade_dataset) { - if (params.nextclade_dataset.endsWith('.tar.gz')) { + if (nextclade_dataset) { + if (nextclade_dataset.endsWith('.tar.gz')) { UNTAR_NEXTCLADE_DB ( - [ [:], params.nextclade_dataset ] + [ [:], nextclade_dataset ] ) ch_nextclade_db = UNTAR_NEXTCLADE_DB.out.untar.map { it[1] } ch_versions = ch_versions.mix(UNTAR_NEXTCLADE_DB.out.versions) } else { - ch_nextclade_db = Channel.value(file(params.nextclade_dataset)) + ch_nextclade_db = Channel.value(file(nextclade_dataset)) } - } else if (params.nextclade_dataset_name) { + } else if (nextclade_dataset_name) { NEXTCLADE_DATASETGET ( - params.nextclade_dataset_name, - params.nextclade_dataset_reference, - params.nextclade_dataset_tag + nextclade_dataset_name, + nextclade_dataset_reference, + nextclade_dataset_tag ) ch_nextclade_db = NEXTCLADE_DATASETGET.out.dataset ch_versions = ch_versions.mix(NEXTCLADE_DATASETGET.out.versions) diff --git a/subworkflows/local/prepare_genome_nanopore.nf b/subworkflows/local/prepare_genome_nanopore.nf index 1153b709..77a30645 100644 --- a/subworkflows/local/prepare_genome_nanopore.nf +++ b/subworkflows/local/prepare_genome_nanopore.nf @@ -12,6 +12,17 @@ include { COLLAPSE_PRIMERS } from '../../modules/local/collapse_prime include { SNPEFF_BUILD } from '../../modules/local/snpeff_build' workflow PREPARE_GENOME { + + take: + fasta + gff + primer_bed + bowtie2_index + nextclade_dataset + nextclade_dataset_name + nextclade_dataset_reference + nextclade_dataset_tag + main: ch_versions = Channel.empty() @@ -19,29 +30,29 @@ workflow PREPARE_GENOME { // // Uncompress genome fasta file if required // - if (params.fasta.endsWith('.gz')) { + if (fasta.endsWith('.gz')) { GUNZIP_FASTA ( - [ [:], params.fasta ] + [ [:], fasta ] ) ch_fasta = GUNZIP_FASTA.out.gunzip.map { it[1] } ch_versions = ch_versions.mix(GUNZIP_FASTA.out.versions) } else { - ch_fasta = Channel.value(file(params.fasta)) + ch_fasta = Channel.value(file(fasta)) } // // Uncompress GFF annotation file // ch_gff = Channel.empty() - if (params.gff) { - if (params.gff.endsWith('.gz')) { + if (gff) { + if (gff.endsWith('.gz')) { GUNZIP_GFF ( - [ [:], params.gff ] + [ [:], gff ] ) ch_gff = GUNZIP_GFF.out.gunzip.map { it[1] } ch_versions = ch_versions.mix(GUNZIP_GFF.out.versions) } else { - ch_gff = Channel.value(file(params.gff)) + ch_gff = Channel.value(file(gff)) } } @@ -59,15 +70,15 @@ workflow PREPARE_GENOME { // Uncompress primer BED file // ch_primer_bed = Channel.empty() - if (params.primer_bed) { - if (params.primer_bed.endsWith('.gz')) { + if (primer_bed) { + if (primer_bed.endsWith('.gz')) { GUNZIP_PRIMER_BED ( - [ [:], params.primer_bed ] + [ [:], primer_bed ] ) ch_primer_bed = GUNZIP_PRIMER_BED.out.gunzip.map { it[1] } ch_versions = ch_versions.mix(GUNZIP_PRIMER_BED.out.versions) } else { - ch_primer_bed = Channel.value(file(params.primer_bed)) + ch_primer_bed = Channel.value(file(primer_bed)) } } @@ -90,21 +101,21 @@ workflow PREPARE_GENOME { // ch_nextclade_db = Channel.empty() if (!params.skip_consensus && !params.skip_nextclade) { - if (params.nextclade_dataset) { - if (params.nextclade_dataset.endsWith('.tar.gz')) { + if (nextclade_dataset) { + if (nextclade_dataset.endsWith('.tar.gz')) { UNTAR ( - [ [:], params.nextclade_dataset ] + [ [:], nextclade_dataset ] ) ch_nextclade_db = UNTAR.out.untar.map { it[1] } ch_versions = ch_versions.mix(UNTAR.out.versions) } else { - ch_nextclade_db = Channel.value(file(params.nextclade_dataset)) + ch_nextclade_db = Channel.value(file(nextclade_dataset)) } - } else if (params.nextclade_dataset_name) { + } else if (nextclade_dataset_name) { NEXTCLADE_DATASETGET ( - params.nextclade_dataset_name, - params.nextclade_dataset_reference, - params.nextclade_dataset_tag + nextclade_dataset_name, + nextclade_dataset_reference, + nextclade_dataset_tag ) ch_nextclade_db = NEXTCLADE_DATASETGET.out.dataset ch_versions = ch_versions.mix(NEXTCLADE_DATASETGET.out.versions) diff --git a/subworkflows/local/utils_nfcore_viralrecon_pipeline/main.nf b/subworkflows/local/utils_nfcore_viralrecon_pipeline/main.nf new file mode 100644 index 00000000..ed116d33 --- /dev/null +++ b/subworkflows/local/utils_nfcore_viralrecon_pipeline/main.nf @@ -0,0 +1,309 @@ +// +// Subworkflow with functionality specific to the nf-core/viralrecon pipeline +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' +include { paramsSummaryMap } from 'plugin/nf-validation' +include { fromSamplesheet } from 'plugin/nf-validation' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' +include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' +include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' +include { imNotification } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' +include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' + +/* +======================================================================================== + SUBWORKFLOW TO INITIALISE PIPELINE +======================================================================================== +*/ + +workflow PIPELINE_INITIALISATION { + + take: + version // boolean: Display version and exit + help // boolean: Display help text + validate_params // boolean: Boolean whether to validate parameters against the schema at runtime + monochrome_logs // boolean: Do not use coloured log outputs + nextflow_cli_args // array: List of positional nextflow CLI args + outdir // string: The output directory where the results will be saved + input // string: Path to input samplesheet + + main: + + ch_versions = Channel.empty() + + // + // Print version and exit if required and dump pipeline parameters to JSON file + // + UTILS_NEXTFLOW_PIPELINE ( + version, + true, + outdir, + workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 + ) + + // + // Validate parameters and generate parameter summary to stdout + // + pre_help_text = nfCoreLogo(monochrome_logs) + post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) + def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " + UTILS_NFVALIDATION_PLUGIN ( + help, + workflow_command, + pre_help_text, + post_help_text, + validate_params, + "nextflow_schema.json" + ) + + // + // Check config provided to the pipeline + // + UTILS_NFCORE_PIPELINE ( + nextflow_cli_args + ) + // + // Custom validation for pipeline parameters + // + validateInputParameters() + + // + // Create channel from input file provided through params.input + // + if (params.platform == 'illumina') { + Channel + .fromSamplesheet("input") + .map { + meta, fastq_1, fastq_2, barcode-> + if (!fastq_2) { + return [ meta.id, meta + [ single_end:true ], [ fastq_1 ] ] + } else { + return [ meta.id, meta + [ single_end:false ], [ fastq_1, fastq_2 ] ] + } + } + .groupTuple() + .map { + validateInputSamplesheet(it) + } + .map { + meta, fastqs -> + return [ meta, fastqs.flatten() ] + } + .set { ch_samplesheet } + } else { + ch_samplesheet = Channel.empty() + if (input){ + Channel + .fromSamplesheet("input") + .map { + meta, fastq_1, fastq_2, barcode-> + tuple( "barcode"+ String.format('%02d', barcode).toString(), meta.id) + } + .set { ch_samplesheet } + } + } + + emit: + samplesheet = ch_samplesheet + versions = ch_versions +} + +/* +======================================================================================== + SUBWORKFLOW FOR PIPELINE COMPLETION +======================================================================================== +*/ + +workflow PIPELINE_COMPLETION { + + take: + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + hook_url // string: hook URL for notifications + multiqc_report // string: Path to MultiQC report + + main: + + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + + // + // Completion email and summary + // + workflow.onComplete { + if (email || email_on_fail) { + completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, multiqc_report.toList()) + } + + completionSummary(monochrome_logs) + + if (hook_url) { + imNotification(summary_params, hook_url) + } + } +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ +// +// Check and validate pipeline parameters +// +def validateInputParameters() { + genomeExistsError() +} + +// +// Validate channels from input samplesheet +// +def validateInputSamplesheet(input) { + def (metas, fastqs) = input[1..2] + + // Check that multiple runs of the same sample are of the same datatype i.e. single-end / paired-end + def endedness_ok = metas.collect{ it.single_end }.unique().size == 1 + if (!endedness_ok) { + error("Please check input samplesheet -> Multiple runs of a sample must be of the same datatype i.e. single-end or paired-end: ${metas[0].id}") + } + + return [ metas[0], fastqs ] +} +// +// Get attribute from genome config file e.g. fasta +// +def getGenomeAttribute(attribute, primer_set='', primer_set_version=0) { + def val = '' + def support_link = " The default genome config used by the pipeline can be found here:\n" + + " - https://github.com/nf-core/configs/blob/master/conf/pipeline/viralrecon/genomes.config\n\n" + + " If you would still like to blame us please come and find us on nf-core Slack:\n" + + " - https://nf-co.re/viralrecon#contributions-and-support\n" + + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + + if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { + def genome_map = params.genomes[ params.genome ] + if (primer_set) { + if (genome_map.containsKey('primer_sets')) { + genome_map = genome_map[ 'primer_sets' ] + if (genome_map.containsKey(primer_set)) { + genome_map = genome_map[ primer_set ] + primer_set_version = primer_set_version.toString() + if (genome_map.containsKey(primer_set_version)) { + genome_map = genome_map[ primer_set_version ] + } else { + Nextflow.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " --primer_set_version '${primer_set_version}' not found!\n\n" + + " Currently, the available primer set version keys are: ${genome_map.keySet().join(", ")}\n\n" + + " Please check:\n" + + " - The value provided to --primer_set_version (currently '${primer_set_version}')\n" + + " - The value provided to --primer_set (currently '${primer_set}')\n" + + " - The value provided to --genome (currently '${params.genome}')\n" + + " - Any custom config files provided to the pipeline.\n\n" + support_link) + } + } else { + Nextflow.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " --primer_set '${primer_set}' not found!\n\n" + + " Currently, the available primer set keys are: ${genome_map.keySet().join(", ")}\n\n" + + " Please check:\n" + + " - The value provided to --primer_set (currently '${primer_set}')\n" + + " - The value provided to --genome (currently '${params.genome}')\n" + + " - Any custom config files provided to the pipeline.\n\n" + support_link) + } + } else { + Nextflow.error("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " Genome '${params.genome}' does not contain any primer sets!\n\n" + + " Please check:\n" + + " - The value provided to --genome (currently '${params.genome}')\n" + + " - Any custom config files provided to the pipeline.\n\n" + support_link) + } + } + if (genome_map.containsKey(attribute)) { + val = genome_map[ attribute ] + } else if (params.genomes[ params.genome ].containsKey(attribute)) { + val = params.genomes[ params.genome ][ attribute ] + } + } + return val +} + +// +// Exit pipeline if incorrect --genome key provided +// +def genomeExistsError() { + if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + + " Currently, the available genome keys are:\n" + + " ${params.genomes.keySet().join(", ")}\n" + + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + error(error_string) + } +} + +// +// Generate methods description for MultiQC +// +def toolCitationText() { + // TODO nf-core: Optionally add in-text citation tools to this list. + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "Tool (Foo et al. 2023)" : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def citation_text = [ + "Tools used in the workflow included:", + "FastQC (Andrews 2010),", + "MultiQC (Ewels et al. 2016)", + "." + ].join(' ').trim() + + return citation_text +} + +def toolBibliographyText() { + // TODO nf-core: Optionally add bibliographic entries to this list. + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "
  • Author (2023) Pub name, Journal, DOI
    • " : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def reference_text = [ + "
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ", + "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • " + ].join(' ').trim() + + return reference_text +} + +def methodsDescriptionText(mqc_methods_yaml) { + // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file + def meta = [:] + meta.workflow = workflow.toMap() + meta["manifest_map"] = workflow.manifest.toMap() + + // Pipeline DOI + meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" + meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + + // Tool references + meta["tool_citations"] = "" + meta["tool_bibliography"] = "" + + // TODO nf-core: Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled! + // meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") + // meta["tool_bibliography"] = toolBibliographyText() + + + def methods_text = mqc_methods_yaml.text + + def engine = new groovy.text.SimpleTemplateEngine() + def description_html = engine.createTemplate(methods_text).make(meta) + + return description_html.toString() +} diff --git a/subworkflows/nf-core/bam_markduplicates_picard/main.nf b/subworkflows/nf-core/bam_markduplicates_picard/main.nf index de8130fb..2de059b8 100644 --- a/subworkflows/nf-core/bam_markduplicates_picard/main.nf +++ b/subworkflows/nf-core/bam_markduplicates_picard/main.nf @@ -9,7 +9,7 @@ include { BAM_STATS_SAMTOOLS } from '../bam_stats_samtools/main' workflow BAM_MARKDUPLICATES_PICARD { take: - ch_bam // channel: [ val(meta), path(bam) ] + ch_reads // channel: [ val(meta), path(reads) ] ch_fasta // channel: [ path(fasta) ] ch_fai // channel: [ path(fai) ] @@ -17,27 +17,33 @@ workflow BAM_MARKDUPLICATES_PICARD { ch_versions = Channel.empty() - PICARD_MARKDUPLICATES ( ch_bam, ch_fasta, ch_fai ) + PICARD_MARKDUPLICATES ( ch_reads, ch_fasta, ch_fai ) ch_versions = ch_versions.mix(PICARD_MARKDUPLICATES.out.versions.first()) - SAMTOOLS_INDEX ( PICARD_MARKDUPLICATES.out.bam ) + ch_markdup = PICARD_MARKDUPLICATES.out.bam.mix(PICARD_MARKDUPLICATES.out.cram) + + SAMTOOLS_INDEX ( ch_markdup ) ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) - ch_bam_bai = PICARD_MARKDUPLICATES.out.bam - .join(SAMTOOLS_INDEX.out.bai, by: [0], remainder: true) - .join(SAMTOOLS_INDEX.out.csi, by: [0], remainder: true) - .map{meta, bam, bai, csi -> - if (bai) [ meta, bam, bai ] - else [ meta, bam, csi ] + ch_reads_index = ch_markdup + .join(SAMTOOLS_INDEX.out.bai, by: [0], remainder: true) + .join(SAMTOOLS_INDEX.out.crai, by: [0], remainder: true) + .join(SAMTOOLS_INDEX.out.csi, by: [0], remainder: true) + .map{meta, reads, bai, crai, csi -> + if (bai) [ meta, reads, bai ] + else if (crai) [ meta, reads, crai ] + else [ meta, reads, csi ] } - BAM_STATS_SAMTOOLS ( ch_bam_bai, ch_fasta ) + BAM_STATS_SAMTOOLS ( ch_reads_index, ch_fasta ) ch_versions = ch_versions.mix(BAM_STATS_SAMTOOLS.out.versions) emit: bam = PICARD_MARKDUPLICATES.out.bam // channel: [ val(meta), path(bam) ] - metrics = PICARD_MARKDUPLICATES.out.metrics // channel: [ val(meta), path(bam) ] + cram = PICARD_MARKDUPLICATES.out.cram // channel: [ val(meta), path(cram) ] + metrics = PICARD_MARKDUPLICATES.out.metrics // channel: [ val(meta), path(metrics) ] bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), path(bai) ] + crai = SAMTOOLS_INDEX.out.crai // channel: [ val(meta), path(crai) ] csi = SAMTOOLS_INDEX.out.csi // channel: [ val(meta), path(csi) ] stats = BAM_STATS_SAMTOOLS.out.stats // channel: [ val(meta), path(stats) ] diff --git a/subworkflows/nf-core/bam_markduplicates_picard/meta.yml b/subworkflows/nf-core/bam_markduplicates_picard/meta.yml index fe63068e..433d35b2 100644 --- a/subworkflows/nf-core/bam_markduplicates_picard/meta.yml +++ b/subworkflows/nf-core/bam_markduplicates_picard/meta.yml @@ -14,13 +14,13 @@ components: - samtools/flagstat - bam_stats_samtools input: - - ch_bam: + - ch_reads: description: | - BAM/CRAM/SAM file - Structure: [ val(meta), path(bam) ] + Sequence reads in BAM/CRAM/SAM format + Structure: [ val(meta), path(reads) ] - ch_fasta: description: | - Reference genome fasta file + Reference genome fasta file required for CRAM input Structure: [ path(fasta) ] - ch_fasta: description: | @@ -29,12 +29,20 @@ input: output: - bam: description: | - processed BAM/CRAM/SAM file + processed BAM/SAM file Structure: [ val(meta), path(bam) ] - bai: description: | - BAM/CRAM/SAM samtools index + BAM/SAM samtools index Structure: [ val(meta), path(bai) ] + - cram: + description: | + processed CRAM file + Structure: [ val(meta), path(cram) ] + - crai: + description: | + CRAM samtools index + Structure: [ val(meta), path(crai) ] - csi: description: | CSI samtools index diff --git a/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test b/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test index d8d24290..5ef337dc 100644 --- a/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test +++ b/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test @@ -79,8 +79,8 @@ nextflow_workflow { assertAll( { assert workflow.success}, { assert snapshot( - path(workflow.out.bam[0][1]), - path(workflow.out.bai[0][1]), + file(workflow.out.cram[0][1]).name, + path(workflow.out.crai[0][1]), path(workflow.out.flagstat[0][1]), path(workflow.out.idxstats[0][1]), path(workflow.out.stats[0][1]), diff --git a/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test.snap b/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test.snap index a208d101..caf4ac8a 100644 --- a/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test.snap +++ b/subworkflows/nf-core/bam_markduplicates_picard/tests/main.nf.test.snap @@ -1,13 +1,17 @@ { "homo_sapiens - cram": { "content": [ - "test.bam:md5,6641dc05efa8384a061f378d86d922cd", - "test.bam.bai:md5,c41c60d8a94adebe53b6df80b6e90d38", + "test.cram", + "test.cram.crai:md5,78d47ba01ac4e05f3ae1e353902a989e", "test.flagstat:md5,93b0ef463df947ede1f42ff60396c34d", "test.idxstats:md5,e179601fa7b8ebce81ac3765206f6c15", - "test.stats:md5,0035ac8900d85e9a790f4c1f48b76947" + "test.stats:md5,c2f74a4d9b2377bcf4f4f184da3801af" ], - "timestamp": "2023-12-05T17:45:12.484869" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-20T20:45:38.364189" }, "sarscov2 - bam": { "content": [ @@ -15,8 +19,12 @@ "test.bam.bai:md5,4d3ae8d013444b55e17aa0149a2ab404", "test.flagstat:md5,4f7ffd1e6a5e85524d443209ac97d783", "test.idxstats:md5,df60a8c8d6621100d05178c93fb053a2", - "test.stats:md5,e32e7e49dce1fbe327a89e0fb7bc01b1" + "test.stats:md5,d7796222a087b9bb97f631f1c21b9c95" ], - "timestamp": "2023-12-05T17:43:58.582652" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-21T11:38:08.434529" } -} +} \ No newline at end of file diff --git a/subworkflows/nf-core/bam_sort_stats_samtools/main.nf b/subworkflows/nf-core/bam_sort_stats_samtools/main.nf index fc1c652b..b716375b 100644 --- a/subworkflows/nf-core/bam_sort_stats_samtools/main.nf +++ b/subworkflows/nf-core/bam_sort_stats_samtools/main.nf @@ -15,7 +15,7 @@ workflow BAM_SORT_STATS_SAMTOOLS { ch_versions = Channel.empty() - SAMTOOLS_SORT ( ch_bam ) + SAMTOOLS_SORT ( ch_bam, ch_fasta ) ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions.first()) SAMTOOLS_INDEX ( SAMTOOLS_SORT.out.bam ) diff --git a/subworkflows/nf-core/bam_sort_stats_samtools/tests/main.nf.test.snap b/subworkflows/nf-core/bam_sort_stats_samtools/tests/main.nf.test.snap index c159eef3..6645a092 100644 --- a/subworkflows/nf-core/bam_sort_stats_samtools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/bam_sort_stats_samtools/tests/main.nf.test.snap @@ -11,6 +11,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2023-10-22T20:25:03.687121177" }, "test_bam_sort_stats_samtools_paired_end_idxstats": { @@ -25,6 +29,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2023-10-22T20:25:03.709648916" }, "test_bam_sort_stats_samtools_single_end_stats": { @@ -35,11 +43,15 @@ "id": "test", "single_end": false }, - "test.stats:md5,f281507081517414eb1a04b2d9c855b2" + "test.stats:md5,cb0bf2b79de52fdf0c61e80efcdb0bb4" ] ] ], - "timestamp": "2024-01-18T17:10:02.818694" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:44:38.553256801" }, "test_bam_sort_stats_samtools_paired_end_stats": { "content": [ @@ -49,11 +61,15 @@ "id": "test", "single_end": false }, - "test.stats:md5,e32e7e49dce1fbe327a89e0fb7bc01b1" + "test.stats:md5,d7796222a087b9bb97f631f1c21b9c95" ] ] ], - "timestamp": "2023-12-04T11:06:59.253905951" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:44:48.355870518" }, "test_bam_sort_stats_samtools_single_end_idxstats": { "content": [ @@ -67,6 +83,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2024-01-18T17:10:02.84631" }, "test_bam_sort_stats_samtools_single_end_flagstats": { @@ -81,6 +101,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2024-01-18T17:10:02.829756" } } \ No newline at end of file diff --git a/subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test.snap b/subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test.snap index 8bf0d379..bf0b0c69 100644 --- a/subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test.snap @@ -11,6 +11,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2023-11-06T09:31:26.194017574" }, "test_bam_stats_samtools_paired_end_stats": { @@ -21,11 +25,15 @@ "id": "test", "single_end": true }, - "test.stats:md5,49e2b43344ff92bc4c02463a58f7ba4a" + "test.stats:md5,ddaf8f33fe9c1ebe9b06933213aec8ed" ] ] ], - "timestamp": "2024-01-18T17:17:27.704335" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:45:06.230091746" }, "test_bam_stats_samtools_paired_end_flagstats": { "content": [ @@ -39,6 +47,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2024-01-18T17:17:27.717482" }, "test_bam_stats_samtools_single_end_flagstats": { @@ -53,6 +65,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2023-11-06T09:26:10.340046381" }, "test_bam_stats_samtools_paired_end_cram_idxstats": { @@ -67,6 +83,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2023-11-06T09:31:26.207052003" }, "test_bam_stats_samtools_single_end_stats": { @@ -77,11 +97,15 @@ "id": "test", "single_end": true }, - "test.stats:md5,5a6667d97806e5002731e9cf23674fad" + "test.stats:md5,dc178e1a4956043aba8abc83e203521b" ] ] ], - "timestamp": "2023-12-04T11:07:06.676820877" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:44:57.442208382" }, "test_bam_stats_samtools_paired_end_idxstats": { "content": [ @@ -95,6 +119,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2024-01-18T17:17:27.726719" }, "test_bam_stats_samtools_single_end_idxstats": { @@ -109,6 +137,10 @@ ] ] ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, "timestamp": "2023-11-06T09:26:10.349439801" }, "test_bam_stats_samtools_paired_end_cram_stats": { @@ -119,10 +151,14 @@ "id": "test", "single_end": false }, - "test.stats:md5,2cf2fe93596ee3d74f946097b204a629" + "test.stats:md5,d3345c4887f4a9ea4f7f56405b495db0" ] ] ], - "timestamp": "2023-12-04T11:07:22.30295557" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-13T16:45:14.997164209" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_bowtie2/main.nf b/subworkflows/nf-core/fastq_align_bowtie2/main.nf index ba4420f7..cafaa9bf 100644 --- a/subworkflows/nf-core/fastq_align_bowtie2/main.nf +++ b/subworkflows/nf-core/fastq_align_bowtie2/main.nf @@ -20,17 +20,17 @@ workflow FASTQ_ALIGN_BOWTIE2 { // // Map reads with Bowtie2 // - BOWTIE2_ALIGN ( ch_reads, ch_index, save_unaligned, sort_bam ) + BOWTIE2_ALIGN ( ch_reads, ch_index, ch_fasta, save_unaligned, sort_bam ) ch_versions = ch_versions.mix(BOWTIE2_ALIGN.out.versions) // // Sort, index BAM file and run samtools stats, flagstat and idxstats // - BAM_SORT_STATS_SAMTOOLS ( BOWTIE2_ALIGN.out.aligned, ch_fasta ) + BAM_SORT_STATS_SAMTOOLS ( BOWTIE2_ALIGN.out.bam, ch_fasta ) ch_versions = ch_versions.mix(BAM_SORT_STATS_SAMTOOLS.out.versions) emit: - bam_orig = BOWTIE2_ALIGN.out.aligned // channel: [ val(meta), aligned ] + bam_orig = BOWTIE2_ALIGN.out.bam // channel: [ val(meta), aligned ] log_out = BOWTIE2_ALIGN.out.log // channel: [ val(meta), log ] fastq = BOWTIE2_ALIGN.out.fastq // channel: [ val(meta), fastq ] diff --git a/subworkflows/nf-core/fastq_align_bowtie2/meta.yml b/subworkflows/nf-core/fastq_align_bowtie2/meta.yml index 58023a89..b18e4054 100644 --- a/subworkflows/nf-core/fastq_align_bowtie2/meta.yml +++ b/subworkflows/nf-core/fastq_align_bowtie2/meta.yml @@ -37,7 +37,7 @@ input: - sort_bam: type: boolean description: | - Save reads that do not map to the reference (true) or discard them (false) + Use samtools sort (true) or samtools view (false) default: false - ch_fasta: type: file diff --git a/subworkflows/nf-core/fastq_align_bowtie2/tests/main.nf.test b/subworkflows/nf-core/fastq_align_bowtie2/tests/main.nf.test new file mode 100644 index 00000000..b5e84f51 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_bowtie2/tests/main.nf.test @@ -0,0 +1,99 @@ +nextflow_workflow { + + name "Test Subworkflow FASTQ_ALIGN_BOWTIE2" + script "../main.nf" + config "./nextflow.config" + workflow "FASTQ_ALIGN_BOWTIE2" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/fastq_align_bowtie2" + tag "subworkflows/bam_sort_stats_samtools" + tag "bowtie2" + tag "bowtie2/build" + tag "bowtie2/align" + + test("test_align_bowtie2_single_end") { + setup { + run("BOWTIE2_BUILD") { + script "../../../../modules/nf-core/bowtie2/build/main.nf" + process { + """ + input[0] = Channel.value([ [ id:'genome' ],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)]) + """ + } + } + } + when { + workflow { + """ + input[0] = Channel.of([[ id:'test', single_end:true ], [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ]]) + input[1] = BOWTIE2_BUILD.out.index + input[2] = false + input[3] = false + input[4] = Channel.value([ [ id:'genome' ],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)]) + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + file(workflow.out.bam_orig[0][1]).name, + workflow.out.fastq, + workflow.out.log_out, + file(workflow.out.bam[0][1]).name, + file(workflow.out.bai[0][1]).name, + workflow.out.csi, + workflow.out.stats, + workflow.out.flagstat, + workflow.out.idxstats, + workflow.out.versions, + ).match()} + ) + } + } + + test("test_align_bowtie2_paired_end") { + setup { + run("BOWTIE2_BUILD") { + script "../../../../modules/nf-core/bowtie2/build/main.nf" + process { + """ + input[0] = Channel.value([ [ id:'genome' ],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)]) + """ + } + } + } + when { + workflow { + """ + input[0] = Channel.of([[ id:'test', single_end:false ], [file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true)]]) + input[1] = BOWTIE2_BUILD.out.index + input[2] = false + input[3] = false + input[4] = Channel.value([ [ id:'genome' ],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)]) + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + file(workflow.out.bam_orig[0][1]).name, + workflow.out.fastq, + workflow.out.log_out, + file(workflow.out.bam[0][1]).name, + file(workflow.out.bai[0][1]).name, + workflow.out.csi, + workflow.out.stats, + workflow.out.flagstat, + workflow.out.idxstats, + workflow.out.versions, + ).match()} + ) + } + } +} diff --git a/subworkflows/nf-core/fastq_align_bowtie2/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_bowtie2/tests/main.nf.test.snap new file mode 100644 index 00000000..c0f3f8bf --- /dev/null +++ b/subworkflows/nf-core/fastq_align_bowtie2/tests/main.nf.test.snap @@ -0,0 +1,126 @@ +{ + "test_align_bowtie2_single_end": { + "content": [ + "test.bam", + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.bowtie2.log:md5,7b8a9e61b7646da1089b041333c41a87" + ] + ], + "test.sorted.bam", + "test.sorted.bam.bai", + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam.stats:md5,9a65272e49581873b1ea211f738e992f" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam.flagstat:md5,e9ce9093133116bc54fd335cfe698372" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted.bam.idxstats:md5,e16eb632f7f462514b0873c7ac8ac905" + ] + ], + [ + "versions.yml:md5,5d5ab1d650a93d8bb5ed142943798a6a", + "versions.yml:md5,666dbae2343fc479e483656c35d3d8a1", + "versions.yml:md5,aab337e63eac9055aadb9a35cec16053", + "versions.yml:md5,c27f74d9c37fbb3365c437a9f7e81c27", + "versions.yml:md5,eb9364a9f1745d6a345b8b4b03aebe25", + "versions.yml:md5,f982efa9031f340ace29f76dd47a8ce1" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-18T14:40:54.318808117" + }, + "test_align_bowtie2_paired_end": { + "content": [ + "test.bam", + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.bowtie2.log:md5,bd89ce1b28c93bf822bae391ffcedd19" + ] + ], + "test.sorted.bam", + "test.sorted.bam.bai", + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.stats:md5,1086d408391af2a5c80c6dee0efa7e59" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.flagstat:md5,49f3d51a8804ce58fe9cecd2549d279b" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.idxstats:md5,29ff2fa56d35b2a47625b8f517f1a947" + ] + ], + [ + "versions.yml:md5,5d5ab1d650a93d8bb5ed142943798a6a", + "versions.yml:md5,666dbae2343fc479e483656c35d3d8a1", + "versions.yml:md5,aab337e63eac9055aadb9a35cec16053", + "versions.yml:md5,c27f74d9c37fbb3365c437a9f7e81c27", + "versions.yml:md5,eb9364a9f1745d6a345b8b4b03aebe25", + "versions.yml:md5,f982efa9031f340ace29f76dd47a8ce1" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-18T14:41:11.243874685" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_bowtie2/tests/nextflow.config b/subworkflows/nf-core/fastq_align_bowtie2/tests/nextflow.config new file mode 100644 index 00000000..2f85e807 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_bowtie2/tests/nextflow.config @@ -0,0 +1,8 @@ +process { + withName: '.*:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_.*' { + ext.prefix = { "${meta.id}.sorted" } + } + withName: '.*:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' { + ext.prefix = { "${meta.id}.sorted.bam" } + } +} diff --git a/subworkflows/nf-core/fastq_align_bowtie2/tests/tags.yml b/subworkflows/nf-core/fastq_align_bowtie2/tests/tags.yml new file mode 100644 index 00000000..267bcc77 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_bowtie2/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/fastq_align_bowtie2: + - subworkflows/nf-core/fastq_align_bowtie2/** diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf new file mode 100644 index 00000000..ac31f28f --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -0,0 +1,126 @@ +// +// Subworkflow with functionality that may be useful for any Nextflow pipeline +// + +import org.yaml.snakeyaml.Yaml +import groovy.json.JsonOutput +import nextflow.extension.FilesEx + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NEXTFLOW_PIPELINE { + + take: + print_version // boolean: print version + dump_parameters // boolean: dump parameters + outdir // path: base directory used to publish pipeline results + check_conda_channels // boolean: check conda channels + + main: + + // + // Print workflow version and exit on --version + // + if (print_version) { + log.info "${workflow.manifest.name} ${getWorkflowVersion()}" + System.exit(0) + } + + // + // Dump pipeline parameters to a JSON file + // + if (dump_parameters && outdir) { + dumpParametersToJSON(outdir) + } + + // + // When running with Conda, warn if channels have not been set-up appropriately + // + if (check_conda_channels) { + checkCondaChannels() + } + + emit: + dummy_emit = true +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Generate version string +// +def getWorkflowVersion() { + String version_string = "" + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Dump pipeline parameters to a JSON file +// +def dumpParametersToJSON(outdir) { + def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") + def jsonStr = JsonOutput.toJson(params) + temp_pf.text = JsonOutput.prettyPrint(jsonStr) + + FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() +} + +// +// When running with -profile conda, warn if channels have not been set-up appropriately +// +def checkCondaChannels() { + Yaml parser = new Yaml() + def channels = [] + try { + def config = parser.load("conda config --show channels".execute().text) + channels = config.channels + } catch(NullPointerException | IOException e) { + log.warn "Could not verify conda channel configuration." + return + } + + // Check that all channels are present + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean + + // Check that they are in the right order + def channel_priority_violation = false + def n = required_channels_in_order.size() + for (int i = 0; i < n - 1; i++) { + channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + } + + if (channels_missing | channel_priority_violation) { + log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " There is a problem with your Conda configuration!\n\n" + + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + + " Please refer to https://bioconda.github.io/\n" + + " The observed channel order is \n" + + " ${channels}\n" + + " but the following channel order is required:\n" + + " ${required_channels_in_order}\n" + + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml new file mode 100644 index 00000000..e5c3a0a8 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml @@ -0,0 +1,38 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NEXTFLOW_PIPELINE" +description: Subworkflow with functionality that may be useful for any Nextflow pipeline +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - print_version: + type: boolean + description: | + Print the version of the pipeline and exit + - dump_parameters: + type: boolean + description: | + Dump the parameters of the pipeline to a JSON file + - output_directory: + type: directory + description: Path to output dir to write JSON file to. + pattern: "results/" + - check_conda_channel: + type: boolean + description: | + Check if the conda channel priority is correct. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" + - "@drpatelh" +maintainers: + - "@adamrtalbot" + - "@drpatelh" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test new file mode 100644 index 00000000..68718e4f --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -0,0 +1,54 @@ + +nextflow_function { + + name "Test Functions" + script "subworkflows/nf-core/utils_nextflow_pipeline/main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Test Function getWorkflowVersion") { + + function "getWorkflowVersion" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dumpParametersToJSON") { + + function "dumpParametersToJSON" + + when { + function { + """ + // define inputs of the function here. Example: + input[0] = "$outputDir" + """.stripIndent() + } + } + + then { + assertAll( + { assert function.success } + ) + } + } + + test("Test Function checkCondaChannels") { + + function "checkCondaChannels" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 00000000..e3f0baf4 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,20 @@ +{ + "Test Function getWorkflowVersion": { + "content": [ + "v9.9.9" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:05.308243" + }, + "Test Function checkCondaChannels": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:12.425833" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test new file mode 100644 index 00000000..ca964ce8 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,111 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NEXTFLOW_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + workflow "UTILS_NEXTFLOW_PIPELINE" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Should run no inputs") { + + when { + workflow { + """ + print_version = false + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should print version") { + + when { + workflow { + """ + print_version = true + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.stdout.contains("nextflow_workflow v9.9.9") } + ) + } + } + + test("Should dump params") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = 'results' + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should not create params JSON if no output directory") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = null + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config new file mode 100644 index 00000000..d0a926bf --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml new file mode 100644 index 00000000..f8476112 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nextflow_pipeline: + - subworkflows/nf-core/utils_nextflow_pipeline/** diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf new file mode 100644 index 00000000..a8b55d6f --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -0,0 +1,440 @@ +// +// Subworkflow with utility functions specific to the nf-core pipeline template +// + +import org.yaml.snakeyaml.Yaml +import nextflow.extension.FilesEx + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NFCORE_PIPELINE { + + take: + nextflow_cli_args + + main: + valid_config = checkConfigProvided() + checkProfileProvided(nextflow_cli_args) + + emit: + valid_config +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Warn if a -profile or Nextflow config has not been provided to run the pipeline +// +def checkConfigProvided() { + valid_config = true + if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { + log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + + "Please refer to the quick start section and usage docs for the pipeline.\n " + valid_config = false + } + return valid_config +} + +// +// Exit pipeline if --profile contains spaces +// +def checkProfileProvided(nextflow_cli_args) { + if (workflow.profile.endsWith(',')) { + error "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + } + if (nextflow_cli_args[0]) { + log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + } +} + +// +// Citation string for pipeline +// +def workflowCitation() { + return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + + "* The pipeline\n" + + " ${workflow.manifest.doi}\n\n" + + "* The nf-core framework\n" + + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + + "* Software dependencies\n" + + " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" +} + +// +// Generate workflow version string +// +def getWorkflowVersion() { + String version_string = "" + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Get software versions for pipeline +// +def processVersionsFromYAML(yaml_file) { + Yaml yaml = new Yaml() + versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + return yaml.dumpAsMap(versions).trim() +} + +// +// Get workflow version for pipeline +// +def workflowVersionToYAML() { + return """ + Workflow: + $workflow.manifest.name: ${getWorkflowVersion()} + Nextflow: $workflow.nextflow.version + """.stripIndent().trim() +} + +// +// Get channel of software versions used in pipeline in YAML format +// +def softwareVersionsToYAML(ch_versions) { + return ch_versions + .unique() + .map { processVersionsFromYAML(it) } + .unique() + .mix(Channel.of(workflowVersionToYAML())) +} + +// +// Get workflow summary for MultiQC +// +def paramsSummaryMultiqc(summary_params) { + def summary_section = '' + for (group in summary_params.keySet()) { + def group_params = summary_params.get(group) // This gets the parameters of that particular group + if (group_params) { + summary_section += "

    $group

    \n" + summary_section += "
    \n" + for (param in group_params.keySet()) { + summary_section += "
    $param
    ${group_params.get(param) ?: 'N/A'}
    \n" + } + summary_section += "
    \n" + } + } + + String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + + return yaml_file_text +} + +// +// nf-core logo +// +def nfCoreLogo(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + String.format( + """\n + ${dashedLine(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} + ${dashedLine(monochrome_logs)} + """.stripIndent() + ) +} + +// +// Return dashed line +// +def dashedLine(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" +} + +// +// ANSII colours used for terminal logging +// +def logColours(monochrome_logs=true) { + Map colorcodes = [:] + + // Reset / Meta + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" + + // Regular Colors + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + + // Bold + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + + // Underline + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + + // High Intensity + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + + // Bold High Intensity + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + + return colorcodes +} + +// +// Attach the multiqc report to email +// +def attachMultiqcReport(multiqc_report) { + def mqc_report = null + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + } + mqc_report = mqc_report[0] + } + } + } catch (all) { + if (multiqc_report) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } + } + return mqc_report +} + +// +// Construct and send completion email +// +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { + + // Set up the e-mail variables + def subject = "[$workflow.manifest.name] Successful: $workflow.runName" + if (!workflow.success) { + subject = "[$workflow.manifest.name] FAILED: $workflow.runName" + } + + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['Date Started'] = workflow.start + misc_fields['Date Completed'] = workflow.complete + misc_fields['Pipeline script file path'] = workflow.scriptFile + misc_fields['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository + if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId + if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build + misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + def email_fields = [:] + email_fields['version'] = getWorkflowVersion() + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary << misc_fields + + // On success try attach the multiqc report + def mqc_report = attachMultiqcReport(multiqc_report) + + // Check if we are only sending emails on failure + def email_address = email + if (!email && email_on_fail && !workflow.success) { + email_address = email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("${workflow.projectDir}/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("${workflow.projectDir}/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] + def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + Map colors = logColours(monochrome_logs) + if (email_address) { + try { + if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } + [ 'sendmail', '-t' ].execute() << sendmail_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" + } catch (all) { + // Catch failures and try with plaintext + def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] + mail_cmd.execute() << email_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" + } + } + + // Write summary e-mail HTML to a file + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") + output_hf.withWriter { w -> w << email_html } + FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") + output_tf.withWriter { w -> w << email_txt } + FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + output_tf.delete() +} + +// +// Print pipeline summary on completion +// +def completionSummary(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + if (workflow.success) { + if (workflow.stats.ignoredCount == 0) { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" + } else { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" + } + } else { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" + } +} + +// +// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack +// +def imNotification(summary_params, hook_url) { + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) misc_fields['repository'] = workflow.repository + if (workflow.commitId) misc_fields['commitid'] = workflow.commitId + if (workflow.revision) misc_fields['revision'] = workflow.revision + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = getWorkflowVersion() + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + // Different JSON depending on the service provider + // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format + def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" + def hf = new File("${workflow.projectDir}/assets/${json_path}") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection(); + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")); + def postRC = post.getResponseCode(); + if (! postRC.equals(200)) { + log.warn(post.getErrorStream().getText()); + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml new file mode 100644 index 00000000..d08d2434 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFCORE_PIPELINE" +description: Subworkflow with utility functions specific to the nf-core pipeline template +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - nextflow_cli_args: + type: list + description: | + Nextflow CLI positional arguments +output: + - success: + type: boolean + description: | + Dummy output to indicate success +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test new file mode 100644 index 00000000..1dc317f8 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -0,0 +1,134 @@ + +nextflow_function { + + name "Test Functions" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Test Function checkConfigProvided") { + + function "checkConfigProvided" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function checkProfileProvided") { + + function "checkProfileProvided" + + when { + function { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function workflowCitation") { + + function "workflowCitation" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function nfCoreLogo") { + + function "nfCoreLogo" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dashedLine") { + + function "dashedLine" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function without logColours") { + + function "logColours" + + when { + function { + """ + input[0] = true + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function with logColours") { + function "logColours" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 00000000..1037232c --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,166 @@ +{ + "Test Function checkProfileProvided": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:03.360873" + }, + "Test Function checkConfigProvided": { + "content": [ + true + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:59.729647" + }, + "Test Function nfCoreLogo": { + "content": [ + "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:10.562934" + }, + "Test Function workflowCitation": { + "content": [ + "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:07.019761" + }, + "Test Function without logColours": { + "content": [ + { + "reset": "", + "bold": "", + "dim": "", + "underlined": "", + "blink": "", + "reverse": "", + "hidden": "", + "black": "", + "red": "", + "green": "", + "yellow": "", + "blue": "", + "purple": "", + "cyan": "", + "white": "", + "bblack": "", + "bred": "", + "bgreen": "", + "byellow": "", + "bblue": "", + "bpurple": "", + "bcyan": "", + "bwhite": "", + "ublack": "", + "ured": "", + "ugreen": "", + "uyellow": "", + "ublue": "", + "upurple": "", + "ucyan": "", + "uwhite": "", + "iblack": "", + "ired": "", + "igreen": "", + "iyellow": "", + "iblue": "", + "ipurple": "", + "icyan": "", + "iwhite": "", + "biblack": "", + "bired": "", + "bigreen": "", + "biyellow": "", + "biblue": "", + "bipurple": "", + "bicyan": "", + "biwhite": "" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:17.969323" + }, + "Test Function dashedLine": { + "content": [ + "-\u001b[2m----------------------------------------------------\u001b[0m-" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:14.366181" + }, + "Test Function with logColours": { + "content": [ + { + "reset": "\u001b[0m", + "bold": "\u001b[1m", + "dim": "\u001b[2m", + "underlined": "\u001b[4m", + "blink": "\u001b[5m", + "reverse": "\u001b[7m", + "hidden": "\u001b[8m", + "black": "\u001b[0;30m", + "red": "\u001b[0;31m", + "green": "\u001b[0;32m", + "yellow": "\u001b[0;33m", + "blue": "\u001b[0;34m", + "purple": "\u001b[0;35m", + "cyan": "\u001b[0;36m", + "white": "\u001b[0;37m", + "bblack": "\u001b[1;30m", + "bred": "\u001b[1;31m", + "bgreen": "\u001b[1;32m", + "byellow": "\u001b[1;33m", + "bblue": "\u001b[1;34m", + "bpurple": "\u001b[1;35m", + "bcyan": "\u001b[1;36m", + "bwhite": "\u001b[1;37m", + "ublack": "\u001b[4;30m", + "ured": "\u001b[4;31m", + "ugreen": "\u001b[4;32m", + "uyellow": "\u001b[4;33m", + "ublue": "\u001b[4;34m", + "upurple": "\u001b[4;35m", + "ucyan": "\u001b[4;36m", + "uwhite": "\u001b[4;37m", + "iblack": "\u001b[0;90m", + "ired": "\u001b[0;91m", + "igreen": "\u001b[0;92m", + "iyellow": "\u001b[0;93m", + "iblue": "\u001b[0;94m", + "ipurple": "\u001b[0;95m", + "icyan": "\u001b[0;96m", + "iwhite": "\u001b[0;97m", + "biblack": "\u001b[1;90m", + "bired": "\u001b[1;91m", + "bigreen": "\u001b[1;92m", + "biyellow": "\u001b[1;93m", + "biblue": "\u001b[1;94m", + "bipurple": "\u001b[1;95m", + "bicyan": "\u001b[1;96m", + "biwhite": "\u001b[1;97m" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:21.714424" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test new file mode 100644 index 00000000..8940d32d --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap new file mode 100644 index 00000000..859d1030 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config new file mode 100644 index 00000000..d0a926bf --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml new file mode 100644 index 00000000..ac8523c9 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfcore_pipeline: + - subworkflows/nf-core/utils_nfcore_pipeline/** diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf new file mode 100644 index 00000000..2585b65d --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf @@ -0,0 +1,62 @@ +// +// Subworkflow that uses the nf-validation plugin to render help text and parameter summary +// + +/* +======================================================================================== + IMPORT NF-VALIDATION PLUGIN +======================================================================================== +*/ + +include { paramsHelp } from 'plugin/nf-validation' +include { paramsSummaryLog } from 'plugin/nf-validation' +include { validateParameters } from 'plugin/nf-validation' + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NFVALIDATION_PLUGIN { + + take: + print_help // boolean: print help + workflow_command // string: default commmand used to run pipeline + pre_help_text // string: string to be printed before help text and summary log + post_help_text // string: string to be printed after help text and summary log + validate_params // boolean: validate parameters + schema_filename // path: JSON schema file, null to use default value + + main: + + log.debug "Using schema file: ${schema_filename}" + + // Default values for strings + pre_help_text = pre_help_text ?: '' + post_help_text = post_help_text ?: '' + workflow_command = workflow_command ?: '' + + // + // Print help message if needed + // + if (print_help) { + log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text + System.exit(0) + } + + // + // Print parameter summary to stdout + // + log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text + + // + // Validate parameters relative to the parameter JSON schema + // + if (validate_params){ + validateParameters(parameters_schema: schema_filename) + } + + emit: + dummy_emit = true +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml new file mode 100644 index 00000000..3d4a6b04 --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFVALIDATION_PLUGIN" +description: Use nf-validation to initiate and validate a pipeline +keywords: + - utility + - pipeline + - initialise + - validation +components: [] +input: + - print_help: + type: boolean + description: | + Print help message and exit + - workflow_command: + type: string + description: | + The command to run the workflow e.g. "nextflow run main.nf" + - pre_help_text: + type: string + description: | + Text to print before the help message + - post_help_text: + type: string + description: | + Text to print after the help message + - validate_params: + type: boolean + description: | + Validate the parameters and error if invalid. + - schema_filename: + type: string + description: | + The filename of the schema to validate against. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test new file mode 100644 index 00000000..5784a33f --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test @@ -0,0 +1,200 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFVALIDATION_PLUGIN" + script "../main.nf" + workflow "UTILS_NFVALIDATION_PLUGIN" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "plugin/nf-validation" + tag "'plugin/nf-validation'" + tag "utils_nfvalidation_plugin" + tag "subworkflows/utils_nfvalidation_plugin" + + test("Should run nothing") { + + when { + + params { + monochrome_logs = true + test_data = '' + } + + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should run help") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) + } + } + + test("Should run help with command") { + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) + } + } + + test("Should run help with extra text") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = "pre-help-text" + post_help_text = "post-help-text" + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('pre-help-text') } }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } }, + { assert workflow.stdout.any { it.contains('post-help-text') } } + ) + } + } + + test("Should validate params") { + + when { + + params { + monochrome_logs = true + test_data = '' + outdir = 1 + } + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = true + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json new file mode 100644 index 00000000..7626c1c9 --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", + "title": ". pipeline parameters", + "description": "", + "type": "object", + "definitions": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["outdir"], + "properties": { + "validate_params": { + "type": "boolean", + "description": "Validate parameters?", + "default": true, + "hidden": true + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "test_data_base": { + "type": "string", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/modules", + "description": "Base for test data directory", + "hidden": true + }, + "test_data": { + "type": "string", + "description": "Fake test data param", + "hidden": true + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "help": { + "type": "boolean", + "description": "Display help text.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "logo": { + "type": "boolean", + "default": true, + "description": "Display nf-core logo in console output.", + "fa_icon": "fas fa-image", + "hidden": true + }, + "singularity_pull_docker_container": { + "type": "boolean", + "description": "Pull Singularity container from Docker?", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Use monochrome_logs", + "hidden": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/input_output_options" + }, + { + "$ref": "#/definitions/generic_options" + } + ] +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml new file mode 100644 index 00000000..60b1cfff --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfvalidation_plugin: + - subworkflows/nf-core/utils_nfvalidation_plugin/** diff --git a/workflows/illumina.nf b/workflows/illumina.nf index 38abdf56..ff7407a3 100644 --- a/workflows/illumina.nf +++ b/workflows/illumina.nf @@ -5,13 +5,9 @@ */ include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' - -def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) -def citation = '\n' + WorkflowMain.citation(workflow) + '\n' -def summary_params = paramsSummaryMap(workflow) - -// Print parameter summary log to screen -log.info logo + paramsSummaryLog(workflow) + citation +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_viralrecon_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -27,9 +23,6 @@ def valid_params = [ spades_modes : ['rnaviral', 'corona', 'metaviral', 'meta', 'metaplasmid', 'plasmid', 'isolate', 'rna', 'bio'] ] -// Validate input parameters -WorkflowIllumina.initialise(params, log, valid_params) - // Check input path parameters to see if they exist def checkPathParamList = [ params.input, params.fasta, params.gff, params.bowtie2_index, @@ -77,7 +70,6 @@ include { PLOT_MOSDEPTH_REGIONS as PLOT_MOSDEPTH_REGIONS_AMPLICON } from '../mod // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -include { INPUT_CHECK } from '../subworkflows/local/input_check' include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome_illumina' include { VARIANTS_IVAR } from '../subworkflows/local/variants_ivar' include { VARIANTS_BCFTOOLS } from '../subworkflows/local/variants_bcftools' @@ -103,7 +95,6 @@ include { CAT_FASTQ } from '../modules/nf-core/cat/fastq/mai include { FASTQC } from '../modules/nf-core/fastqc/main' include { KRAKEN2_KRAKEN2 } from '../modules/nf-core/kraken2/kraken2/main' include { PICARD_COLLECTMULTIPLEMETRICS } from '../modules/nf-core/picard/collectmultiplemetrics/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' include { MOSDEPTH as MOSDEPTH_GENOME } from '../modules/nf-core/mosdepth/main' include { MOSDEPTH as MOSDEPTH_AMPLICON } from '../modules/nf-core/mosdepth/main' @@ -121,18 +112,40 @@ include { BAM_VARIANT_DEMIX_BOOT_FREYJA } from '../subworkflows/nf-core/bam_vari */ // Info required for completion email and summary -def multiqc_report = [] def pass_mapped_reads = [:] def fail_mapped_reads = [:] workflow ILLUMINA { - ch_versions = Channel.empty() + take: + ch_samplesheet // channel: samplesheet read in from --input + ch_genome_fasta + ch_genome_gff + ch_primer_bed + ch_bowtie2_index + ch_nextclade_dataset + ch_nextclade_dataset_name + ch_nextclade_dataset_reference + ch_nextclade_dataset_tag + + main: + ch_versions = Channel.empty() + ch_multiqc_files = Channel.empty() + multiqc_report = Channel.empty() // // SUBWORKFLOW: Uncompress and prepare reference genome files // - PREPARE_GENOME () + PREPARE_GENOME ( + ch_genome_fasta, + ch_genome_gff, + ch_primer_bed, + ch_bowtie2_index, + ch_nextclade_dataset, + ch_nextclade_dataset_name, + ch_nextclade_dataset_reference, + ch_nextclade_dataset_tag + ) ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) // Check genome fasta only contains a single contig @@ -172,40 +185,15 @@ workflow ILLUMINA { } } - // - // SUBWORKFLOW: Read in samplesheet, validate and stage input files - // - INPUT_CHECK ( - ch_input, - params.platform - ) - .sample_info - .map { - meta, fastq -> - meta.id = meta.id.split('_')[0..-2].join('_') - [ meta, fastq ] - } - .groupTuple(by: [0]) - .branch { - meta, fastq -> - single : fastq.size() == 1 - return [ meta, fastq.flatten() ] - multiple: fastq.size() > 1 - return [ meta, fastq.flatten() ] - } - .set { ch_fastq } - ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) - // // MODULE: Concatenate FastQ files from same sample if required // CAT_FASTQ ( - ch_fastq.multiple + ch_samplesheet ) .reads - .mix(ch_fastq.single) .set { ch_cat_fastq } - ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first()) // // SUBWORKFLOW: Read QC and trim adapters @@ -268,7 +256,7 @@ workflow ILLUMINA { params.kraken2_variants_host_filter || params.kraken2_assembly_host_filter ) ch_kraken2_multiqc = KRAKEN2_KRAKEN2.out.report - ch_versions = ch_versions.mix(KRAKEN2_KRAKEN2.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(KRAKEN2_KRAKEN2.out.versions.first()) if (params.kraken2_variants_host_filter) { ch_variants_fastq = KRAKEN2_KRAKEN2.out.unclassified_reads_fastq @@ -383,7 +371,7 @@ workflow ILLUMINA { PREPARE_GENOME.out.fasta.map { [ [:], it ] }, [ [:], [] ] ) - ch_versions = ch_versions.mix(PICARD_COLLECTMULTIPLEMETRICS.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(PICARD_COLLECTMULTIPLEMETRICS.out.versions.first()) } // @@ -399,7 +387,7 @@ workflow ILLUMINA { [ [:], [] ], ) ch_mosdepth_multiqc = MOSDEPTH_GENOME.out.global_txt - ch_versions = ch_versions.mix(MOSDEPTH_GENOME.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(MOSDEPTH_GENOME.out.versions.first()) PLOT_MOSDEPTH_REGIONS_GENOME ( MOSDEPTH_GENOME.out.regions_bed.collect { it[1] } @@ -413,7 +401,7 @@ workflow ILLUMINA { .join(PREPARE_GENOME.out.primer_collapsed_bed), [ [:], [] ], ) - ch_versions = ch_versions.mix(MOSDEPTH_AMPLICON.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(MOSDEPTH_AMPLICON.out.versions.first()) PLOT_MOSDEPTH_REGIONS_AMPLICON ( MOSDEPTH_AMPLICON.out.regions_bed.collect { it[1] } @@ -573,13 +561,13 @@ workflow ILLUMINA { ) ch_assembly_fastq = CUTADAPT.out.reads ch_cutadapt_multiqc = CUTADAPT.out.log - ch_versions = ch_versions.mix(CUTADAPT.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(CUTADAPT.out.versions.first()) if (!params.skip_fastqc) { FASTQC ( CUTADAPT.out.reads ) - ch_versions = ch_versions.mix(FASTQC.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(FASTQC.out.versions.first()) } } @@ -634,23 +622,30 @@ workflow ILLUMINA { } // - // MODULE: Pipeline reporting + // Collate and save software versions // - CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') - ) + softwareVersionsToYAML(ch_versions) + .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) + .set { ch_collated_versions } // // MODULE: MultiQC // if (!params.skip_multiqc) { - workflow_summary = WorkflowCommons.paramsSummaryMultiqc(workflow, summary_params) - ch_workflow_summary = Channel.value(workflow_summary) + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) + ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) MULTIQC ( + ch_multiqc_files.collect(), ch_multiqc_config, ch_multiqc_custom_config, - CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect(), + ch_multiqc_logo.toList(), ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml'), ch_fail_reads_multiqc.collectFile(name: 'fail_mapped_reads_mqc.tsv').ifEmpty([]), ch_fail_mapping_multiqc.collectFile(name: 'fail_mapped_samples_mqc.tsv').ifEmpty([]), @@ -675,22 +670,14 @@ workflow ILLUMINA { ch_minia_quast_multiqc.collect{it[1]}.ifEmpty([]), ch_freyja_multiqc.collect{it[1]}.ifEmpty([]), ) + multiqc_report = MULTIQC.out.report.toList() } -} -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - COMPLETION EMAIL AND SUMMARY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ + emit: + multiqc_report // channel: /path/to/multiqc_report.html + versions = ch_versions // channel: [ path(versions.yml) ] -workflow.onComplete { - if (params.email || params.email_on_fail) { - NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report, fail_mapped_reads) - } - NfcoreTemplate.dump_parameters(workflow, params) - NfcoreTemplate.summary(workflow, params, log, fail_mapped_reads, pass_mapped_reads) } /* diff --git a/workflows/nanopore.nf b/workflows/nanopore.nf index 87afd075..c613a35c 100644 --- a/workflows/nanopore.nf +++ b/workflows/nanopore.nf @@ -5,13 +5,9 @@ */ include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' - -def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) -def citation = '\n' + WorkflowMain.citation(workflow) + '\n' -def summary_params = paramsSummaryMap(workflow) - -// Print parameter summary log to screen -log.info logo + paramsSummaryLog(workflow) + citation +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_viralrecon_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -24,9 +20,6 @@ def valid_params = [ artic_minion_aligner : ['minimap2', 'bwa'] ] -// Validate input parameters -WorkflowNanopore.initialise(params, log, valid_params) - def checkPathParamList = [ params.input, params.fastq_dir, params.fast5_dir, params.sequencing_summary, params.gff, @@ -34,7 +27,6 @@ def checkPathParamList = [ ] for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } } -if (params.input) { ch_input = file(params.input) } if (params.fast5_dir) { ch_fast5_dir = file(params.fast5_dir) } else { ch_fast5_dir = [] } if (params.sequencing_summary) { ch_sequencing_summary = file(params.sequencing_summary) } else { ch_sequencing_summary = [] } @@ -72,7 +64,6 @@ include { PLOT_MOSDEPTH_REGIONS as PLOT_MOSDEPTH_REGIONS_AMPLICON } from '../mod // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -include { INPUT_CHECK } from '../subworkflows/local/input_check' include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome_nanopore' include { SNPEFF_SNPSIFT } from '../subworkflows/local/snpeff_snpsift' include { VARIANTS_LONG_TABLE } from '../subworkflows/local/variants_long_table' @@ -98,7 +89,6 @@ include { BCFTOOLS_STATS } from '../modules/nf-core/bcftools/stat include { QUAST } from '../modules/nf-core/quast/main' include { PANGOLIN } from '../modules/nf-core/pangolin/main' include { NEXTCLADE_RUN } from '../modules/nf-core/nextclade/run/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' include { MOSDEPTH as MOSDEPTH_GENOME } from '../modules/nf-core/mosdepth/main' include { MOSDEPTH as MOSDEPTH_AMPLICON } from '../modules/nf-core/mosdepth/main' @@ -109,13 +99,27 @@ include { MOSDEPTH as MOSDEPTH_AMPLICON } from '../modules/nf-core/mosdepth/main */ // Info required for completion email and summary -def multiqc_report = [] def pass_barcode_reads = [:] def fail_barcode_reads = [:] workflow NANOPORE { - ch_versions = Channel.empty() + take: + ch_samplesheet // channel: samplesheet read in from --input + ch_genome_fasta + ch_genome_gff + ch_primer_bed + ch_artic_scheme + ch_bowtie2_index + ch_nextclade_dataset + ch_nextclade_dataset_name + ch_nextclade_dataset_reference + ch_nextclade_dataset_tag + + main: + ch_versions = Channel.empty() + ch_multiqc_files = Channel.empty() + multiqc_report = Channel.empty() // // MODULE: PycoQC on sequencing summary file @@ -132,7 +136,16 @@ workflow NANOPORE { // // SUBWORKFLOW: Uncompress and prepare reference genome files // - PREPARE_GENOME () + PREPARE_GENOME ( + ch_genome_fasta, + ch_genome_gff, + ch_primer_bed, + ch_bowtie2_index, + ch_nextclade_dataset, + ch_nextclade_dataset_name, + ch_nextclade_dataset_reference, + ch_nextclade_dataset_tag + ) ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) // Check primer BED file only contains suffixes provided --primer_left_suffix / --primer_right_suffix @@ -179,14 +192,9 @@ workflow NANOPORE { // SUBWORKFLOW: Read in samplesheet containing sample to barcode mappings // if (params.input) { - INPUT_CHECK ( - ch_input, - params.platform - ) - .sample_info + ch_samplesheet .join(ch_fastq_dirs, remainder: true) .set { ch_fastq_dirs } - ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) // // MODULE: Create custom content file for MultiQC to report barcodes were allocated reads >= params.min_barcode_reads but no sample name in samplesheet @@ -273,7 +281,7 @@ workflow NANOPORE { ARTIC_GUPPYPLEX ( ch_fastq_dirs ) - ch_versions = ch_versions.mix(ARTIC_GUPPYPLEX.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(ARTIC_GUPPYPLEX.out.versions.first()) // // MODULE: Create custom content file for MultiQC to report samples with reads < params.min_guppyplex_reads @@ -307,7 +315,7 @@ workflow NANOPORE { NANOPLOT ( ARTIC_GUPPYPLEX.out.fastq ) - ch_versions = ch_versions.mix(NANOPLOT.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(NANOPLOT.out.versions.first()) } // @@ -321,10 +329,10 @@ workflow NANOPORE { PREPARE_GENOME.out.primer_bed.collect(), ch_medaka_model.collect().ifEmpty([]), params.artic_minion_medaka_model ?: '', - params.artic_scheme, + ch_artic_scheme, params.primer_set_version ) - ch_versions = ch_versions.mix(ARTIC_MINION.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(ARTIC_MINION.out.versions.first()) // // MODULE: Remove duplicate variants @@ -332,7 +340,7 @@ workflow NANOPORE { VCFLIB_VCFUNIQ ( ARTIC_MINION.out.vcf.join(ARTIC_MINION.out.tbi, by: [0]), ) - ch_versions = ch_versions.mix(VCFLIB_VCFUNIQ.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(VCFLIB_VCFUNIQ.out.versions.first()) // // MODULE: Index VCF file @@ -340,7 +348,7 @@ workflow NANOPORE { TABIX_TABIX ( VCFLIB_VCFUNIQ.out.vcf ) - ch_versions = ch_versions.mix(TABIX_TABIX.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(TABIX_TABIX.out.versions.first()) // // MODULE: VCF stats with bcftools stats @@ -353,7 +361,7 @@ workflow NANOPORE { [ [:], [] ], [ [:], [] ] ) - ch_versions = ch_versions.mix(BCFTOOLS_STATS.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(BCFTOOLS_STATS.out.versions.first()) // // SUBWORKFLOW: Filter unmapped reads from BAM @@ -378,7 +386,7 @@ workflow NANOPORE { [ [:], [] ] ) ch_mosdepth_multiqc = MOSDEPTH_GENOME.out.global_txt - ch_versions = ch_versions.mix(MOSDEPTH_GENOME.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(MOSDEPTH_GENOME.out.versions.first()) PLOT_MOSDEPTH_REGIONS_GENOME ( MOSDEPTH_GENOME.out.regions_bed.collect { it[1] } @@ -389,7 +397,7 @@ workflow NANOPORE { ARTIC_MINION.out.bam_primertrimmed.join(ARTIC_MINION.out.bai_primertrimmed, by: [0]).join(PREPARE_GENOME.out.primer_collapsed_bed), [ [:], [] ] ) - ch_versions = ch_versions.mix(MOSDEPTH_AMPLICON.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(MOSDEPTH_AMPLICON.out.versions.first()) PLOT_MOSDEPTH_REGIONS_AMPLICON ( MOSDEPTH_AMPLICON.out.regions_bed.collect { it[1] } @@ -407,7 +415,7 @@ workflow NANOPORE { ARTIC_MINION.out.fasta ) ch_pangolin_multiqc = PANGOLIN.out.report - ch_versions = ch_versions.mix(PANGOLIN.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(PANGOLIN.out.versions.first()) } // @@ -419,7 +427,7 @@ workflow NANOPORE { ARTIC_MINION.out.fasta, PREPARE_GENOME.out.nextclade_db.collect() ) - ch_versions = ch_versions.mix(NEXTCLADE_RUN.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(NEXTCLADE_RUN.out.versions.first()) // // MODULE: Get Nextclade clade information for MultiQC report @@ -518,7 +526,7 @@ workflow NANOPORE { params.asciigenome_window_size, params.asciigenome_read_depth ) - ch_versions = ch_versions.mix(ASCIIGENOME.out.versions.first().ifEmpty(null)) + ch_versions = ch_versions.mix(ASCIIGENOME.out.versions.first()) } // @@ -537,21 +545,29 @@ workflow NANOPORE { // // MODULE: Pipeline reporting // - CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') - ) + softwareVersionsToYAML(ch_versions) + .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) + .set { ch_collated_versions } + // // MODULE: MultiQC // if (!params.skip_multiqc) { - workflow_summary = WorkflowCommons.paramsSummaryMultiqc(workflow, summary_params) - ch_workflow_summary = Channel.value(workflow_summary) + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) + ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) MULTIQC ( + ch_multiqc_files.collect(), ch_multiqc_config, ch_multiqc_custom_config, - CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect(), + ch_multiqc_logo.toList(), ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml'), ch_custom_no_sample_name_multiqc.collectFile(name: 'fail_barcodes_no_sample_mqc.tsv').ifEmpty([]), ch_custom_no_barcodes_multiqc.collectFile(name: 'fail_no_barcode_samples_mqc.tsv').ifEmpty([]), @@ -569,22 +585,14 @@ workflow NANOPORE { ch_nextclade_multiqc.collectFile(name: 'nextclade_clade_mqc.tsv').ifEmpty([]), ch_freyja_multiqc.collect{it[1]}.ifEmpty([]), ) + multiqc_report = MULTIQC.out.report.toList() } -} -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - COMPLETION EMAIL AND SUMMARY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ + emit: + multiqc_report // channel: /path/to/multiqc_report.html + versions = ch_versions // channel: [ path(versions.yml) ] -workflow.onComplete { - if (params.email || params.email_on_fail) { - NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) - } - NfcoreTemplate.dump_parameters(workflow, params) - NfcoreTemplate.summary(workflow, params, log) } /*