diff --git a/.github/workflows/code-linting.yml b/.github/workflows/code-linting.yml deleted file mode 100644 index 831453f8ea2..00000000000 --- a/.github/workflows/code-linting.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: code-lint -on: - push: - branches: [master] - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - Prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Install NodeJS - uses: actions/setup-node@v2 - - - name: Install Prettier - run: npm install -g prettier - - - name: Run Prettier --check - run: prettier --check . - - EditorConfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v2 - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) diff --git a/.github/workflows/nf-core-linting.yml b/.github/workflows/nf-core-linting.yml deleted file mode 100644 index 123decf27e6..00000000000 --- a/.github/workflows/nf-core-linting.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: nf-core-lint -run-name: nf-core-lint -# This workflow is triggered on pushes and PRs to the repository. -# It runs the `nf-core lint` tests to ensure that the module code meets the nf-core guidelines -on: - push: - branches: [master] - pull_request: - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - name: check-changes - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - - lint: - runs-on: ubuntu-20.04 - name: lint - needs: changes - if: needs.changes.outputs.modules != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.changes.outputs.modules) }}"] - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - name: Install pip - run: python -m pip install --upgrade pip - - - name: Install nf-core tools - run: python -m pip install nf-core - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Lint ${{ matrix.tags }} - run: nf-core modules lint ${{ matrix.tags }} - # HACK - if: startsWith( matrix.tags, 'subworkflow' ) != true - - - uses: actions/cache@v3 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }} diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml deleted file mode 100644 index 57df4fef4c2..00000000000 --- a/.github/workflows/nf-test.yml +++ /dev/null @@ -1,182 +0,0 @@ -name: nf-test -run-name: nf-test -on: - push: - branches: [master] - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - name: check-changes - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/nftest_modules.yml" - - nf-test: - runs-on: ubuntu-20.04 - name: nf-test - needs: changes - if: needs.changes.outputs.modules != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.changes.outputs.modules) }}"] - profile: ["docker", "singularity", "conda"] - exclude: - - profile: "conda" - tags: annotsv - - profile: "conda" - tags: bases2fastq - - profile: "conda" - tags: bcl2fastq - - profile: "conda" - tags: bclconvert - - profile: "conda" - tags: cellranger/count - - profile: "conda" - tags: cellranger/mkfastq - - profile: "conda" - tags: cellranger/mkgtf - - profile: "conda" - tags: cellranger/mkref - - profile: "conda" - tags: deepvariant - - profile: "conda" - tags: ensemblvep/vep - - profile: "conda" - tags: fastk/fastk - - profile: "conda" - tags: fastk/histex - - profile: "conda" - tags: fastk/merge - - profile: "conda" - tags: fcs/fcsadaptor - - profile: "conda" - tags: fcs/fcsgx - - profile: "conda" - tags: gatk4/cnnscorevariants - - profile: "conda" - tags: gatk4/determinegermlinecontigploidy - - profile: "conda" - tags: genescopefk - - profile: "conda" - tags: ilastik/multicut - - profile: "conda" - tags: ilastik/pixelclassification - - profile: "conda" - tags: imputeme/vcftoprs - - profile: "conda" - tags: merquryfk/katcomp - - profile: "conda" - tags: merquryfk/katgc - - profile: "conda" - tags: merquryfk/merquryfk - - profile: "conda" - tags: merquryfk/ploidyplot - - profile: "conda" - tags: sentieon/bwaindex - - profile: "conda" - tags: sentieon/bwamem - - profile: "conda" - tags: universc - - profile: "singularity" - tags: universc - - profile: "conda" - tags: subworkflows/vcf_annotate_ensemblvep - env: - NXF_ANSI_LOG: false - SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }}-nextflow- - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Cache nf-test installation - id: cache-software - uses: actions/cache@v3 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest - - - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ - - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - - # Set up secrets - - name: Set up nextflow secrets - if: env.SENTIEON_LICENSE_BASE64 != null - run: | - nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} - nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} - SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) - SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) - SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) - nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 - - # Test the module - - name: Run nf-test - run: | - nf-test test \ - --profile=${{ matrix.profile }} \ - --tag ${{ matrix.tags }} \ - --tap=test.tap - - - uses: pcolby/tap-summary@v1 - with: - path: >- - test.tap diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml deleted file mode 100644 index dcfe6db7cf0..00000000000 --- a/.github/workflows/pytest-workflow.yml +++ /dev/null @@ -1,252 +0,0 @@ -name: pytest -run-name: pytest -on: - push: - branches: [master] - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - name: check-changes - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - - pytest: - runs-on: ubuntu-20.04 - name: pytest - needs: changes - if: needs.changes.outputs.modules != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.changes.outputs.modules) }}"] - profile: ["docker", "singularity", "conda"] - exclude: - - profile: "conda" - tags: annotsv - - profile: "conda" - tags: cellpose - - profile: "conda" - tags: mcquant - - profile: "conda" - tags: bases2fastq - - profile: "conda" - tags: backsub - - profile: "conda" - tags: basicpy - - profile: "conda" - tags: bcl2fastq - - profile: "conda" - tags: bclconvert - - profile: "conda" - tags: cellranger/count - - profile: "conda" - tags: cellranger/mkfastq - - profile: "conda" - tags: cellranger/mkgtf - - profile: "conda" - tags: cellranger/mkref - - profile: "conda" - tags: cellranger/mkvdjref - - profile: "conda" - tags: cellranger/multi - - profile: "conda" - tags: cellranger/vdj - - profile: "conda" - tags: coreograph - - profile: "conda" - tags: deepcell/mesmer - - profile: "conda" - tags: deepvariant - - profile: "conda" - tags: fastk/fastk - - profile: "conda" - tags: fastk/histex - - profile: "conda" - tags: fastk/merge - - profile: "conda" - tags: fcs/fcsadaptor - - profile: "conda" - tags: fcs/fcsgx - - profile: "conda" - tags: gatk4/baserecalibratorspark - - profile: "conda" - tags: gatk4/cnnscorevariants - - profile: "conda" - tags: gatk4/determinegermlinecontigploidy - - profile: "conda" - tags: gatk4/germlinecnvcaller - - profile: "conda" - tags: gatk4/markduplicatesspark - - profile: "conda" - tags: gatk4/postprocessgermlinecnvcalls - - profile: "conda" - tags: genescopefk - - profile: "conda" - tags: ilastik/multicut - - profile: "conda" - tags: ilastik/pixelclassification - - profile: "conda" - tags: imputeme/vcftoprs - - profile: "conda" - tags: merquryfk/katcomp - - profile: "conda" - tags: merquryfk/katgc - - profile: "conda" - tags: merquryfk/merquryfk - - profile: "conda" - tags: merquryfk/ploidyplot - - profile: "conda" - tags: mitohifi/findmitoreference - - profile: "conda" - tags: scimap/mcmicro - - profile: "conda" - tags: sentieon/applyvarcal - - profile: "conda" - tags: sentieon/bwaindex - - profile: "conda" - tags: sentieon/bwamem - - profile: "conda" - tags: sentieon/dedup - - profile: "conda" - tags: sentieon/gvcftyper - - profile: "conda" - tags: sentieon/haplotyper - - profile: "conda" - tags: sentieon/varcal - - profile: "conda" - tags: spaceranger/count - - profile: "conda" - tags: spaceranger/mkgtf - - profile: "conda" - tags: spaceranger/mkref - - profile: "conda" - tags: universc - - profile: "singularity" - tags: universc - - profile: "singularity" - tags: gatk4/determinegermlinecontigploidy - - profile: "conda" - tags: subworkflows/bcl_demultiplex - - profile: "conda" - tags: subworkflows/fasta_clean_fcs - - profile: "conda" - tags: islandpath - env: - NXF_ANSI_LOG: false - SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography - - - uses: actions/cache@v3 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }}-nextflow- - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - - # Set up secrets - - name: Set up nextflow secrets - if: env.SENTIEON_LICENSE_BASE64 != null - run: | - nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} - nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} - SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) - SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) - SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) - nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get update > /dev/null - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - - - name: Setting global variables - uses: actions/github-script@v6 - id: parsed - with: - script: | - return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') - result-encoding: string - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} - path: | - /home/runner/pytest_workflow_*/*/.nextflow.log - /home/runner/pytest_workflow_*/*/log.out - /home/runner/pytest_workflow_*/*/log.err - /home/runner/pytest_workflow_*/*/work - !/home/runner/pytest_workflow_*/*/work/conda - !/home/runner/pytest_workflow_*/*/work/singularity diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..f992ded1f68 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,522 @@ +name: test +on: + push: + branches: [master] + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + prettier: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Install NodeJS + uses: actions/setup-node@v3 + + - name: Install Prettier + run: npm install -g prettier + + - name: Run Prettier --check + run: prettier --check . + + editorconfig: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + + - name: Install editorconfig-checker + run: npm install -g editorconfig-checker + + - name: Run ECLint check + run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) + + pytest-changes: + name: pytest-changes + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'modules' output variable + modules: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + + nf-core-lint: + runs-on: ubuntu-20.04 + name: nf-core-lint + needs: [pytest-changes] + if: needs.pytest-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"] + exclude: + - tags: "nf-test" + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install pip + run: python -m pip install --upgrade pip + + - name: Install nf-core tools + run: python -m pip install nf-core + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Lint ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} + # HACK + if: startsWith( matrix.tags, 'subworkflow' ) != true + + - uses: actions/cache@v3 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }} + + pytest: + runs-on: ubuntu-20.04 + name: pytest + needs: [pytest-changes] + if: needs.pytest-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"] + profile: ["docker", "singularity", "conda"] + exclude: + - tags: "nf-test" + - profile: "conda" + tags: annotsv + - profile: "conda" + tags: cellpose + - profile: "conda" + tags: mcquant + - profile: "conda" + tags: bases2fastq + - profile: "conda" + tags: backsub + - profile: "conda" + tags: basicpy + - profile: "conda" + tags: bcl2fastq + - profile: "conda" + tags: bclconvert + - profile: "conda" + tags: cellranger/count + - profile: "conda" + tags: cellranger/mkfastq + - profile: "conda" + tags: cellranger/mkgtf + - profile: "conda" + tags: cellranger/mkref + - profile: "conda" + tags: cellranger/mkvdjref + - profile: "conda" + tags: cellranger/multi + - profile: "conda" + tags: cellranger/vdj + - profile: "conda" + tags: coreograph + - profile: "conda" + tags: deepcell/mesmer + - profile: "conda" + tags: deepvariant + - profile: "conda" + tags: fastk/fastk + - profile: "conda" + tags: fastk/histex + - profile: "conda" + tags: fastk/merge + - profile: "conda" + tags: fcs/fcsadaptor + - profile: "conda" + tags: fcs/fcsgx + - profile: "conda" + tags: gatk4/baserecalibratorspark + - profile: "conda" + tags: gatk4/cnnscorevariants + - profile: "conda" + tags: gatk4/determinegermlinecontigploidy + - profile: "conda" + tags: gatk4/germlinecnvcaller + - profile: "conda" + tags: gatk4/markduplicatesspark + - profile: "conda" + tags: gatk4/postprocessgermlinecnvcalls + - profile: "conda" + tags: genescopefk + - profile: "conda" + tags: ilastik/multicut + - profile: "conda" + tags: ilastik/pixelclassification + - profile: "conda" + tags: imputeme/vcftoprs + - profile: "conda" + tags: merquryfk/katcomp + - profile: "conda" + tags: merquryfk/katgc + - profile: "conda" + tags: merquryfk/merquryfk + - profile: "conda" + tags: merquryfk/ploidyplot + - profile: "conda" + tags: mitohifi/findmitoreference + - profile: "conda" + tags: scimap/mcmicro + - profile: "conda" + tags: sentieon/applyvarcal + - profile: "conda" + tags: sentieon/bwaindex + - profile: "conda" + tags: sentieon/bwamem + - profile: "conda" + tags: sentieon/dedup + - profile: "conda" + tags: sentieon/gvcftyper + - profile: "conda" + tags: sentieon/haplotyper + - profile: "conda" + tags: sentieon/varcal + - profile: "conda" + tags: spaceranger/count + - profile: "conda" + tags: spaceranger/mkgtf + - profile: "conda" + tags: spaceranger/mkref + - profile: "conda" + tags: universc + - profile: "singularity" + tags: universc + - profile: "singularity" + tags: gatk4/determinegermlinecontigploidy + - profile: "conda" + tags: subworkflows/bcl_demultiplex + - profile: "conda" + tags: subworkflows/fasta_clean_fcs + - profile: "conda" + tags: islandpath + env: + NXF_ANSI_LOG: false + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow cryptography + + - uses: actions/cache@v3 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_LICENSE_BASE64 != null + run: | + nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} + nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} + SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) + SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) + SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) + nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt-get update > /dev/null + sudo apt-get install bat > /dev/null + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} + + - name: Setting global variables + uses: actions/github-script@v6 + id: parsed + with: + script: | + return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') + result-encoding: string + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} + path: | + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity + + nf-test-changes: + name: nf-test-changes + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'modules' output variable + modules: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: "tests/config/nftest_modules.yml" + + nf-test: + runs-on: ubuntu-20.04 + name: nf-test + needs: [nf-test-changes] + if: needs.nf-test-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.nf-test-changes.outputs.modules) }}"] + profile: ["docker", "singularity", "conda"] + exclude: + - tags: "nf-test" + - profile: "conda" + tags: annotsv + - profile: "conda" + tags: bases2fastq + - profile: "conda" + tags: bcl2fastq + - profile: "conda" + tags: bclconvert + - profile: "conda" + tags: cellranger/count + - profile: "conda" + tags: cellranger/mkfastq + - profile: "conda" + tags: cellranger/mkgtf + - profile: "conda" + tags: cellranger/mkref + - profile: "conda" + tags: deepvariant + - profile: "conda" + tags: ensemblvep/vep + - profile: "conda" + tags: fastk/fastk + - profile: "conda" + tags: fastk/histex + - profile: "conda" + tags: fastk/merge + - profile: "conda" + tags: fcs/fcsadaptor + - profile: "conda" + tags: fcs/fcsgx + - profile: "conda" + tags: gatk4/cnnscorevariants + - profile: "conda" + tags: gatk4/determinegermlinecontigploidy + - profile: "conda" + tags: genescopefk + - profile: "conda" + tags: ilastik/multicut + - profile: "conda" + tags: ilastik/pixelclassification + - profile: "conda" + tags: imputeme/vcftoprs + - profile: "conda" + tags: merquryfk/katcomp + - profile: "conda" + tags: merquryfk/katgc + - profile: "conda" + tags: merquryfk/merquryfk + - profile: "conda" + tags: merquryfk/ploidyplot + - profile: "conda" + tags: sentieon/bwaindex + - profile: "conda" + tags: sentieon/bwamem + - profile: "conda" + tags: universc + - profile: "singularity" + tags: universc + - profile: "conda" + tags: subworkflows/vcf_annotate_ensemblvep + env: + NXF_ANSI_LOG: false + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Cache nf-test installation + id: cache-software + uses: actions/cache@v3 + with: + path: | + /usr/local/bin/nf-test + /home/runner/.nf-test/nf-test.jar + key: ${{ runner.os }}-nftest + + - name: Install nf-test + if: steps.cache-software.outputs.cache-hit != 'true' + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_LICENSE_BASE64 != null + run: | + nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} + nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} + SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) + SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) + SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) + nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 + + # Test the module + - name: Run nf-test + run: | + nf-test test \ + --profile=${{ matrix.profile }} \ + --tag ${{ matrix.tags }} \ + --tap=test.tap + + - uses: pcolby/tap-summary@v1 + with: + path: >- + test.tap + + confirm-pass: + runs-on: ubuntu-latest + needs: [prettier, editorconfig, pytest-changes, nf-core-lint, pytest, nf-test-changes, nf-test] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/modules/nf-core/bases2fastq/main.nf b/modules/nf-core/bases2fastq/main.nf index c26d9aca9cf..44d83102dec 100644 --- a/modules/nf-core/bases2fastq/main.nf +++ b/modules/nf-core/bases2fastq/main.nf @@ -4,11 +4,6 @@ process BASES2FASTQ { container "nf-core/bases2fastq:1.1.0" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "BASES2FASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: tuple val(meta), path(run_manifest), path(run_dir) @@ -26,6 +21,10 @@ process BASES2FASTQ { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "BASES2FASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def runManifest = run_manifest ? "-r ${run_manifest}" : "" diff --git a/modules/nf-core/basicpy/main.nf b/modules/nf-core/basicpy/main.nf index bfebe901ce8..cfcf9ac1ae8 100644 --- a/modules/nf-core/basicpy/main.nf +++ b/modules/nf-core/basicpy/main.nf @@ -2,11 +2,6 @@ process BASICPY { tag "$meta.id" label 'process_single' - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "Basicpy module does not support Conda. Please use Docker / Singularity instead." - } - container "docker.io/yfukai/basicpy-docker-mcmicro:0.2.1" input: @@ -20,6 +15,10 @@ process BASICPY { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "Basicpy module does not support Conda. Please use Docker / Singularity instead." + } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def VERSION = "1.0.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping @@ -33,6 +32,10 @@ process BASICPY { """ stub: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "Basicpy module does not support Conda. Please use Docker / Singularity instead." + } """ touch ${prefix}.-dfp.tiff touch ${prefix}.-dfp.tiff diff --git a/modules/nf-core/bcl2fastq/main.nf b/modules/nf-core/bcl2fastq/main.nf index 8bf02ddd8aa..1797bc67c76 100644 --- a/modules/nf-core/bcl2fastq/main.nf +++ b/modules/nf-core/bcl2fastq/main.nf @@ -4,11 +4,6 @@ process BCL2FASTQ { container "nf-core/bcl2fastq:2.20.0.422" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "BCL2FASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: tuple val(meta), path(samplesheet), path(run_dir) @@ -26,6 +21,10 @@ process BCL2FASTQ { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "BCL2FASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def args3 = task.ext.args3 ?: '' diff --git a/modules/nf-core/bclconvert/main.nf b/modules/nf-core/bclconvert/main.nf index 122950dca9c..2d8b7857df6 100644 --- a/modules/nf-core/bclconvert/main.nf +++ b/modules/nf-core/bclconvert/main.nf @@ -4,11 +4,6 @@ process BCLCONVERT { container "nf-core/bclconvert:4.0.3" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "BCLCONVERT module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: tuple val(meta), path(samplesheet), path(run_dir) @@ -26,6 +21,10 @@ process BCLCONVERT { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "BCLCONVERT module does not support Conda. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def args3 = task.ext.args3 ?: '' diff --git a/modules/nf-core/cellpose/main.nf b/modules/nf-core/cellpose/main.nf index f0f462e0ff2..a2b2206923a 100644 --- a/modules/nf-core/cellpose/main.nf +++ b/modules/nf-core/cellpose/main.nf @@ -2,10 +2,6 @@ process CELLPOSE { tag "$meta.id" label 'process_medium' - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "I did not manage to create a cellpose module in Conda that works in all OSes. Please use Docker / Singularity / Podman instead."} - container "docker.io/biocontainers/cellpose:2.1.1_cv2" input: @@ -20,6 +16,10 @@ process CELLPOSE { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "I did not manage to create a cellpose module in Conda that works in all OSes. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def model_command = model ? "--pretrained_model $model" : "" @@ -38,6 +38,10 @@ process CELLPOSE { END_VERSIONS """ stub: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "I did not manage to create a cellpose module in Conda that works in all OSes. Please use Docker / Singularity / Podman instead." + } def prefix = task.ext.prefix ?: "${meta.id}" def VERSION = "2.1.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions. """ diff --git a/modules/nf-core/cellranger/count/main.nf b/modules/nf-core/cellranger/count/main.nf index 0a76cbd4169..d7a191fc721 100644 --- a/modules/nf-core/cellranger/count/main.nf +++ b/modules/nf-core/cellranger/count/main.nf @@ -4,11 +4,6 @@ process CELLRANGER_COUNT { container "nf-core/cellranger:7.1.0" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "CELLRANGER_COUNT module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: tuple val(meta), path(reads, stageAs: "fastq_???/*") path reference @@ -21,11 +16,19 @@ process CELLRANGER_COUNT { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CELLRANGER_COUNT module does not support Conda. Please use Docker / Singularity / Podman instead." + } args = task.ext.args ?: '' prefix = task.ext.prefix ?: "${meta.id}" template "cellranger_count.py" stub: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CELLRANGER_COUNT module does not support Conda. Please use Docker / Singularity / Podman instead." + } def prefix = task.ext.prefix ?: "${meta.id}" """ mkdir -p "${prefix}/outs/" diff --git a/modules/nf-core/cellranger/mkfastq/main.nf b/modules/nf-core/cellranger/mkfastq/main.nf index f57780afbf0..7a128edbd0b 100644 --- a/modules/nf-core/cellranger/mkfastq/main.nf +++ b/modules/nf-core/cellranger/mkfastq/main.nf @@ -4,11 +4,6 @@ process CELLRANGER_MKFASTQ { container "docker.io/nfcore/cellrangermkfastq:7.1.0" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "CELLRANGER_MKFASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: path bcl path csv @@ -21,6 +16,10 @@ process CELLRANGER_MKFASTQ { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CELLRANGER_MKFASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${bcl.getSimpleName()}" """ @@ -38,6 +37,10 @@ process CELLRANGER_MKFASTQ { """ stub: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CELLRANGER_MKFASTQ module does not support Conda. Please use Docker / Singularity / Podman instead." + } def prefix = task.ext.prefix ?: "${bcl.getSimpleName()}" """ mkdir -p "${prefix}/outs/fastq_path/" diff --git a/modules/nf-core/cellranger/mkgtf/main.nf b/modules/nf-core/cellranger/mkgtf/main.nf index 5411b0fdbfc..e0b0dd67e55 100644 --- a/modules/nf-core/cellranger/mkgtf/main.nf +++ b/modules/nf-core/cellranger/mkgtf/main.nf @@ -4,11 +4,6 @@ process CELLRANGER_MKGTF { container "nf-core/cellranger:7.1.0" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "CELLRANGER_MKGTF module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: path gtf @@ -20,6 +15,10 @@ process CELLRANGER_MKGTF { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CELLRANGER_MKGTF module does not support Conda. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${gtf.baseName}.filtered" """ diff --git a/modules/nf-core/cellranger/mkref/main.nf b/modules/nf-core/cellranger/mkref/main.nf index b4b9c547369..986891b8e48 100644 --- a/modules/nf-core/cellranger/mkref/main.nf +++ b/modules/nf-core/cellranger/mkref/main.nf @@ -4,11 +4,6 @@ process CELLRANGER_MKREF { container "nf-core/cellranger:7.1.0" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "CELLRANGER_MKREF module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: path fasta path gtf @@ -22,6 +17,10 @@ process CELLRANGER_MKREF { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CELLRANGER_MKREF module does not support Conda. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' """ cellranger \\ diff --git a/modules/nf-core/cellranger/mkvdjref/main.nf b/modules/nf-core/cellranger/mkvdjref/main.nf index 6d2ccc3949a..64e877f7e15 100644 --- a/modules/nf-core/cellranger/mkvdjref/main.nf +++ b/modules/nf-core/cellranger/mkvdjref/main.nf @@ -4,11 +4,6 @@ process CELLRANGER_MKVDJREF { container "nf-core/cellranger:7.1.0" - // Exit if running this module with -profile conda / -profile mamba - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "CELLRANGER_MKREF module does not support Conda. Please use Docker / Singularity / Podman instead." - } - input: path fasta path gtf @@ -22,6 +17,10 @@ process CELLRANGER_MKVDJREF { task.ext.when == null || task.ext.when script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CELLRANGER_MKVDJREF module does not support Conda. Please use Docker / Singularity / Podman instead." + } def args = task.ext.args ?: '' """ cellranger \\ diff --git a/modules/nf-core/fcs/fcsgx/main.nf b/modules/nf-core/fcs/fcsgx/main.nf index dc40a3fac74..de52886f2b3 100644 --- a/modules/nf-core/fcs/fcsgx/main.nf +++ b/modules/nf-core/fcs/fcsgx/main.nf @@ -3,8 +3,8 @@ process FCS_FCSGX { label 'process_low' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/FCS/releases/0.2.3/fcs-gx.0.2.3.sif': - 'docker.io/ncbi/fcs-gx:0.2.3' }" + 'https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/FCS/releases/0.4.0/fcs-gx.sif': + 'docker.io/ncbi/fcs-gx:0.4.0' }" // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { @@ -26,7 +26,7 @@ process FCS_FCSGX { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def FCSGX_VERSION = '0.2.3' + def FCSGX_VERSION = '0.4.0' """ python3 /app/bin/run_gx \\ @@ -42,4 +42,20 @@ process FCS_FCSGX { FCS-GX: $FCSGX_VERSION END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + def FCSGX_VERSION = '0.4.0' + + """ + mkdir -p out + touch out/${prefix}.fcs_gx_report.txt + touch out/${prefix}.taxonomy.rpt + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + python: \$(python3 --version 2>&1 | sed -e "s/Python //g") + FCS-GX: $FCSGX_VERSION + END_VERSIONS + """ } diff --git a/modules/nf-core/kmcp/profile/main.nf b/modules/nf-core/kmcp/profile/main.nf index cc520ef8907..a46721224b1 100644 --- a/modules/nf-core/kmcp/profile/main.nf +++ b/modules/nf-core/kmcp/profile/main.nf @@ -9,8 +9,7 @@ process KMCP_PROFILE { input: tuple val(meta), path(search_results) - path taxdump - path taxid + path (db) val mode output: @@ -24,11 +23,13 @@ process KMCP_PROFILE { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ + taxid=`find -L ${db} -name "*map"` + taxdump=`find -L ${db}/*/ -type d -not -name "R001"` kmcp \\ profile \\ $args \\ - -X $taxdump \\ - -T $taxid \\ + -X \$taxdump \\ + -T \$taxid \\ -m $mode \\ -j $task.cpus \\ -o ${prefix}.profile \\ diff --git a/modules/nf-core/kmcp/profile/meta.yml b/modules/nf-core/kmcp/profile/meta.yml index 512a882eab4..14f292c758e 100644 --- a/modules/nf-core/kmcp/profile/meta.yml +++ b/modules/nf-core/kmcp/profile/meta.yml @@ -23,18 +23,13 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - db: + type: directory + description: Database directory containing taxdump files and taxid file - search_results: type: file description: Gzipped file output from kmcp search module pattern: "*.gz" - - taxdump: - type: directory - description: Directory of NCBI taxonomy dump files - names.dmp, nodes.dmp - pattern: "*" - - taxid: - type: file - description: Tabular two-column file(s) mapping reference IDs to TaxIds. - pattern: "*.{csv,tsv}" - mode: type: integer description: Profiling mode. diff --git a/modules/nf-core/kmcp/search/main.nf b/modules/nf-core/kmcp/search/main.nf index b0b5e960406..cb2d68435b7 100644 --- a/modules/nf-core/kmcp/search/main.nf +++ b/modules/nf-core/kmcp/search/main.nf @@ -40,8 +40,7 @@ process KMCP_SEARCH { def args = task.ext.args ?: '' prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix} - gzip ${prefix} + touch ${prefix}.gz cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/kmcp/search/meta.yml b/modules/nf-core/kmcp/search/meta.yml index d86ae8d172c..5526a179107 100644 --- a/modules/nf-core/kmcp/search/meta.yml +++ b/modules/nf-core/kmcp/search/meta.yml @@ -1,4 +1,4 @@ -name: "kmcp_compute" +name: "kmcp_search" description: Search sequences against database keywords: - metagenomics diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 2f27c552f76..3a12336e73e 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -2045,10 +2045,18 @@ kmcp/index: - modules/nf-core/kmcp/index/** - tests/modules/nf-core/kmcp/index/** +kmcp/merge: + - modules/nf-core/kmcp/merge/** + - tests/modules/nf-core/kmcp/merge/** + kmcp/profile: - modules/nf-core/kmcp/profile/** - tests/modules/nf-core/kmcp/profile/** +kmcp/search: + - modules/nf-core/kmcp/search/** + - tests/modules/nf-core/kmcp/search/** + kofamscan: - modules/nf-core/kofamscan/** - tests/modules/nf-core/kofamscan/** diff --git a/tests/modules/nf-core/bases2fastq/test.yml b/tests/modules/nf-core/bases2fastq/test.yml index 88fe9c238da..9e2aa9ac129 100644 --- a/tests/modules/nf-core/bases2fastq/test.yml +++ b/tests/modules/nf-core/bases2fastq/test.yml @@ -5,14 +5,14 @@ files: - path: output/bases2fastq/output/Bases2Fastq-Sim_QC.html - path: output/bases2fastq/output/Metrics.csv - md5sum: 0ec6da2b82e191098283474356024abf + md5sum: a8094b308b5653071ac029a27733e4a6 - path: output/bases2fastq/output/RunManifest.json md5sum: a07dce8ee25c2a6f9355b677c26b53e2 - path: output/bases2fastq/output/RunStats.json - path: output/bases2fastq/output/UnassignedSequences.csv md5sum: 11c1693830ce941b8cfb8d2431a59097 - path: output/bases2fastq/output/Samples/DefaultSample/DefaultSample_R1.fastq.gz - md5sum: 218abc70f61e8e8199a68f83ae836184 + md5sum: 831c90ea31eff881e825cda691da77ae - path: output/bases2fastq/output/Samples/DefaultSample/DefaultSample_R2.fastq.gz - md5sum: b95109bfb204daa150b61239d3368ee2 + md5sum: e6c1f7ec00910ef195ff3d427c7a9f23 - path: output/bases2fastq/output/Samples/DefaultSample/DefaultSample_stats.json diff --git a/tests/modules/nf-core/kmcp/compute/main.nf b/tests/modules/nf-core/kmcp/compute/main.nf index 73679ebae12..59860296890 100644 --- a/tests/modules/nf-core/kmcp/compute/main.nf +++ b/tests/modules/nf-core/kmcp/compute/main.nf @@ -19,7 +19,7 @@ workflow test_kmcp_compute_directory { input = UNTAR ( [ [ id:'test' ], - file("https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/kmcp.tar.gz", checkIfExists: true) + file("https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/kmcp/kmcp_profile.tar.gz", checkIfExists: true) ]).untar KMCP_COMPUTE ( input ) diff --git a/tests/modules/nf-core/kmcp/compute/test.yml b/tests/modules/nf-core/kmcp/compute/test.yml index e10ea1c2873..a8dbb2a24ee 100644 --- a/tests/modules/nf-core/kmcp/compute/test.yml +++ b/tests/modules/nf-core/kmcp/compute/test.yml @@ -1,8 +1,8 @@ - name: kmcp compute test_kmcp_compute command: nextflow run ./tests/modules/nf-core/kmcp/compute -entry test_kmcp_compute -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/kmcp/compute/nextflow.config tags: - - kmcp - kmcp/compute + - kmcp files: - path: output/kmcp/test_/_info.txt contains: ["#path name chunkIdx idxNum genomeSize kmers"] @@ -27,5 +27,4 @@ - path: output/kmcp/test_/_info.txt contains: ["#path name chunkIdx idxNum genomeSize kmers"] - path: output/kmcp/test_/genome.fasta.unik - - path: output/kmcp/test_/transcriptome.fasta.unik - path: output/kmcp/versions.yml diff --git a/tests/modules/nf-core/kmcp/profile/main.nf b/tests/modules/nf-core/kmcp/profile/main.nf index e41f4a2a7c1..bed70144213 100644 --- a/tests/modules/nf-core/kmcp/profile/main.nf +++ b/tests/modules/nf-core/kmcp/profile/main.nf @@ -10,23 +10,21 @@ include { KMCP_PROFILE } from '../../../../../modules/nf-core/kmcp/profile/main. workflow test_kmcp_profile { - input_compute = [ - [ id:'test', single_end:false ], // meta map - file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/kmcp/NC_045512.2.fasta', checkIfExists: true) - ] + input_compute= UNTAR ( [ + [ id:'test' ], + file("https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/kmcp/kmcp_profile.tar.gz", checkIfExists: true) + ]).untar + input = [ [ id:'test', single_end:true ], // meta map file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] - taxdump = [ [],file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/kmcp/kmcp_profile.tar.gz', checkIfExists: true)] - ch_taxid = file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/kmcp/seqid2taxid.map', checkIfExists: true) mode = 3 - UNTAR (taxdump) - KMCP_COMPUTE ( input_compute ) + KMCP_COMPUTE (input_compute ) KMCP_INDEX ( KMCP_COMPUTE.out.outdir ) KMCP_SEARCH ( KMCP_INDEX.out.kmcp.map{it[1]}, input ) - KMCP_PROFILE ( KMCP_SEARCH.out.result, UNTAR.out.untar.map{ it[1] }, ch_taxid, mode ) + KMCP_PROFILE ( KMCP_SEARCH.out.result, input_compute.map{ it[1] }, mode ) } diff --git a/tests/modules/nf-core/kmcp/profile/nextflow.config b/tests/modules/nf-core/kmcp/profile/nextflow.config index dafa8f3f73f..98a493be45f 100644 --- a/tests/modules/nf-core/kmcp/profile/nextflow.config +++ b/tests/modules/nf-core/kmcp/profile/nextflow.config @@ -2,7 +2,7 @@ process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - withName: KMCP_INDEX { + withName: KMCP_COMPUTE { ext.prefix = { "${meta.id}_" } } diff --git a/tests/modules/nf-core/kmcp/profile/test.yml b/tests/modules/nf-core/kmcp/profile/test.yml index 4d537bbd5ea..6dac88d16e3 100644 --- a/tests/modules/nf-core/kmcp/profile/test.yml +++ b/tests/modules/nf-core/kmcp/profile/test.yml @@ -1,17 +1,17 @@ - name: kmcp profile test_kmcp_profile command: nextflow run ./tests/modules/nf-core/kmcp/profile -entry test_kmcp_profile -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/kmcp/profile/nextflow.config tags: - - kmcp/profile - kmcp + - kmcp/profile files: - path: output/kmcp/test.profile - - path: output/untar/versions.yml + - path: output/kmcp/versions.yml - name: kmcp profile test_kmcp_profile stub_run command: nextflow run ./tests/modules/nf-core/kmcp/profile -entry test_kmcp_profile -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/kmcp/profile/nextflow.config -stub-run tags: - - kmcp/profile - kmcp + - kmcp/profile files: - path: output/kmcp/test.profile - - path: output/untar/versions.yml + - path: output/kmcp/versions.yml diff --git a/tests/modules/nf-core/kmcp/search/nextflow.config b/tests/modules/nf-core/kmcp/search/nextflow.config index 4eb081d2033..dafa8f3f73f 100644 --- a/tests/modules/nf-core/kmcp/search/nextflow.config +++ b/tests/modules/nf-core/kmcp/search/nextflow.config @@ -6,8 +6,4 @@ process { ext.prefix = { "${meta.id}_" } } - withName: KMCP_SEARCH { - ext.prefix = { "${meta.id}_" } - } - } diff --git a/tests/modules/nf-core/kmcp/search/test.yml b/tests/modules/nf-core/kmcp/search/test.yml index eb356e8755e..7669b72d494 100644 --- a/tests/modules/nf-core/kmcp/search/test.yml +++ b/tests/modules/nf-core/kmcp/search/test.yml @@ -4,9 +4,7 @@ - kmcp - kmcp/search files: - - path: output/kmcp/test_/R001/__db.yml - - path: output/kmcp/test_/R001/__name_mapping.tsv - - path: output/kmcp/test_/R001/_block001.uniki + - path: output/kmcp/test.gz - path: output/kmcp/versions.yml - name: kmcp search test_kmcp_search stub_run @@ -15,7 +13,5 @@ - kmcp - kmcp/search files: - - path: output/kmcp/test_/R001/__db.yml - - path: output/kmcp/test_/R001/__name_mapping.tsv - - path: output/kmcp/test_/R001/_block001.uniki + - path: output/kmcp/test.gz - path: output/kmcp/versions.yml