From 3a52c173053ad677c742ecdf72e6b8308682a0cb Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Wed, 2 Aug 2023 13:35:41 +0100 Subject: [PATCH] Hierarchy of tests applied in order (#3679) * Add wait-for-status workflow * Rename and tidy all GHA workflows for branch protection * Changing some stuff to kick off real CI/CD * Apply suggestions from code review Co-authored-by: Harshil Patel * Update .github/workflows/code-linting.yml Co-authored-by: Harshil Patel * Should reduce concurrent burden at the expense of time. Code is required to lint before running. This will help us during hackathons etc when there is a high burden on the GHA runners, but will be slower in general use. Changes: - linting and testing in one GHA file - Linting is required to pass before pytest or nf-test runs - Check for changes only happens once * Fix dependencies in GHA * Remove files again * More dependency fixes * Correct nf-test filtering yaml * Change to fix-linting for consistency * Bump setup-python version to v4 * bump eWaterCycle/setup-singularity to v7 * Add final job to test for pass status of all previous jobs * Add checking for changes to test requirement * Revert module changes * Add fake change to FASTQC * Test in parallel * Revert "Add fake change to FASTQC" This reverts commit f050865cd6260c2e6278bdeaef2cb62f14a91a7e. * Change logic to allow skipped processes * Format changes * Change logic again * Change logic again * Change logic again * Add debug step to help work it out * Caught pytest-changes bug * Reintroduce change to FASTQC module * Remove debug statements * Revert "Reintroduce change to FASTQC module" This reverts commit 197ffeb0efdba72ab6c14c6d05dba583874790dd. * Mess up FASTQC deliberately to test failures * Revert "Mess up FASTQC deliberately to test failures" This reverts commit c28fef48f041da12116d008463394c97080c2568. * Changed to all lower case --------- Co-authored-by: Harshil Patel --- .github/workflows/code-linting.yml | 43 --- .github/workflows/nf-core-linting.yml | 83 ---- .github/workflows/nf-test.yml | 182 --------- .github/workflows/pytest-workflow.yml | 252 ------------- .github/workflows/test.yml | 522 ++++++++++++++++++++++++++ 5 files changed, 522 insertions(+), 560 deletions(-) delete mode 100644 .github/workflows/code-linting.yml delete mode 100644 .github/workflows/nf-core-linting.yml delete mode 100644 .github/workflows/nf-test.yml delete mode 100644 .github/workflows/pytest-workflow.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/code-linting.yml b/.github/workflows/code-linting.yml deleted file mode 100644 index 831453f8ea2..00000000000 --- a/.github/workflows/code-linting.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: code-lint -on: - push: - branches: [master] - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - Prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Install NodeJS - uses: actions/setup-node@v2 - - - name: Install Prettier - run: npm install -g prettier - - - name: Run Prettier --check - run: prettier --check . - - EditorConfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v2 - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) diff --git a/.github/workflows/nf-core-linting.yml b/.github/workflows/nf-core-linting.yml deleted file mode 100644 index 123decf27e6..00000000000 --- a/.github/workflows/nf-core-linting.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: nf-core-lint -run-name: nf-core-lint -# This workflow is triggered on pushes and PRs to the repository. -# It runs the `nf-core lint` tests to ensure that the module code meets the nf-core guidelines -on: - push: - branches: [master] - pull_request: - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - name: check-changes - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - - lint: - runs-on: ubuntu-20.04 - name: lint - needs: changes - if: needs.changes.outputs.modules != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.changes.outputs.modules) }}"] - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - name: Install pip - run: python -m pip install --upgrade pip - - - name: Install nf-core tools - run: python -m pip install nf-core - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Lint ${{ matrix.tags }} - run: nf-core modules lint ${{ matrix.tags }} - # HACK - if: startsWith( matrix.tags, 'subworkflow' ) != true - - - uses: actions/cache@v3 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }} diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml deleted file mode 100644 index 57df4fef4c2..00000000000 --- a/.github/workflows/nf-test.yml +++ /dev/null @@ -1,182 +0,0 @@ -name: nf-test -run-name: nf-test -on: - push: - branches: [master] - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - name: check-changes - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/nftest_modules.yml" - - nf-test: - runs-on: ubuntu-20.04 - name: nf-test - needs: changes - if: needs.changes.outputs.modules != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.changes.outputs.modules) }}"] - profile: ["docker", "singularity", "conda"] - exclude: - - profile: "conda" - tags: annotsv - - profile: "conda" - tags: bases2fastq - - profile: "conda" - tags: bcl2fastq - - profile: "conda" - tags: bclconvert - - profile: "conda" - tags: cellranger/count - - profile: "conda" - tags: cellranger/mkfastq - - profile: "conda" - tags: cellranger/mkgtf - - profile: "conda" - tags: cellranger/mkref - - profile: "conda" - tags: deepvariant - - profile: "conda" - tags: ensemblvep/vep - - profile: "conda" - tags: fastk/fastk - - profile: "conda" - tags: fastk/histex - - profile: "conda" - tags: fastk/merge - - profile: "conda" - tags: fcs/fcsadaptor - - profile: "conda" - tags: fcs/fcsgx - - profile: "conda" - tags: gatk4/cnnscorevariants - - profile: "conda" - tags: gatk4/determinegermlinecontigploidy - - profile: "conda" - tags: genescopefk - - profile: "conda" - tags: ilastik/multicut - - profile: "conda" - tags: ilastik/pixelclassification - - profile: "conda" - tags: imputeme/vcftoprs - - profile: "conda" - tags: merquryfk/katcomp - - profile: "conda" - tags: merquryfk/katgc - - profile: "conda" - tags: merquryfk/merquryfk - - profile: "conda" - tags: merquryfk/ploidyplot - - profile: "conda" - tags: sentieon/bwaindex - - profile: "conda" - tags: sentieon/bwamem - - profile: "conda" - tags: universc - - profile: "singularity" - tags: universc - - profile: "conda" - tags: subworkflows/vcf_annotate_ensemblvep - env: - NXF_ANSI_LOG: false - SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }}-nextflow- - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Cache nf-test installation - id: cache-software - uses: actions/cache@v3 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest - - - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ - - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - - # Set up secrets - - name: Set up nextflow secrets - if: env.SENTIEON_LICENSE_BASE64 != null - run: | - nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} - nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} - SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) - SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) - SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) - nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 - - # Test the module - - name: Run nf-test - run: | - nf-test test \ - --profile=${{ matrix.profile }} \ - --tag ${{ matrix.tags }} \ - --tap=test.tap - - - uses: pcolby/tap-summary@v1 - with: - path: >- - test.tap diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml deleted file mode 100644 index dcfe6db7cf0..00000000000 --- a/.github/workflows/pytest-workflow.yml +++ /dev/null @@ -1,252 +0,0 @@ -name: pytest -run-name: pytest -on: - push: - branches: [master] - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - name: check-changes - runs-on: ubuntu-latest - outputs: - # Expose matched filters as job 'modules' output variable - modules: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - - pytest: - runs-on: ubuntu-20.04 - name: pytest - needs: changes - if: needs.changes.outputs.modules != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.changes.outputs.modules) }}"] - profile: ["docker", "singularity", "conda"] - exclude: - - profile: "conda" - tags: annotsv - - profile: "conda" - tags: cellpose - - profile: "conda" - tags: mcquant - - profile: "conda" - tags: bases2fastq - - profile: "conda" - tags: backsub - - profile: "conda" - tags: basicpy - - profile: "conda" - tags: bcl2fastq - - profile: "conda" - tags: bclconvert - - profile: "conda" - tags: cellranger/count - - profile: "conda" - tags: cellranger/mkfastq - - profile: "conda" - tags: cellranger/mkgtf - - profile: "conda" - tags: cellranger/mkref - - profile: "conda" - tags: cellranger/mkvdjref - - profile: "conda" - tags: cellranger/multi - - profile: "conda" - tags: cellranger/vdj - - profile: "conda" - tags: coreograph - - profile: "conda" - tags: deepcell/mesmer - - profile: "conda" - tags: deepvariant - - profile: "conda" - tags: fastk/fastk - - profile: "conda" - tags: fastk/histex - - profile: "conda" - tags: fastk/merge - - profile: "conda" - tags: fcs/fcsadaptor - - profile: "conda" - tags: fcs/fcsgx - - profile: "conda" - tags: gatk4/baserecalibratorspark - - profile: "conda" - tags: gatk4/cnnscorevariants - - profile: "conda" - tags: gatk4/determinegermlinecontigploidy - - profile: "conda" - tags: gatk4/germlinecnvcaller - - profile: "conda" - tags: gatk4/markduplicatesspark - - profile: "conda" - tags: gatk4/postprocessgermlinecnvcalls - - profile: "conda" - tags: genescopefk - - profile: "conda" - tags: ilastik/multicut - - profile: "conda" - tags: ilastik/pixelclassification - - profile: "conda" - tags: imputeme/vcftoprs - - profile: "conda" - tags: merquryfk/katcomp - - profile: "conda" - tags: merquryfk/katgc - - profile: "conda" - tags: merquryfk/merquryfk - - profile: "conda" - tags: merquryfk/ploidyplot - - profile: "conda" - tags: mitohifi/findmitoreference - - profile: "conda" - tags: scimap/mcmicro - - profile: "conda" - tags: sentieon/applyvarcal - - profile: "conda" - tags: sentieon/bwaindex - - profile: "conda" - tags: sentieon/bwamem - - profile: "conda" - tags: sentieon/dedup - - profile: "conda" - tags: sentieon/gvcftyper - - profile: "conda" - tags: sentieon/haplotyper - - profile: "conda" - tags: sentieon/varcal - - profile: "conda" - tags: spaceranger/count - - profile: "conda" - tags: spaceranger/mkgtf - - profile: "conda" - tags: spaceranger/mkref - - profile: "conda" - tags: universc - - profile: "singularity" - tags: universc - - profile: "singularity" - tags: gatk4/determinegermlinecontigploidy - - profile: "conda" - tags: subworkflows/bcl_demultiplex - - profile: "conda" - tags: subworkflows/fasta_clean_fcs - - profile: "conda" - tags: islandpath - env: - NXF_ANSI_LOG: false - SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography - - - uses: actions/cache@v3 - with: - path: /usr/local/bin/nextflow - key: ${{ runner.os }} - restore-keys: | - ${{ runner.os }}-nextflow- - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 - with: - singularity-version: 3.7.1 - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - - # Set up secrets - - name: Set up nextflow secrets - if: env.SENTIEON_LICENSE_BASE64 != null - run: | - nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} - nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} - SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) - SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) - SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) - nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get update > /dev/null - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} - - - name: Setting global variables - uses: actions/github-script@v6 - id: parsed - with: - script: | - return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') - result-encoding: string - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} - path: | - /home/runner/pytest_workflow_*/*/.nextflow.log - /home/runner/pytest_workflow_*/*/log.out - /home/runner/pytest_workflow_*/*/log.err - /home/runner/pytest_workflow_*/*/work - !/home/runner/pytest_workflow_*/*/work/conda - !/home/runner/pytest_workflow_*/*/work/singularity diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..f992ded1f68 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,522 @@ +name: test +on: + push: + branches: [master] + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + prettier: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Install NodeJS + uses: actions/setup-node@v3 + + - name: Install Prettier + run: npm install -g prettier + + - name: Run Prettier --check + run: prettier --check . + + editorconfig: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + + - name: Install editorconfig-checker + run: npm install -g editorconfig-checker + + - name: Run ECLint check + run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) + + pytest-changes: + name: pytest-changes + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'modules' output variable + modules: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + + nf-core-lint: + runs-on: ubuntu-20.04 + name: nf-core-lint + needs: [pytest-changes] + if: needs.pytest-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"] + exclude: + - tags: "nf-test" + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install pip + run: python -m pip install --upgrade pip + + - name: Install nf-core tools + run: python -m pip install nf-core + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Lint ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} + # HACK + if: startsWith( matrix.tags, 'subworkflow' ) != true + + - uses: actions/cache@v3 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }} + + pytest: + runs-on: ubuntu-20.04 + name: pytest + needs: [pytest-changes] + if: needs.pytest-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"] + profile: ["docker", "singularity", "conda"] + exclude: + - tags: "nf-test" + - profile: "conda" + tags: annotsv + - profile: "conda" + tags: cellpose + - profile: "conda" + tags: mcquant + - profile: "conda" + tags: bases2fastq + - profile: "conda" + tags: backsub + - profile: "conda" + tags: basicpy + - profile: "conda" + tags: bcl2fastq + - profile: "conda" + tags: bclconvert + - profile: "conda" + tags: cellranger/count + - profile: "conda" + tags: cellranger/mkfastq + - profile: "conda" + tags: cellranger/mkgtf + - profile: "conda" + tags: cellranger/mkref + - profile: "conda" + tags: cellranger/mkvdjref + - profile: "conda" + tags: cellranger/multi + - profile: "conda" + tags: cellranger/vdj + - profile: "conda" + tags: coreograph + - profile: "conda" + tags: deepcell/mesmer + - profile: "conda" + tags: deepvariant + - profile: "conda" + tags: fastk/fastk + - profile: "conda" + tags: fastk/histex + - profile: "conda" + tags: fastk/merge + - profile: "conda" + tags: fcs/fcsadaptor + - profile: "conda" + tags: fcs/fcsgx + - profile: "conda" + tags: gatk4/baserecalibratorspark + - profile: "conda" + tags: gatk4/cnnscorevariants + - profile: "conda" + tags: gatk4/determinegermlinecontigploidy + - profile: "conda" + tags: gatk4/germlinecnvcaller + - profile: "conda" + tags: gatk4/markduplicatesspark + - profile: "conda" + tags: gatk4/postprocessgermlinecnvcalls + - profile: "conda" + tags: genescopefk + - profile: "conda" + tags: ilastik/multicut + - profile: "conda" + tags: ilastik/pixelclassification + - profile: "conda" + tags: imputeme/vcftoprs + - profile: "conda" + tags: merquryfk/katcomp + - profile: "conda" + tags: merquryfk/katgc + - profile: "conda" + tags: merquryfk/merquryfk + - profile: "conda" + tags: merquryfk/ploidyplot + - profile: "conda" + tags: mitohifi/findmitoreference + - profile: "conda" + tags: scimap/mcmicro + - profile: "conda" + tags: sentieon/applyvarcal + - profile: "conda" + tags: sentieon/bwaindex + - profile: "conda" + tags: sentieon/bwamem + - profile: "conda" + tags: sentieon/dedup + - profile: "conda" + tags: sentieon/gvcftyper + - profile: "conda" + tags: sentieon/haplotyper + - profile: "conda" + tags: sentieon/varcal + - profile: "conda" + tags: spaceranger/count + - profile: "conda" + tags: spaceranger/mkgtf + - profile: "conda" + tags: spaceranger/mkref + - profile: "conda" + tags: universc + - profile: "singularity" + tags: universc + - profile: "singularity" + tags: gatk4/determinegermlinecontigploidy + - profile: "conda" + tags: subworkflows/bcl_demultiplex + - profile: "conda" + tags: subworkflows/fasta_clean_fcs + - profile: "conda" + tags: islandpath + env: + NXF_ANSI_LOG: false + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow cryptography + + - uses: actions/cache@v3 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_LICENSE_BASE64 != null + run: | + nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} + nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} + SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) + SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) + SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) + nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt-get update > /dev/null + sudo apt-get install bat > /dev/null + batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err} + + - name: Setting global variables + uses: actions/github-script@v6 + id: parsed + with: + script: | + return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') + result-encoding: string + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} + path: | + /home/runner/pytest_workflow_*/*/.nextflow.log + /home/runner/pytest_workflow_*/*/log.out + /home/runner/pytest_workflow_*/*/log.err + /home/runner/pytest_workflow_*/*/work + !/home/runner/pytest_workflow_*/*/work/conda + !/home/runner/pytest_workflow_*/*/work/singularity + + nf-test-changes: + name: nf-test-changes + runs-on: ubuntu-latest + outputs: + # Expose matched filters as job 'modules' output variable + modules: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: "tests/config/nftest_modules.yml" + + nf-test: + runs-on: ubuntu-20.04 + name: nf-test + needs: [nf-test-changes] + if: needs.nf-test-changes.outputs.modules != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.nf-test-changes.outputs.modules) }}"] + profile: ["docker", "singularity", "conda"] + exclude: + - tags: "nf-test" + - profile: "conda" + tags: annotsv + - profile: "conda" + tags: bases2fastq + - profile: "conda" + tags: bcl2fastq + - profile: "conda" + tags: bclconvert + - profile: "conda" + tags: cellranger/count + - profile: "conda" + tags: cellranger/mkfastq + - profile: "conda" + tags: cellranger/mkgtf + - profile: "conda" + tags: cellranger/mkref + - profile: "conda" + tags: deepvariant + - profile: "conda" + tags: ensemblvep/vep + - profile: "conda" + tags: fastk/fastk + - profile: "conda" + tags: fastk/histex + - profile: "conda" + tags: fastk/merge + - profile: "conda" + tags: fcs/fcsadaptor + - profile: "conda" + tags: fcs/fcsgx + - profile: "conda" + tags: gatk4/cnnscorevariants + - profile: "conda" + tags: gatk4/determinegermlinecontigploidy + - profile: "conda" + tags: genescopefk + - profile: "conda" + tags: ilastik/multicut + - profile: "conda" + tags: ilastik/pixelclassification + - profile: "conda" + tags: imputeme/vcftoprs + - profile: "conda" + tags: merquryfk/katcomp + - profile: "conda" + tags: merquryfk/katgc + - profile: "conda" + tags: merquryfk/merquryfk + - profile: "conda" + tags: merquryfk/ploidyplot + - profile: "conda" + tags: sentieon/bwaindex + - profile: "conda" + tags: sentieon/bwamem + - profile: "conda" + tags: universc + - profile: "singularity" + tags: universc + - profile: "conda" + tags: subworkflows/vcf_annotate_ensemblvep + env: + NXF_ANSI_LOG: false + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + + - name: Install Nextflow + env: + CAPSULE_LOG: none + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Cache nf-test installation + id: cache-software + uses: actions/cache@v3 + with: + path: | + /usr/local/bin/nf-test + /home/runner/.nf-test/nf-test.jar + key: ${{ runner.os }}-nftest + + - name: Install nf-test + if: steps.cache-software.outputs.cache-hit != 'true' + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.7.1 + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_LICENSE_BASE64 != null + run: | + nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} + nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} + SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) + SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) + SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) + nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 + + # Test the module + - name: Run nf-test + run: | + nf-test test \ + --profile=${{ matrix.profile }} \ + --tag ${{ matrix.tags }} \ + --tap=test.tap + + - uses: pcolby/tap-summary@v1 + with: + path: >- + test.tap + + confirm-pass: + runs-on: ubuntu-latest + needs: [prettier, editorconfig, pytest-changes, nf-core-lint, pytest, nf-test-changes, nf-test] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"