diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index a86fd9bb10a..a9463bd542d 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -21,7 +21,7 @@ jobs: if: github.event.pull_request.head.repo.full_name == github.repository outputs: conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }} - docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }} + dockerfile-matrix: ${{ steps.docker-diff.outputs.all_changed_files }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -48,7 +48,7 @@ jobs: # if: github.repository == 'nf-core/modules' if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }} needs: generate-matrix - name: Build Conda-based ${{ matrix.profile }} Container + name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -57,6 +57,7 @@ jobs: matrix: files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}" profile: [docker, singularity] + platform: ["linux/amd64", "linux/arm64"] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -76,23 +77,27 @@ jobs: $PROFILE \ --freeze \ --await \ + --platform ${{ matrix.platform }} \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} - docker-wave: + dockerfile-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: ${{ needs.generate-matrix.outputs.docker-matrix != '[]' }} + if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '[]' }} needs: generate-matrix - name: Build Dockerfile-based ${{ matrix.profile }} Container + name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false max-parallel: 4 matrix: - files: "${{ fromJson(needs.generate-matrix.outputs.docker-matrix) }}" - profile: [docker, singularity] + files: "${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }}" + # NOTE singularity build requires a Singularity definition file in place of Dockerfile + # https://nextflow.slack.com/archives/C0477AS31T5/p1731755350230149?thread_ts=1731697852.661849&cid=C0477AS31T5 + profile: [docker] + platform: ["linux/amd64", "linux/arm64"] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -113,15 +118,13 @@ jobs: - name: Build ${{ matrix.profile }} container # NOTE If you're changing a Dockerfile and this is running, try to update the Dockerfile to build with wave continue-on-error: false - env: - PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} run: | wave -f "${{ matrix.files }}" \ - $PROFILE \ --freeze \ --await \ + --platform ${{ matrix.platform }} \ --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \ - --cache-repository quay.io/nf-core/wave-cache \ + --cache-repository quay.io/nf-core/wave-cache/${{ matrix.profile }}/${{ matrix.platform }} \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} diff --git a/modules/nf-core/quartonotebook/Dockerfile b/modules/nf-core/quartonotebook/Dockerfile index 26624b7ea6d..06b7bee80b4 100644 --- a/modules/nf-core/quartonotebook/Dockerfile +++ b/modules/nf-core/quartonotebook/Dockerfile @@ -1,25 +1,17 @@ -# -# First stage: Quarto installation -# -FROM ubuntu:24.04 as quarto +FROM condaforge/mambaforge:24.9.2-0 +#@sha256:e4edd160d0a10c6f577b127021d853b460e7e8c1bf1850c9aa4a52c4b8d9c8ee ARG QUARTO_VERSION=1.3.433 ARG TARGETARCH -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - && apt-get clean -RUN mkdir -p /opt/quarto \ - && curl -o quarto.tar.gz -L "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-${TARGETARCH}.tar.gz" \ - && tar -zxvf quarto.tar.gz -C /opt/quarto/ --strip-components=1 \ - && rm quarto.tar.gz +LABEL org.opencontainers.image.authors="Erik Fasterius" +LABEL org.opencontainers.image.description="Dockerfile for the quartonotebook nf-core module" +LABEL org.opencontainers.image.version="$QUARTO_VERSION" +LABEL org.opencontainers.image.documentation="https://github.com/nf-core/modules/blob/master/modules/nf-core/quartonotebook/README.md" +LABEL org.opencontainers.image.source="https://github.com/nf-core/modules" +LABEL org.opencontainers.image.vendor="nf-core" +LABEL org.opencontainers.image.license="https://github.com/nf-core/modules/blob/master/LICENSE" -# -# Second stage: Conda environment -# -FROM condaforge/mambaforge:24.1.2-0@sha256:64c45c1a743737f61cf201f54cae974b5c853be94f9c1a84f5e82e0e854f0407 -COPY --from=quarto /opt/quarto /opt/quarto +ADD https://github.com/quarto-dev/quarto-cli#v${QUARTO_VERSION} /opt/quarto ENV PATH="${PATH}:/opt/quarto/bin" # Install packages using Mamba; also remove static libraries, python bytecode @@ -32,7 +24,3 @@ RUN mamba env update --name base --file environment.yml \ && find /opt/conda -follow -type f -name '*.js.map' -delete CMD /bin/bash - -LABEL \ - authors = "Erik Fasterius" \ - description = "Dockerfile for the quartonotebook nf-core module" diff --git a/modules/nf-core/quartonotebook/environment.yml b/modules/nf-core/quartonotebook/environment.yml index 3a48c788a76..b79fb2e075e 100644 --- a/modules/nf-core/quartonotebook/environment.yml +++ b/modules/nf-core/quartonotebook/environment.yml @@ -1,9 +1,15 @@ channels: - conda-forge - bioconda - dependencies: + # renovate: datasource=conda depName=conda-forge/quarto + # FIXME https://github.com/nf-core/modules/issues/7006 + - conda-forge::quarto=1.5.57 + # renovate: datasource=conda depName=conda-forge/jupyter - conda-forge::jupyter=1.0.0 + # renovate: datasource=conda depName=conda-forge/matplotlib - conda-forge::matplotlib=3.4.3 + # renovate: datasource=conda depName=conda-forge/papermill - conda-forge::papermill=2.4.0 + # renovate: datasource=conda depName=conda-forge/r-rmarkdown - conda-forge::r-rmarkdown=2.25 diff --git a/modules/nf-core/quartonotebook/main.nf b/modules/nf-core/quartonotebook/main.nf index 45a2d187d10..97990a717d6 100644 --- a/modules/nf-core/quartonotebook/main.nf +++ b/modules/nf-core/quartonotebook/main.nf @@ -1,14 +1,20 @@ -include { dumpParamsYaml; indentCodeBlock } from "./parametrize" +include { + dumpParamsYaml ; + indentCodeBlock +} from "./parametrize" +// NB: You'll likely want to override this with a container containing all +// required dependencies for your analyses. Or use wave to build the container +// for you from the environment.yml You'll at least need Quarto itself, +// Papermill and whatever language you are running your analyses on; you can see +// an example in this module's Dockerfile. process QUARTONOTEBOOK { - tag "$meta.id" + tag "${meta.id}" label 'process_low' - - // NB: You'll likely want to override this with a container containing all - // required dependencies for your analyses. You'll at least need Quarto - // itself, Papermill and whatever language you are running your analyses on; - // you can see an example in this module's Dockerfile. - container "docker.io/erikfas/quartonotebook" + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/25/25d0a9decefd5d369b8f9b8c934640bd61493df2c95c39b0c580e765e0d2a644/data' + : 'community.wave.seqera.io/library/quarto_jupyter_matplotlib_papermill_r-rmarkdown:95c6620495eabcd1'}" input: tuple val(meta), path(notebook) @@ -17,29 +23,33 @@ process QUARTONOTEBOOK { path extensions output: - tuple val(meta), path("*.html") , emit: html + tuple val(meta), path("*.html"), emit: html tuple val(meta), path("${notebook}"), emit: notebook tuple val(meta), path("artifacts/*"), emit: artifacts, optional: true - tuple val(meta), path("params.yml") , emit: params_yaml, optional: true + tuple val(meta), path("params.yml"), emit: params_yaml, optional: true tuple val(meta), path("_extensions"), emit: extensions, optional: true - path "versions.yml" , emit: versions + path "versions.yml", emit: versions when: task.ext.when == null || task.ext.when script: - // Exit if running this module with -profile conda / -profile mamba + // FIXME https://github.com/nf-core/modules/issues/7006 + // Exit if running this module with -profile conda / -profile mamba on ARM64 // This is because of issues with getting a homogenous environment across // both AMD64 and ARM64 architectures; please find more information at // https://github.com/nf-core/modules/pull/4876#discussion_r1483541037. if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - exit 1, "The QUARTONOTEBOOK module does not support Conda/Mamba, please use Docker / Singularity / Podman instead." + arch = System.getProperty("os.arch") + if (arch == "arm64" || arch == "aarch64") { + exit(1, "The QUARTONOTEBOOK module does not support Conda/Mamba on ARM64. Please use Docker / Singularity / Podman instead.") + } } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def parametrize = (task.ext.parametrize == null) ? true : task.ext.parametrize - def implicit_params = (task.ext.implicit_params == null) ? true : task.ext.implicit_params - def meta_params = (task.ext.meta_params == null) ? true : task.ext.meta_params + def parametrize = task.ext.parametrize == null ? true : task.ext.parametrize + def implicit_params = task.ext.implicit_params == null ? true : task.ext.implicit_params + def meta_params = task.ext.meta_params == null ? true : task.ext.meta_params // Dump parameters to yaml file. // Using a YAML file over using the CLI params because @@ -74,10 +84,10 @@ process QUARTONOTEBOOK { export XDG_DATA_HOME="./.xdg_data_home" # Set parallelism for BLAS/MKL etc. to avoid over-booking of resources - export MKL_NUM_THREADS="$task.cpus" - export OPENBLAS_NUM_THREADS="$task.cpus" - export OMP_NUM_THREADS="$task.cpus" - export NUMBA_NUM_THREADS="$task.cpus" + export MKL_NUM_THREADS="${task.cpus}" + export OPENBLAS_NUM_THREADS="${task.cpus}" + export OMP_NUM_THREADS="${task.cpus}" + export NUMBA_NUM_THREADS="${task.cpus}" # Render notebook quarto render \\ diff --git a/modules/nf-core/quartonotebook/tests/main.nf.test b/modules/nf-core/quartonotebook/tests/main.nf.test index 4fa0ac22001..8d49678b7eb 100644 --- a/modules/nf-core/quartonotebook/tests/main.nf.test +++ b/modules/nf-core/quartonotebook/tests/main.nf.test @@ -57,11 +57,11 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot( - process.out.versions, process.out.artifacts, process.out.params_yaml, + path(process.out.versions.get(0)).yaml, ).match() }, - { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } } + { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } }, ) } @@ -116,11 +116,11 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot( - process.out.versions, process.out.artifacts, process.out.params_yaml, + path(process.out.versions.get(0)).yaml, ).match() }, - { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } } + { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } }, ) } diff --git a/modules/nf-core/quartonotebook/tests/main.nf.test.snap b/modules/nf-core/quartonotebook/tests/main.nf.test.snap index f0f04cb5ec8..e9b992bc9bf 100644 --- a/modules/nf-core/quartonotebook/tests/main.nf.test.snap +++ b/modules/nf-core/quartonotebook/tests/main.nf.test.snap @@ -28,7 +28,7 @@ ], "5": [ - "versions.yml:md5,93481281b24bb1b44ecc4387e0957a0e" + "versions.yml:md5,ffc26b6a27e3d215b616594faba5c2a9" ], "artifacts": [ @@ -56,15 +56,15 @@ ], "versions": [ - "versions.yml:md5,93481281b24bb1b44ecc4387e0957a0e" + "versions.yml:md5,ffc26b6a27e3d215b616594faba5c2a9" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-02-09T11:06:33.408525" + "timestamp": "2024-11-16T13:53:41.964249" }, "test notebook - [qmd:r]": { "content": [ @@ -74,7 +74,7 @@ { "id": "test" }, - "test.html:md5,f09282296a5eee0154665975d842c07e" + "test.html:md5,b9b82aa1595b9987a5b34cb7355daa5f" ] ], "1": [ @@ -95,7 +95,7 @@ ], "5": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ], "artifacts": [ @@ -108,7 +108,7 @@ { "id": "test" }, - "test.html:md5,f09282296a5eee0154665975d842c07e" + "test.html:md5,b9b82aa1595b9987a5b34cb7355daa5f" ] ], "notebook": [ @@ -123,21 +123,18 @@ ], "versions": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-02-09T11:05:50.985424" + "timestamp": "2024-11-16T13:52:40.074638" }, "test notebook - parametrized - [qmd:python]": { "content": [ - [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" - ], [ [ { @@ -153,13 +150,19 @@ }, "params.yml:md5,efd62bc975f429e8749ba787a93042dd" ] - ] + ], + { + "QUARTONOTEBOOK": { + "quarto": "1.5.57", + "papermill": "2.4.0" + } + } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-02-09T09:21:18.194591" + "timestamp": "2024-11-16T14:11:57.879957" }, "test notebook - parametrized - [rmd]": { "content": [ @@ -169,7 +172,7 @@ { "id": "test" }, - "test.html:md5,2b2026646ed8b59d49fdcbd54cb3a463" + "test.html:md5,423772071657c5d3c89493fd4e255f48" ] ], "1": [ @@ -200,7 +203,7 @@ ], "5": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ], "artifacts": [ [ @@ -218,7 +221,7 @@ { "id": "test" }, - "test.html:md5,2b2026646ed8b59d49fdcbd54cb3a463" + "test.html:md5,423772071657c5d3c89493fd4e255f48" ] ], "notebook": [ @@ -238,15 +241,15 @@ ] ], "versions": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-02-09T11:06:25.046249" + "timestamp": "2024-11-16T13:53:28.301993" }, "test notebook - parametrized - [ipynb]": { "content": [ @@ -256,7 +259,7 @@ { "id": "test" }, - "test.html:md5,d7378ec0d1fd83b44424a68bf03a8fc3" + "test.html:md5,69914f2a054bb4b4a8e953fef551883e" ] ], "1": [ @@ -282,7 +285,7 @@ ], "5": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ], "artifacts": [ @@ -295,7 +298,7 @@ { "id": "test" }, - "test.html:md5,d7378ec0d1fd83b44424a68bf03a8fc3" + "test.html:md5,69914f2a054bb4b4a8e953fef551883e" ] ], "notebook": [ @@ -315,33 +318,36 @@ ] ], "versions": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-02-09T11:06:30.278412" + "timestamp": "2024-11-16T13:53:36.84715" }, "test notebook - [qmd:python]": { "content": [ - [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" - ], [ ], [ - ] + ], + { + "QUARTONOTEBOOK": { + "quarto": "1.5.57", + "papermill": "2.4.0" + } + } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-02-09T09:21:00.324109" + "timestamp": "2024-11-16T14:11:31.953539" }, "test notebook - parametrized - [qmd:r]": { "content": [ @@ -351,7 +357,7 @@ { "id": "test" }, - "test.html:md5,a25cdff28851a163d28669d4e62655af" + "test.html:md5,398d97ba8c2aca840fda4a064b1f037b" ] ], "1": [ @@ -382,7 +388,7 @@ ], "5": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ], "artifacts": [ [ @@ -400,7 +406,7 @@ { "id": "test" }, - "test.html:md5,a25cdff28851a163d28669d4e62655af" + "test.html:md5,398d97ba8c2aca840fda4a064b1f037b" ] ], "notebook": [ @@ -420,14 +426,14 @@ ] ], "versions": [ - "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + "versions.yml:md5,cb29d1dcc3f1531b5b219f5147739463" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-02-09T11:06:08.013103" + "timestamp": "2024-11-16T13:53:06.685421" } } \ No newline at end of file