From 2626fe91003abd78599fade2f7ac9799ad6c6fac Mon Sep 17 00:00:00 2001 From: Toni Hermoso Pulido Date: Fri, 1 Nov 2024 21:42:23 +0100 Subject: [PATCH] simplify --- docker/Rscript/Dockerfile | 13 +++--- main.nf | 6 +-- modules/local/exorthist/check_input.nf | 2 +- modules/local/exorthist/collapse_matches.nf | 2 +- modules/local/exorthist/format_output.nf | 2 +- .../local/exorthist/generate_annotations.nf | 2 +- modules/local/exorthist/join_matches.nf | 2 +- modules/local/exorthist/merge_aligns.nf | 2 +- modules/local/exorthist/plot/plot.nf | 1 - modules/local/exorthist/realign_pairs.nf | 2 +- modules/local/exorthist/recluster_exs.nf | 2 +- modules/local/exorthist/recluster_genes.nf | 2 +- modules/local/exorthist/score_matches.nf | 2 +- nextflow.config | 42 +++++++++++++------ nextflow_schema.json | 16 +++++++ 15 files changed, 66 insertions(+), 32 deletions(-) diff --git a/docker/Rscript/Dockerfile b/docker/Rscript/Dockerfile index 206bf74..c6e0754 100644 --- a/docker/Rscript/Dockerfile +++ b/docker/Rscript/Dockerfile @@ -1,9 +1,11 @@ -FROM rocker/r-ver:4.4.1 +FROM rocker/r-ver:4.4.2 -RUN apt-get -y update && apt-get install -y libnlopt-dev zlib1g-dev \ +RUN apt-get -y update && apt-get upgrade -y +RUN apt-get install -y build-essential cmake +RUN apt-get install -y libnlopt-dev zlib1g-dev \ libzmq3-dev libharfbuzz-dev libfribidi-dev libfreetype6-dev libpng-dev \ - libtiff5-dev libjpeg-dev build-essential libcurl4-gnutls-dev libxml2-dev \ - libssl-dev libfontconfig1-dev cmake \ + libtiff5-dev libjpeg-dev libcurl4-gnutls-dev libxml2-dev \ + libssl-dev libfontconfig1-dev \ && rm -rf /var/lib/apt/lists/* ARG REPO=http://cran.us.r-project.org @@ -11,7 +13,8 @@ ARG REPO=http://cran.us.r-project.org RUN install2.r -s -e -r ${REPO} remotes RUN Rscript -e "remotes::install_github('nathan-russell/hashmap')" RUN install2.r -s -e -r ${REPO} igraph ggplot2 -RUN install2.r -s -e -r ${REPO} pbkrtest lme4 pbkrtest lme4 +RUN install2.r -s -e -r ${REPO} lme4 +RUN install2.r -s -e -r ${REPO} pbkrtest RUN install2.r -s -e -r ${REPO} ggpubr cowplot reshape2 nloptr RUN rm -rf /tmp/downloaded_packages \ diff --git a/main.nf b/main.nf index e5af91c..0b87e61 100644 --- a/main.nf +++ b/main.nf @@ -16,7 +16,7 @@ ExOrthist pipeline for Bioinformatics Core @ CRG =========================================================== */ -nextflow.enable.dsl=2 +nextflow.enable.dsl = 2 // if( !workflow.resume ) { // println "Removing the output folder" @@ -28,7 +28,8 @@ WORKFLOWS='./workflows/' include { PIPELINE_COMPLETION; PIPELINE_INITIALISATION } from "${LOCAL_SUBWORKFLOWS}/util.nf" -include { MAIN; PLOT } from "${WORKFLOWS}/plot.nf" +include { MAIN } from "${WORKFLOWS}/main.nf" +include { PLOT } from "${WORKFLOWS}/plot.nf" workflow { @@ -58,5 +59,4 @@ workflow { params.email, params.hook_url ) - } diff --git a/modules/local/exorthist/check_input.nf b/modules/local/exorthist/check_input.nf index 59db1ed..3a84dc9 100644 --- a/modules/local/exorthist/check_input.nf +++ b/modules/local/exorthist/check_input.nf @@ -1,6 +1,6 @@ process CHECK_INPUT { tag "Checking input files" - publishDir "${params.output}", mode: 'copy' + label 'publish' input: path evodisfile diff --git a/modules/local/exorthist/collapse_matches.nf b/modules/local/exorthist/collapse_matches.nf index 74ef11b..d39052e 100644 --- a/modules/local/exorthist/collapse_matches.nf +++ b/modules/local/exorthist/collapse_matches.nf @@ -1,5 +1,5 @@ process COLLAPSE_OVERLAPPING_MATCHES { - publishDir "${params.output}/", mode: "copy" + label 'publish' input: path scores diff --git a/modules/local/exorthist/format_output.nf b/modules/local/exorthist/format_output.nf index 40ab620..5a8e7ab 100644 --- a/modules/local/exorthist/format_output.nf +++ b/modules/local/exorthist/format_output.nf @@ -1,5 +1,5 @@ process FORMAT_EX_CLUSTERS_OUTPUT { - publishDir "${params.output}/", mode: 'copy' + label 'publish' input: path ex_clusters diff --git a/modules/local/exorthist/generate_annotations.nf b/modules/local/exorthist/generate_annotations.nf index 32efd15..b014615 100644 --- a/modules/local/exorthist/generate_annotations.nf +++ b/modules/local/exorthist/generate_annotations.nf @@ -1,7 +1,7 @@ process GENERATE_ANNOTATIONS { tag { genomeid } label 'big_cpus' - publishDir "${params.output}/", mode: 'copy' + label 'publish' input: tuple val(genomeid), path(genome), path(annotation) diff --git a/modules/local/exorthist/join_matches.nf b/modules/local/exorthist/join_matches.nf index fd3c99d..d5001e4 100644 --- a/modules/local/exorthist/join_matches.nf +++ b/modules/local/exorthist/join_matches.nf @@ -1,5 +1,5 @@ process JOIN_FILTERED_EX_MATCHES { - publishDir "${params.output}/", mode: 'copy' + label 'publish' input: path "filtered_best_scored-*" diff --git a/modules/local/exorthist/merge_aligns.nf b/modules/local/exorthist/merge_aligns.nf index 5546ffa..95defa6 100644 --- a/modules/local/exorthist/merge_aligns.nf +++ b/modules/local/exorthist/merge_aligns.nf @@ -1,6 +1,6 @@ process MERGE_PROT_EX_INT_ALN_INFO { tag { comp_id } - label 'incr_time_cpus' + label 'aligners' stageInMode 'copy' diff --git a/modules/local/exorthist/plot/plot.nf b/modules/local/exorthist/plot/plot.nf index 2e10857..b089b4c 100644 --- a/modules/local/exorthist/plot/plot.nf +++ b/modules/local/exorthist/plot/plot.nf @@ -2,7 +2,6 @@ process PLOT_EXINT { tag "${my_geneID}" label 'rscript' // containerOptions '-B $PWD:/tmp' # TODO: To consider if this should be moved to nextflow.config - publishDir "${params.output_plot}", mode: 'copy' input: val(my_geneID) diff --git a/modules/local/exorthist/realign_pairs.nf b/modules/local/exorthist/realign_pairs.nf index 3efad32..e2c5970 100644 --- a/modules/local/exorthist/realign_pairs.nf +++ b/modules/local/exorthist/realign_pairs.nf @@ -1,5 +1,5 @@ process REALIGN_EX_PAIRS { - label 'incr_time_cpus' + label 'aligners' input: path blosumfile diff --git a/modules/local/exorthist/recluster_exs.nf b/modules/local/exorthist/recluster_exs.nf index ae42fc0..75d1be3 100644 --- a/modules/local/exorthist/recluster_exs.nf +++ b/modules/local/exorthist/recluster_exs.nf @@ -1,6 +1,6 @@ process RECLUSTER_EXS_BY_SPECIES_PAIR { - publishDir "${params.output}/reclustering", mode: 'copy' label 'pandas' + label 'publish_reclustering' tag { combid } input: diff --git a/modules/local/exorthist/recluster_genes.nf b/modules/local/exorthist/recluster_genes.nf index 2f9a8c8..799a1fc 100644 --- a/modules/local/exorthist/recluster_genes.nf +++ b/modules/local/exorthist/recluster_genes.nf @@ -1,6 +1,6 @@ process RECLUSTER_GENES_BY_SPECIES_PAIR { - publishDir "${params.output}/reclustering", mode: 'copy' label 'pandas' + label 'publish_reclustering' tag { combid } input: diff --git a/modules/local/exorthist/score_matches.nf b/modules/local/exorthist/score_matches.nf index 08bc653..899d953 100644 --- a/modules/local/exorthist/score_matches.nf +++ b/modules/local/exorthist/score_matches.nf @@ -1,6 +1,6 @@ process SCORE_EX_MATCHES { tag { comp_id } - label 'big_mem_retry' + label 'big_mem' storeDir "${params.output}/${comp_id}" diff --git a/nextflow.config b/nextflow.config index 5d41356..20b4408 100644 --- a/nextflow.config +++ b/nextflow.config @@ -16,7 +16,7 @@ manifest { } plugins { - id 'nf-schema@2.1.2' + id 'nf-schema@2.2.0' } validation { @@ -91,6 +91,8 @@ params { custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" config_profile_contact = null config_profile_url = null + + publish_dir_mode = 'copy' } process { @@ -102,7 +104,7 @@ process { scratch = false container = 'perl:5.24-threaded-buster' - withLabel: incr_time_cpus { + withLabel: aligners { errorStrategy = 'retry' memory = '5G' cpus = 2 @@ -110,14 +112,10 @@ process { maxRetries = 4 container = 'biocorecrg/exorthist_aligners:2.0.0' } + withLabel: big_mem { - cpus = 1 - memory = '5G' - } - withLabel: big_mem_retry { errorStrategy = 'retry' memory = {5.GB * task.attempt} - cpus = 1 maxRetries = 4 } @@ -125,14 +123,9 @@ process { container = 'quay.io/biocontainers/pandas:1.5.2' } - withLabel: aligners { - cpus = 2 - memory = '5G' - container = 'biocorecrg/exorthist_aligners:2.0.0' - } withLabel: rscript { - container = 'biocorecrg/exorthist_rscript:2.0.0' + container = 'biocorecrg/exorthist_rscript:2.0.1' } } @@ -208,3 +201,26 @@ singularity.cacheDir = "$projectDir/singularity" singularity.autoMounts = false apptainer.cacheDir = "$projectDir/apptainer" apptainer.autoMounts = false + +// Publish stuff - Eventually to move to https://www.nextflow.io/docs/latest/workflow.html#publishing-outputs +process { + withName: 'PLOT_EXINT' { + publishDir = [ + path: { "${params.output_plot}" }, + mode: params.publish_dir_mode + ] + } + withLabel: 'publish' { + publishDir = [ + path: { "${params.output}" }, + mode: params.publish_dir_mode + ] + } + withLabel: 'publish_reclustering' { + publishDir = [ + path: { "${params.output}/reclustering" }, + mode: params.publish_dir_mode + ] + } +} + diff --git a/nextflow_schema.json b/nextflow_schema.json index 1f481d6..1e5787b 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -16,6 +16,22 @@ "description": "Workflow to be run, by default the main one. Alternative is plot", "default": "main" }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], + "hidden": true + }, "validate_params": { "type": "boolean", "description": "Boolean whether to validate parameters against the schema at runtime",