simplify

docker/Rscript/Dockerfile

@@ -1,17 +1,20 @@
-FROM rocker/r-ver:4.4.1
+FROM rocker/r-ver:4.4.2
 
-RUN apt-get -y update && apt-get install -y libnlopt-dev zlib1g-dev \
+RUN apt-get -y update && apt-get upgrade -y
+RUN apt-get install -y build-essential cmake
+RUN apt-get install -y libnlopt-dev zlib1g-dev \
   libzmq3-dev libharfbuzz-dev libfribidi-dev libfreetype6-dev libpng-dev \
-  libtiff5-dev libjpeg-dev build-essential libcurl4-gnutls-dev libxml2-dev \ 
-  libssl-dev libfontconfig1-dev cmake \
+  libtiff5-dev libjpeg-dev libcurl4-gnutls-dev libxml2-dev \ 
+  libssl-dev libfontconfig1-dev \
   && rm -rf /var/lib/apt/lists/*
 
 ARG REPO=http://cran.us.r-project.org
 
 RUN install2.r -s -e -r ${REPO} remotes
 RUN Rscript -e "remotes::install_github('nathan-russell/hashmap')"
 RUN install2.r -s -e -r ${REPO} igraph ggplot2
-RUN install2.r -s -e -r ${REPO} pbkrtest lme4 pbkrtest lme4
+RUN install2.r -s -e -r ${REPO} lme4
+RUN install2.r -s -e -r ${REPO} pbkrtest
 RUN install2.r -s -e -r ${REPO} ggpubr cowplot reshape2 nloptr
 
 RUN rm -rf /tmp/downloaded_packages \

main.nf

@@ -16,7 +16,7 @@ ExOrthist pipeline for Bioinformatics Core @ CRG
 ===========================================================
 */
 
-nextflow.enable.dsl=2
+nextflow.enable.dsl = 2
 
 // if( !workflow.resume ) {
 //     println "Removing the output folder"
@@ -28,7 +28,8 @@ WORKFLOWS='./workflows/'
 
 include { PIPELINE_COMPLETION; PIPELINE_INITIALISATION } from "${LOCAL_SUBWORKFLOWS}/util.nf"
 
-include { MAIN; PLOT } from "${WORKFLOWS}/plot.nf"
+include { MAIN } from "${WORKFLOWS}/main.nf"
+include { PLOT } from "${WORKFLOWS}/plot.nf"
 
 
 workflow {
@@ -58,5 +59,4 @@ workflow {
         params.email,
         params.hook_url
     )
-
 }

modules/local/exorthist/check_input.nf

@@ -1,6 +1,6 @@
 process CHECK_INPUT {
     tag "Checking input files"
-    publishDir "${params.output}", mode: 'copy'
+    label 'publish'
 
     input:
     path evodisfile

modules/local/exorthist/collapse_matches.nf

@@ -1,5 +1,5 @@
 process COLLAPSE_OVERLAPPING_MATCHES {
-    publishDir "${params.output}/", mode: "copy"
+    label 'publish'
 
     input:
     path scores

modules/local/exorthist/format_output.nf

@@ -1,5 +1,5 @@
 process FORMAT_EX_CLUSTERS_OUTPUT {
-    publishDir "${params.output}/", mode: 'copy'
+    label 'publish'
 
     input:
     path ex_clusters

modules/local/exorthist/generate_annotations.nf

@@ -1,7 +1,7 @@
 process GENERATE_ANNOTATIONS {
     tag { genomeid }
     label 'big_cpus'
-    publishDir "${params.output}/", mode: 'copy'
+    label 'publish'
 
     input:
     tuple val(genomeid), path(genome), path(annotation)

modules/local/exorthist/join_matches.nf

@@ -1,5 +1,5 @@
 process JOIN_FILTERED_EX_MATCHES {
-    publishDir "${params.output}/", mode: 'copy'
+    label 'publish'
 
     input:
     path "filtered_best_scored-*"

modules/local/exorthist/merge_aligns.nf

@@ -1,6 +1,6 @@
 process MERGE_PROT_EX_INT_ALN_INFO {
     tag { comp_id }
-    label 'incr_time_cpus'
+    label 'aligners'
 
     stageInMode 'copy'
 

modules/local/exorthist/plot/plot.nf

@@ -2,7 +2,6 @@ process PLOT_EXINT {
     tag "${my_geneID}"
     label 'rscript'
     // containerOptions '-B $PWD:/tmp' # TODO: To consider if this should be moved to nextflow.config
-    publishDir "${params.output_plot}", mode: 'copy'
 
     input:
     val(my_geneID)

modules/local/exorthist/realign_pairs.nf

@@ -1,5 +1,5 @@
 process REALIGN_EX_PAIRS {
-    label 'incr_time_cpus'
+    label 'aligners'
 
     input:
     path blosumfile

modules/local/exorthist/recluster_exs.nf

@@ -1,6 +1,6 @@
 process RECLUSTER_EXS_BY_SPECIES_PAIR {
-    publishDir "${params.output}/reclustering", mode: 'copy'
     label 'pandas'
+    label 'publish_reclustering'
     tag { combid }
 
     input:

modules/local/exorthist/recluster_genes.nf

@@ -1,6 +1,6 @@
 process RECLUSTER_GENES_BY_SPECIES_PAIR {
-    publishDir "${params.output}/reclustering", mode: 'copy'
     label 'pandas'
+    label 'publish_reclustering'
     tag { combid }
 
     input:

modules/local/exorthist/score_matches.nf

@@ -1,6 +1,6 @@
 process SCORE_EX_MATCHES {
     tag { comp_id }
-    label 'big_mem_retry'
+    label 'big_mem'
 
     storeDir "${params.output}/${comp_id}"
 

nextflow.config

@@ -16,7 +16,7 @@ manifest {
 }
 
 plugins {
-  id 'nf-schema@2.1.2'
+  id 'nf-schema@2.2.0'
 }
 
 validation {
@@ -91,6 +91,8 @@ params {
     custom_config_base         = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
     config_profile_contact     = null
     config_profile_url         = null
+
+    publish_dir_mode           = 'copy'
 }
 
 process {
@@ -102,37 +104,28 @@ process {
     scratch = false
     container = 'perl:5.24-threaded-buster'
 
-    withLabel: incr_time_cpus {
+    withLabel: aligners {
        errorStrategy = 'retry'
        memory = '5G'
        cpus = 2
        time = {6.h * task.attempt * task.attempt}
        maxRetries = 4
        container = 'biocorecrg/exorthist_aligners:2.0.0'
     }
+
     withLabel: big_mem {
-        cpus = 1
-        memory = '5G'
-    }
-    withLabel: big_mem_retry {
        errorStrategy = 'retry'
        memory = {5.GB * task.attempt}
-       cpus = 1
        maxRetries = 4
     }
 
     withLabel: pandas {
      container = 'quay.io/biocontainers/pandas:1.5.2'
     }
 
-    withLabel: aligners {
-      cpus = 2
-      memory = '5G'
-      container = 'biocorecrg/exorthist_aligners:2.0.0'
-    }
 
     withLabel: rscript {
-     container = 'biocorecrg/exorthist_rscript:2.0.0'
+     container = 'biocorecrg/exorthist_rscript:2.0.1'
     }
 
 }
@@ -208,3 +201,26 @@ singularity.cacheDir    = "$projectDir/singularity"
 singularity.autoMounts  = false
 apptainer.cacheDir      = "$projectDir/apptainer"
 apptainer.autoMounts    = false
+
+// Publish stuff - Eventually to move to https://www.nextflow.io/docs/latest/workflow.html#publishing-outputs
+process {
+  withName: 'PLOT_EXINT' {
+      publishDir = [
+        path: { "${params.output_plot}" },
+        mode: params.publish_dir_mode
+      ]
+  }
+  withLabel: 'publish' {
+    publishDir = [
+         path: { "${params.output}" },
+         mode: params.publish_dir_mode
+    ]
+  }
+  withLabel: 'publish_reclustering' {
+    publishDir = [
+         path: { "${params.output}/reclustering" },
+         mode: params.publish_dir_mode
+    ]
+  }
+}
+

nextflow_schema.json

@@ -16,6 +16,22 @@
           "description": "Workflow to be run, by default the main one. Alternative is plot",
           "default": "main"
         },
+        "publish_dir_mode": {
+          "type": "string",
+          "default": "copy",
+          "description": "Method used to save pipeline results to output directory.",
+          "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.",
+          "fa_icon": "fas fa-copy",
+          "enum": [
+            "symlink",
+            "rellink",
+            "link",
+            "copy",
+            "copyNoFollow",
+            "move"
+          ],
+          "hidden": true
+        },
         "validate_params": {
           "type": "boolean",
           "description": "Boolean whether to validate parameters against the schema at runtime",