Parameterized publishing of non-QC non-provenance files (#530)

* Parameterized publishing of non-QC non-provenance files * Added missing mode: * Add qc-files:inherit feature This allows the qc-files setting to "inherit" the value of the publish_dir_mode pipeline param. inherit is now the default value. --------- Co-authored-by: Jeremy Muhlich <[email protected]>
labsyspharm · Dec 21, 2023 · 9d0fb6f · 9d0fb6f
1 parent 049baed
commit 9d0fb6f
Show file tree

Hide file tree

Showing 14 changed files with 39 additions and 25 deletions.
diff --git a/config/defaults.yml b/config/defaults.yml
@@ -1,7 +1,7 @@
 workflow:
   start-at: registration
   stop-at: quantification
-  qc-files: copy
+  qc-files: inherit
   tma: false
   viz: false
   background: false

diff --git a/docs/parameters/workflow.md b/docs/parameters/workflow.md
@@ -188,12 +188,12 @@ workflow:
 ```
 
 ## `qc-files`
-  Whether QC files should be copied, moved or symbolically linked from work directories to the project directory
+  Whether QC files should be copied, moved, hard linked, or symbolically linked from work directories to the project directory. 'inherit' may be specified to use the value of the publish_dir_mode pipeline parameter.
 {: .fs-4}
 {: .fw-300}
 
-  * **Valid values:** `copy`, `move`, `symlink`
-  * **Default:** `copy`
+  * **Valid values:** `copy`, `move`, `link`, `symlink`, `inherit`
+  * **Default:** `inherit`
   * **Example:**
 
 ``` yaml
@@ -229,4 +229,4 @@ workflow:
 ``` yaml
 workflow:
   background-method: backsub
-```
+```
diff --git a/lib/mcmicro/Opts.groovy b/lib/mcmicro/Opts.groovy
@@ -88,7 +88,7 @@ static def cleanParams(pars, mspecs) {
 
     // Protected keywords
     def keywords = ['in', 'cont-pfx', 'roadie', 'workflow',
-        'options', 'modules', 'params']
+        'options', 'modules', 'params', 'publish_dir_mode']
 
     // Clean up the parameter list
     // Separate workflow parameters from module options
@@ -198,6 +198,10 @@ static def parseParams(gp, fns, fnw) {
         mcp.workflow.downstream.contains(it.name)
     }
 
+    // Implement qc-files=inherit which sets qc-files to the value of
+    // publish_dir_mode.
+    if(mcp.workflow['qc-files'] == 'inherit') mcp.workflow['qc-files'] = gp.publish_dir_mode
+
     mcp
 }
 

diff --git a/lib/worker.nf b/lib/worker.nf
@@ -28,9 +28,9 @@ process worker {
     tag "${module.name}-${task.index}"
 
     // Output files in the pre-configured output format (outfmt) and optional plots
-    publishDir "${pubDir}", mode: 'copy',
+    publishDir "${pubDir}", mode: "${params.publish_dir_mode}",
       pattern: "$outfmt", saveAs: {fn -> fnOut != '' ? fnOut : fn}
-    publishDir "${pubDir}", mode: 'copy', pattern: 'plots/**'
+    publishDir "${pubDir}", mode: "${params.publish_dir_mode}", pattern: 'plots/**'
 
     // QC
     publishDir "${Flow.QC(params.in, module.name)}", mode: "${mcp.workflow['qc-files']}",

diff --git a/modules/background.nf b/modules/background.nf
@@ -15,8 +15,10 @@ process backsub {
     // Specify the project subdirectory for writing the outputs to
     // The pattern: specification must match the output: files below
     // Subdirectory: background
-    publishDir "${params.in}/background", mode: 'copy', pattern: "*.ome.tif"
-    publishDir "${params.in}/background", mode: 'copy', pattern:'markers_bs.csv'
+    publishDir "${params.in}/background", mode: "${params.publish_dir_mode}",
+      pattern: "*.ome.tif"
+    publishDir "${params.in}/background", mode: "${params.publish_dir_mode}",
+      pattern:'markers_bs.csv'
 
     // Stores .command.sh and .command.log from the work directory
     //   to the project provenance

diff --git a/modules/dearray.nf b/modules/dearray.nf
@@ -6,7 +6,8 @@ process coreograph {
     container "${params.contPfx}${module.container}:${module.version}"
 
     // Output -- publish masks only, images need to be pyramidized
-    publishDir "${params.in}/dearray", mode: 'copy', pattern: '**mask.tif'
+    publishDir "${params.in}/dearray", mode: "${params.publish_dir_mode}",
+      pattern: '**mask.tif'
 
     // QC
     publishDir "${Flow.QC(params.in, module.name)}",
@@ -46,7 +47,7 @@ workflow dearray {
 
     // Pass the core images through palom to pyramidize them
     inputs = coreograph.out.cores.flatten()
-    roadie('pyramidize', inputs, '', true, "${params.in}/dearray", 'copy')
+    roadie('pyramidize', inputs, '', true, "${params.in}/dearray", "${params.publish_dir_mode}")
 
   emit:
     cores = roadie.out

diff --git a/modules/illumination.nf b/modules/illumination.nf
@@ -12,7 +12,8 @@ process illumination {
     container "${params.contPfx}${module.container}:${module.version}"
 
     // Output profiles
-    publishDir "${params.in}/illumination", mode: 'copy', pattern: '*.tif'
+    publishDir "${params.in}/illumination", mode: "${params.publish_dir_mode}",
+      pattern: '*.tif'
 
     // Provenance
     publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 

diff --git a/modules/quantification.nf b/modules/quantification.nf
@@ -4,7 +4,8 @@ process mcquant {
     container "${params.contPfx}${module.container}:${module.version}"
 
     // Output
-    publishDir "${params.in}/quantification", mode: 'copy', pattern: '*.csv'
+    publishDir "${params.in}/quantification", mode: "${params.publish_dir_mode}",
+      pattern: '*.csv'
 
     // Provenance
     publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 

diff --git a/modules/registration.nf b/modules/registration.nf
@@ -2,7 +2,8 @@ import mcmicro.*
 
 process ashlar {
     container "${params.contPfx}${module.container}:${module.version}"
-    publishDir "${params.in}/registration", mode: 'copy', pattern: '*.tif'
+    publishDir "${params.in}/registration", mode: "${params.publish_dir_mode}",
+      pattern: '*.tif'
 
     // Provenance
     publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 

diff --git a/modules/segmentation.nf b/modules/segmentation.nf
@@ -4,8 +4,8 @@ process s3seg {
     container "${params.contPfx}${module.container}:${module.version}"
 
     // Output
-    publishDir "${pubDir}/$tag",
-      mode: 'copy', pattern: '*/*.ome.tif', saveAs: {f -> file(f).name}
+    publishDir "${pubDir}/$tag", mode: "${params.publish_dir_mode}",
+      pattern: '*/*.ome.tif', saveAs: {f -> file(f).name}
 
     // QC
     publishDir "${Flow.QC(params.in, '/s3seg/' + tag)}",

diff --git a/modules/template.nf b/modules/template.nf
@@ -61,7 +61,8 @@ process snr {
     // The pattern: specification must match the output: files below
     // TODO: replace report with the desired output directory
     // TODO: replace the pattern to match the output: clause below
-    publishDir "${params.in}/report", mode: 'copy', pattern: "*.html"
+    publishDir "${params.in}/report", mode: "${params.publish_dir_mode}",
+      pattern: "*.html"
 
     // Stores .command.sh and .command.log from the work directory
     //   to the project provenance

diff --git a/modules/viz.nf b/modules/viz.nf
@@ -1,12 +1,10 @@
 import mcmicro.*
 
-include {roadie} from "$projectDir/roadie"
-
 process autominerva {
     container "${params.contPfx}${module.container}:${module.version}"
 
     // Output
-    publishDir "${params.in}/viz", mode: 'copy', pattern: "$tag/**"
+    publishDir "${params.in}/viz", mode: "${params.publish_dir_mode}", pattern: "$tag/**"
 
     // Provenance
     publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 

diff --git a/nextflow.config b/nextflow.config
@@ -1,6 +1,9 @@
 // Execution environment for miscellaneous tasks
 params.roadie = 'ghcr.io/labsyspharm/mcmicro:roadie-2023-10-25'
 
+// The mode for transfering output between work and project directories
+params.publish_dir_mode = 'copy'
+
 // Platform-specific profiles
 profiles {
 

diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -150,13 +150,15 @@
               },
               "qc-files": {
                   "type": "string",
-                  "description": "Should QC files should be copied, moved or symbolically linked from work directories to the project directory",
+                  "description": "Should QC files should be copied, moved, hard linked, or symbolically linked from work directories to the project directory. 'inherit' may be specified to use the value of the publish_dir_mode pipeline parameter.",
                   "enum": [
                       "copy",
                       "move",
-                      "symlink"
+                      "link",
+                      "symlink",
+                      "inherit",
                   ],
-                  "default": "copy"
+                  "default": "inherit"
               },
               "segmentation-channel": {
                   "type": "string",
@@ -219,4 +221,4 @@
           "$ref": "#/definitions/parameters_for_individual_modules"
       }
   ]
-}
+}