Merge pull request #273 from jonasscheid/dev

Optimize resource allocations
nf-core · Jun 29, 2023 · 42d61d1 · 42d61d1
2 parents 0401fb4 + 49e8c37
commit 42d61d1
Show file tree

Hide file tree

Showing 20 changed files with 32 additions and 24 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added OpenMS FileFilter to clean mzml after parsing to remove artifacts like empty spectra or precursors with charge 0
 - Made file extension check case insensitive
 - Added option to provide a default comet parameters file
+- Optimize resource allocations
 
 ### `Fixed`
 

diff --git a/bin/ms2pip_cli.py b/bin/ms2pip_cli.py
@@ -184,6 +184,11 @@ def generate_params_config(
     help="List of fixed modifications",
 )
 @click.option("--add_pearson", is_flag=True, help="add pearson spectrum simliartity")
+@click.option(
+    "--num_cpus",
+    type=int,
+    help="number of cpus to use",
+)
 def main(
     input_idxml: str,
     input_mzml: str,
@@ -195,6 +200,7 @@ def main(
     variable_mods: str,
     fixed_mods: str,
     add_pearson: bool,
+    num_cpus: int,
 ):
     LOG.info("Parse idXML")
     protein_ids, peptide_ids = parse_idxml(input_idxml)
@@ -206,12 +212,7 @@ def main(
     scan_nr_seq_to_corr = {}
     for hit_idx in range(num_hits):  # number of hits to consider
         df_peprec, scan_nr_to_seq = peptide_ids_to_peprec_dataframe(peptide_ids, hit_idx)
-        ms2pip = MS2PIP(
-            pep_file=df_peprec,
-            spec_file=input_mzml,
-            params=params,
-            return_results=True,
-        )
+        ms2pip = MS2PIP(pep_file=df_peprec, spec_file=input_mzml, params=params, return_results=True, num_cpu=num_cpus)
         predictions = ms2pip.run()
         correlation_df = get_complete_spectrum_correlation(predictions, "pearson")
 

diff --git a/conf/base.config b/conf/base.config
@@ -56,4 +56,9 @@ process {
     withName:CUSTOM_DUMPSOFTWAREVERSIONS {
         cache = false
     }
+    withName:TDF2MZML {
+        cpus   = { check_max( 10    * task.attempt, 'cpus'    ) }
+        memory = { check_max( 10.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 16.h  * task.attempt, 'time'    ) }
+    }
 }
diff --git a/modules/local/ms2pip.nf b/modules/local/ms2pip.nf
@@ -1,6 +1,6 @@
 process MS2PIP {
     tag "$meta.id"
-    label 'process_medium'
+    label 'process_low'
 
     conda "bioconda::ms2pip=3.11.0 bioconda::pyopenms=2.9.1"
     container 'ghcr.io/jonasscheid/mhcquant:ms2pip'
@@ -28,7 +28,8 @@ process MS2PIP {
         --model_name ${params.ms2pip_model_name} \\
         --fragment_error $fragment_error \\
         --variable_mods '${params.variable_mods}' \\
-        --fixed_mods '${params.fixed_mods}'
+        --fixed_mods '${params.fixed_mods}' \\
+        --num_cpus ${task.cpus}
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

diff --git a/modules/local/openms_decoydatabase.nf b/modules/local/openms_decoydatabase.nf
@@ -1,6 +1,6 @@
 process OPENMS_DECOYDATABASE {
     tag "$meta.id"
-    label 'process_medium'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_falsediscoveryrate.nf b/modules/local/openms_falsediscoveryrate.nf
@@ -1,6 +1,6 @@
 process OPENMS_FALSEDISCOVERYRATE  {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_featurelinkerunlabeledkd.nf b/modules/local/openms_featurelinkerunlabeledkd.nf
@@ -1,6 +1,6 @@
 process OPENMS_FEATURELINKERUNLABELEDKD {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms-thirdparty=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_idconflictresolver.nf b/modules/local/openms_idconflictresolver.nf
@@ -1,6 +1,6 @@
 process OPENMS_IDCONFLICTRESOLVER {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_idfilter.nf b/modules/local/openms_idfilter.nf
@@ -1,6 +1,6 @@
 process OPENMS_IDFILTER {
     tag "$meta.id"
-    label 'process_medium'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_idmerger.nf b/modules/local/openms_idmerger.nf
@@ -1,6 +1,6 @@
 process OPENMS_IDMERGER {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_mapaligneridentification.nf b/modules/local/openms_mapaligneridentification.nf
@@ -1,6 +1,6 @@
 process OPENMS_MAPALIGNERIDENTIFICATION {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_maprttransformer.nf b/modules/local/openms_maprttransformer.nf
@@ -1,6 +1,6 @@
 process OPENMS_MAPRTTRANSFORMER {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_mztabexporter.nf b/modules/local/openms_mztabexporter.nf
@@ -1,6 +1,6 @@
 process OPENMS_MZTABEXPORTER {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_peptideindexer.nf b/modules/local/openms_peptideindexer.nf
@@ -1,6 +1,6 @@
 process OPENMS_PEPTIDEINDEXER {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_percolatoradapter.nf b/modules/local/openms_percolatoradapter.nf
@@ -1,6 +1,6 @@
 process OPENMS_PERCOLATORADAPTER {
     tag "$meta.id"
-    label 'process_high'
+    label 'process_low'
 
     conda (params.enable_conda ? "bioconda::openms-thirdparty=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_rtmodel.nf b/modules/local/openms_rtmodel.nf
@@ -1,6 +1,6 @@
 process OPENMS_RTMODEL {
     tag "$meta.id"
-    label 'process_medium'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_rtpredict.nf b/modules/local/openms_rtpredict.nf
@@ -1,6 +1,6 @@
 process OPENMS_RTPREDICT {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms-thirdparty=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/openms_textexporter.nf b/modules/local/openms_textexporter.nf
@@ -1,6 +1,6 @@
 process OPENMS_TEXTEXPORTER {
     tag "$meta.id"
-    label 'process_low'
+    label 'process_single'
 
     conda (params.enable_conda ? "bioconda::openms=2.8.0" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?

diff --git a/modules/local/tdf2mzml.nf b/modules/local/tdf2mzml.nf
@@ -1,6 +1,6 @@
 process TDF2MZML {
     tag "$meta.id"
-    label 'process_single'
+    //label 'process_medium'
 
     container "docker.io/mfreitas/tdf2mzml"
 

diff --git a/modules/local/thermorawfileparser.nf b/modules/local/thermorawfileparser.nf
@@ -1,6 +1,6 @@
 process THERMORAWFILEPARSER {
     tag "$meta.id"
-    label 'process_medium'
+    label 'process_low'
 
     conda (params.enable_conda ? "bioconda::thermorawfileparser=1.4.2" : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?