diff --git a/data/templates/callback.txt b/data/templates/callback.txt index b977fbc..3f5a71c 100644 --- a/data/templates/callback.txt +++ b/data/templates/callback.txt @@ -1,32 +1,14 @@ -# set up reporting job status to INTERVENE backend - -# need to escape starting curly bracket to prevent templating error -# (the comment above only makes sense in the original template file) -callback_exit_handler() \{ - # report the status of the job to the INTERVENE backend - exit_code=$? - - if [ $exit_code -eq 0 ]; then - body='\{"status": "COMPLETED", "pipelineId": "{name}", "outdir": ""}' - else - body='\{"status": "FAILED", "pipelineId": "{name}", "outdir": ""}' - fi - - # do a callback - url="https://dev.intervenegeneticscores.org/pipeline-manager/pipeline/csc/notify" - # escaping json is a pain - echo $body > callback.txt - - curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $CALLBACK_TOKEN" --data @callback.txt $url - - exit $exit_code -} - # run everything in "strict mode". error early, error often! set -euxo pipefail -trap callback_exit_handler EXIT +# set up reporting job status to INTERVENE backend + +# dependencies for workflow-monitor (python3.10 & requests library) +module load python-data/3.10-23.07 + +# run the monitor in the background +python3 {workflow_monitor_path} & # ------------------------------------------------------------------------------ diff --git a/data/templates/nxf.txt b/data/templates/nxf.txt index 8a07908..1124f3a 100644 --- a/data/templates/nxf.txt +++ b/data/templates/nxf.txt @@ -1,12 +1,12 @@ # time to start doing work! -# step 0. load dependencies +# load dependencies # (curl, jq, and singularity/apptainer are preloaded on Puhti) module load nextflow/22.10.1 module load parallel -# step 1: transfer data in parallel +# transfer data in parallel parallel --col-sep ' ' \ -a {work_dir}/{name}/transfer.txt \ java -Xmx256m \ @@ -23,7 +23,7 @@ set_tmpdir() \{ set_tmpdir -# step 2: run pgsc_calc and calculate some scores +# run pgsc_calc and calculate some scores! nextflow run {pgsc_calc_dir} -profile singularity \ -c {work_dir}/{name}/allas.config \ -params-file {work_dir}/{name}/params.json \ @@ -32,9 +32,8 @@ nextflow run {pgsc_calc_dir} -profile singularity \ --min_overlap 0.01 \ --max_cpus 40 \ --max_memory "32.GB" \ - --parallel - -# step 3: report job status to INTERVENE platform -# (automatically executed in callback_exit_handler by trap) + --parallel \ + -with-weblog http://localhost:8000 \ + -name {name} # ------------------------------------------------------------------------------ diff --git a/src/slurm/job.rs b/src/slurm/job.rs index 1a33ef0..a219086 100644 --- a/src/slurm/job.rs +++ b/src/slurm/job.rs @@ -157,6 +157,7 @@ struct NextflowContext { #[derive(Serialize)] struct CallbackContext { name: String, + workflow_monitor_path: String } /// Write nextflow parameters to working directory @@ -236,7 +237,9 @@ fn render_callback(param: &PipelineParam) -> Callback { let mut tt = TinyTemplate::new(); tt.add_template("callback", CALLBACK).expect("Template"); let name: &String = ¶m.id; - let context = CallbackContext { name: name.clone() }; + static WORKFLOW_MONITOR_PATH: &str = "/scratch/project_2004504/bwingfield/workflow-monitor/main.py"; + let context = CallbackContext { name: name.clone(), + workflow_monitor_path: WORKFLOW_MONITOR_PATH.to_string() }; Callback { content: tt.render("callback", &context).expect("Rendered callback") } }