From c6df368266724161538986787887ea028241c8ff Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 20 Sep 2024 13:12:30 +0200 Subject: [PATCH] core: add new tool externalExecutor object to Job, #TASK-6445 --- .../analysis/binary/BinaryExecutor.java | 8 ++-- .../analysis/tools/OpenCgaToolScopeStudy.java | 13 +++--- .../analysis/workflow/NextFlowExecutor.java | 7 ++- .../main/executors/JobsCommandExecutor.java | 1 + .../cli/main/options/JobsCommandOptions.java | 8 ++-- .../opencga/catalog/db/api/JobDBAdaptor.java | 1 + .../catalog/db/mongodb/JobMongoDBAdaptor.java | 3 +- .../opencga/core/api/FieldConstants.java | 2 + .../opencga/core/models/job/ToolInfo.java | 45 ++++++++++++------- .../core/models/job/ToolInfoExecutor.java | 42 +++++++++++++++++ 10 files changed, 95 insertions(+), 35 deletions(-) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfoExecutor.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/binary/BinaryExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/binary/BinaryExecutor.java index a582723a425..a4f162f6490 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/binary/BinaryExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/binary/BinaryExecutor.java @@ -2,7 +2,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.DockerUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.catalog.utils.InputFileUtils; @@ -12,6 +11,7 @@ import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.job.JobRunDockerParams; import org.opencb.opencga.core.models.job.JobRunParams; +import org.opencb.opencga.core.models.job.ToolInfoExecutor; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; import org.slf4j.Logger; @@ -60,13 +60,11 @@ protected void check() throws Exception { } // Update job tags and attributes - ObjectMap attributes = new ObjectMap() - .append("DOCKER_ID", runParams.getDocker().getId()) - .append("DOCKER_TAG", runParams.getDocker().getTag()); + ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(runParams.getDocker().getId(), runParams.getDocker().getTag()); List tags = new LinkedList<>(); tags.add(ID); tags.add(this.dockerImage); - updateJobInformation(tags, attributes); + updateJobInformation(tags, toolInfoExecutor); // Build input bindings this.inputBindings = new LinkedList<>(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaToolScopeStudy.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaToolScopeStudy.java index 39c4282109d..9e49423c7eb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaToolScopeStudy.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaToolScopeStudy.java @@ -18,10 +18,11 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.JobDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.job.JobUpdateParams; +import org.opencb.opencga.core.models.job.ToolInfoExecutor; import java.util.List; @@ -55,11 +56,11 @@ protected final String getStudyFqn() throws CatalogException { getToken()).first().getFqn(); } - protected void updateJobInformation(List tags, ObjectMap attributes) throws CatalogException { - JobUpdateParams updateParams = new JobUpdateParams() - .setTags(tags) - .setAttributes(attributes); - catalogManager.getJobManager().update(getStudyFqn(), getJobId(), updateParams, QueryOptions.empty(), token); + protected void updateJobInformation(List tags, ToolInfoExecutor executor) throws CatalogException { + ObjectMap params = new ObjectMap() + .append(JobDBAdaptor.QueryParams.TAGS.key(), tags) + .append(JobDBAdaptor.QueryParams.TOOL_EXTERNAL_EXECUTOR.key(), executor); + catalogManager.getJobManager().update(getStudyFqn(), getJobId(), params, QueryOptions.empty(), token); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java index a6cd07e4649..9fe07de30ae 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java @@ -19,6 +19,7 @@ import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.job.ToolInfoExecutor; import org.opencb.opencga.core.models.workflow.NextFlowRunParams; import org.opencb.opencga.core.models.workflow.Workflow; import org.opencb.opencga.core.models.workflow.WorkflowScript; @@ -90,9 +91,7 @@ protected void check() throws Exception { } // Update job tags and attributes - ObjectMap attributes = new ObjectMap() - .append("WORKFLOW_ID", workflow.getManager().getId()) - .append("WORKFLOW_VERSION", workflow.getManager().getVersion()); + ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(workflow.getManager().getId().name(), workflow.getManager().getVersion()); Set tags = new HashSet<>(); tags.add(ID); tags.add(workflow.getManager().getId().name()); @@ -101,7 +100,7 @@ protected void check() throws Exception { if (CollectionUtils.isNotEmpty(workflow.getTags())) { tags.addAll(workflow.getTags()); } - updateJobInformation(new ArrayList<>(tags), attributes); + updateJobInformation(new ArrayList<>(tags), toolInfoExecutor); this.inputBindings = new LinkedList<>(); if (MapUtils.isNotEmpty(nextflowParams.getParams())) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index e30abec6bcb..ba48c6db8f9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -29,6 +29,7 @@ import org.opencb.opencga.core.models.job.JobTop; import org.opencb.opencga.core.models.job.JobUpdateParams; import org.opencb.opencga.core.models.job.ToolInfo; +import org.opencb.opencga.core.models.job.ToolInfoExecutor; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; import org.opencb.opencga.core.tools.annotations.Tool; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java index 9f7e2928f6c..2dc71a214e3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java @@ -120,16 +120,16 @@ public class CreateCommandOptions { public String toolId; @Parameter(names = {"--tool-description"}, description = "Users may provide a description for the entry.", required = false, arity = 1) - public String toolDescription = "No description available"; + public String toolDescription; @Parameter(names = {"--tool-scope"}, description = "Tool info scope can have the values GLOBAL, PROJECT and STUDY.", required = false, arity = 1) - public String toolScope = "No description available"; + public String toolScope; @Parameter(names = {"--tool-type"}, description = "Tool info type can have the values OPERATION and ANALYSIS.", required = false, arity = 1) - public String toolType = "No description available"; + public String toolType; @Parameter(names = {"--tool-resource"}, description = "Tool info resource can have the values AUDIT, USER, PROJECT, STUDY, FILE, SAMPLE, JOB, INDIVIDUAL, COHORT, DISEASE_PANEL, FAMILY, CLINICAL_ANALYSIS, INTERPRETATION, VARIANT, ALIGNMENT, CLINICAL, EXPRESSION, RGA and FUNCTIONAL.", required = false, arity = 1) - public String toolResource = "No description available"; + public String toolResource; @Parameter(names = {"--priority"}, description = "The body web service priority parameter", required = false, arity = 1) public String priority; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index a6016da7441..a8f1e717cae 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -113,6 +113,7 @@ enum QueryParams implements QueryParam { TOOL("tool", OBJECT, ""), TOOL_ID("tool.id", TEXT, ""), TOOL_TYPE("tool.type", TEXT, ""), + TOOL_EXTERNAL_EXECUTOR("tool.externalExecutor", OBJECT, ""), PRIORITY("priority", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 53c9a5f157e..cb59d9dca0b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -441,7 +441,8 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO Enums.Priority.getPriority(parameters.getString(QueryParams.PRIORITY.key())).getValue()); } - String[] acceptedObjectParams = {QueryParams.EXECUTION.key(), QueryParams.STUDY.key(), QueryParams.INTERNAL_STATUS.key()}; + String[] acceptedObjectParams = {QueryParams.EXECUTION.key(), QueryParams.STUDY.key(), QueryParams.INTERNAL_STATUS.key(), + QueryParams.TOOL_EXTERNAL_EXECUTOR.key()}; filterObjectParams(parameters, document.getSet(), acceptedObjectParams); if (document.getSet().containsKey(QueryParams.STUDY.key())) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 1aa1f1ab25a..18a81a14a2c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -305,6 +305,8 @@ public class FieldConstants { public static final String TOOL_INFO_RESOURCE_DESCRIPTION = "Tool info resource can have the values AUDIT, USER, PROJECT, " + "STUDY, FILE, SAMPLE, JOB, INDIVIDUAL, COHORT, DISEASE_PANEL, FAMILY, CLINICAL_ANALYSIS, INTERPRETATION, " + "VARIANT, ALIGNMENT, CLINICAL, EXPRESSION, RGA and FUNCTIONAL."; + public static final String TOOL_INFO_EXTERNAL_EXECUTOR_DESCRIPTION = "Object containing the id and version of the external tool that" + + " is being executed."; // Workflow public static final String WORKFLOW_ID_DESCRIPTION = "Workflow ID."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfo.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfo.java index 8c5600e416c..81db952fcfb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfo.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfo.java @@ -29,43 +29,49 @@ public class ToolInfo { description = FieldConstants.GENERIC_ID_DESCRIPTION) private String id; - @DataField(id = "description", defaultValue = "No description available", - description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) + @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "scope", defaultValue = "No description available", - description = FieldConstants.TOOL_INFO_SCOPE_DESCRIPTION) + @DataField(id = "scope", description = FieldConstants.TOOL_INFO_SCOPE_DESCRIPTION) private Scope scope; - @DataField(id = "type", defaultValue = "No description available", - description = FieldConstants.TOOL_INFO_TYPE_DESCRIPTION) + @DataField(id = "type", description = FieldConstants.TOOL_INFO_TYPE_DESCRIPTION) private Type type; - @DataField(id = "resource", defaultValue = "No description available", - description = FieldConstants.TOOL_INFO_RESOURCE_DESCRIPTION) + @DataField(id = "resource", description = FieldConstants.TOOL_INFO_RESOURCE_DESCRIPTION) private Enums.Resource resource; + @DataField(id = "resource", description = FieldConstants.TOOL_INFO_EXTERNAL_EXECUTOR_DESCRIPTION) + private ToolInfoExecutor externalExecutor; + public ToolInfo() { } public ToolInfo(String id, String description, Scope scope, Type type, Enums.Resource resource) { + this(id, description, scope, type, resource, null); + } + + public ToolInfo(String id, String description, Scope scope, Type type, Enums.Resource resource, ToolInfoExecutor externalExecutor) { this.id = id; this.description = description; this.scope = scope; this.type = type; this.resource = resource; + this.externalExecutor = externalExecutor; } @Override public String toString() { - return "ToolInfo{" + - "id='" + id + '\'' + - ", description='" + description + '\'' + - ", scope=" + scope + - ", type=" + type + - ", resource=" + resource + - '}'; + final StringBuilder sb = new StringBuilder("ToolInfo{"); + sb.append("id='").append(id).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", scope=").append(scope); + sb.append(", type=").append(type); + sb.append(", resource=").append(resource); + sb.append(", externalExecutor=").append(externalExecutor); + sb.append('}'); + return sb.toString(); } public String getId() { @@ -112,4 +118,13 @@ public ToolInfo setResource(Enums.Resource resource) { this.resource = resource; return this; } + + public ToolInfoExecutor getExternalExecutor() { + return externalExecutor; + } + + public ToolInfo setExternalExecutor(ToolInfoExecutor externalExecutor) { + this.externalExecutor = externalExecutor; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfoExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfoExecutor.java new file mode 100644 index 00000000000..0a6552c5559 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/ToolInfoExecutor.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.core.models.job; + +public class ToolInfoExecutor { + + private String id; + private String version; + + public ToolInfoExecutor() { + } + + public ToolInfoExecutor(String id, String version) { + this.id = id; + this.version = version; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ToolInfoExecutor{"); + sb.append("id='").append(id).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public ToolInfoExecutor setId(String id) { + this.id = id; + return this; + } + + public String getVersion() { + return version; + } + + public ToolInfoExecutor setVersion(String version) { + this.version = version; + return this; + } +}