Merge branch 'TASK-6445' of github.com:opencb/opencga into TASK-6445

opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java

@@ -163,7 +163,7 @@ protected void run() throws Exception {
         if (workflow.getRepository() != null && StringUtils.isNotEmpty(workflow.getRepository().getImage())) {
 //            stringBuilder.append(workflow.getRepository().getImage()).append(" -with-docker");
             stringBuilder.append(workflow.getRepository().getImage());
-            dockerParams.put("-v", "/var/run/docker.sock:/var/run/docker.sock");
+//            dockerParams.put("-v", "/var/run/docker.sock:/var/run/docker.sock");
         } else {
             for (WorkflowScript script : workflow.getScripts()) {
                 if (script.isMain()) {

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java

@@ -221,6 +221,10 @@ private RestResponse<Object> distinct() throws Exception {
         queryParams.putIfNotEmpty("uuid", commandOptions.uuid);
         queryParams.putIfNotEmpty("toolId", commandOptions.toolId);
         queryParams.putIfNotEmpty("toolType", commandOptions.toolType);
+        queryParams.putIfNotEmpty("toolExternalExecutorId", commandOptions.toolExternalExecutorId);
+        queryParams.putIfNotEmpty("parentId", commandOptions.parentId);
+        queryParams.putIfNotNull("dryRun", commandOptions.dryRun);
+        queryParams.putIfNotNull("internalKillJobRequested", commandOptions.internalKillJobRequested);
         queryParams.putIfNotEmpty("userId", commandOptions.userId);
         queryParams.putIfNotEmpty("priority", commandOptions.priority);
         queryParams.putIfNotEmpty("status", commandOptions.status);
@@ -297,6 +301,10 @@ private RestResponse<Job> search() throws Exception {
         queryParams.putIfNotEmpty("uuid", commandOptions.uuid);
         queryParams.putIfNotEmpty("toolId", commandOptions.toolId);
         queryParams.putIfNotEmpty("toolType", commandOptions.toolType);
+        queryParams.putIfNotEmpty("toolExternalExecutorId", commandOptions.toolExternalExecutorId);
+        queryParams.putIfNotEmpty("parentId", commandOptions.parentId);
+        queryParams.putIfNotNull("dryRun", commandOptions.dryRun);
+        queryParams.putIfNotNull("internalKillJobRequested", commandOptions.internalKillJobRequested);
         queryParams.putIfNotEmpty("userId", commandOptions.userId);
         queryParams.putIfNotEmpty("priority", commandOptions.priority);
         queryParams.putIfNotEmpty("status", commandOptions.status);

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java

@@ -195,6 +195,18 @@ public class DistinctCommandOptions {
         @Parameter(names = {"--tool-type"}, description = "Tool type executed by the job [OPERATION, ANALYSIS]", required = false, arity = 1)
         public String toolType; 
 
+        @Parameter(names = {"--tool.external-executor.id"}, description = "Id of the external executor. This field is only applicable for jobs executed by an external executor.", required = false, arity = 1)
+        public String toolExternalExecutorId; 
+
+        @Parameter(names = {"--parent-id"}, description = "Job id that generated this job (if any).", required = false, arity = 1)
+        public String parentId; 
+
+        @Parameter(names = {"--dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1)
+        public Boolean dryRun; 
+
+        @Parameter(names = {"--internal.kill-job-requested"}, description = "Flag indicating that the user requested to kill the job.", required = false, arity = 1)
+        public Boolean internalKillJobRequested; 
+
         @Parameter(names = {"--user-id"}, description = "User that created the job", required = false, arity = 1)
         public String userId; 
 
@@ -319,6 +331,18 @@ public class SearchCommandOptions {
         @Parameter(names = {"--tool-type"}, description = "Tool type executed by the job [OPERATION, ANALYSIS]", required = false, arity = 1)
         public String toolType; 
 
+        @Parameter(names = {"--tool.external-executor.id"}, description = "Id of the external executor. This field is only applicable for jobs executed by an external executor.", required = false, arity = 1)
+        public String toolExternalExecutorId; 
+
+        @Parameter(names = {"--parent-id"}, description = "Job id that generated this job (if any).", required = false, arity = 1)
+        public String parentId; 
+
+        @Parameter(names = {"--dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1)
+        public Boolean dryRun; 
+
+        @Parameter(names = {"--internal.kill-job-requested"}, description = "Flag indicating that the user requested to kill the job.", required = false, arity = 1)
+        public Boolean internalKillJobRequested; 
+
         @Parameter(names = {"--user-id"}, description = "User that created the job", required = false, arity = 1)
         public String userId; 
 

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java

@@ -112,8 +112,10 @@ enum QueryParams implements QueryParam {
 
         TOOL("tool", OBJECT, ""),
         TOOL_ID("tool.id", TEXT, ""),
+        TOOL_EXTERNAL_EXECUTOR_ID("tool.externalExecutor.id", TEXT, ""),
         TOOL_TYPE("tool.type", TEXT, ""),
         TOOL_EXTERNAL_EXECUTOR("tool.externalExecutor", OBJECT, ""),
+        PARENT_ID("parentId", TEXT, ""),
 
         PRIORITY("priority", TEXT, ""),
 
@@ -133,6 +135,7 @@ enum QueryParams implements QueryParam {
         INPUT("input", OBJECT, ""),
         OUTPUT("output", OBJECT, ""),
         DEPENDS_ON("dependsOn", TEXT_ARRAY, ""),
+        DRY_RUN("dryRun", BOOLEAN, ""),
         TAGS("tags", TEXT_ARRAY, ""),
 
         EXECUTION("execution", OBJECT, ""),

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java

@@ -29,6 +29,7 @@ enum QueryParams implements QueryParam {
         TAGS("tags", TEXT_ARRAY, ""),
         COMMAND_LINE("commandLine", TEXT, ""),
         MANAGER("manager", OBJECT, ""),
+        MANAGER_ID("manager.id", TEXT, ""),
         SCRIPTS("scripts", OBJECT, ""),
         VARIABLES("variables", OBJECT, ""),
         INTERNAL_REGISTRATION_USER_ID("internal.registrationUserId", TEXT, ""),

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java

@@ -886,7 +886,11 @@ private Bson parseQuery(Query query, Document extraQuery, QueryOptions options,
                     case UUID:
                     case USER_ID:
                     case TOOL_TYPE:
-                    case PRIORITY: // TODO: This filter is not indexed. We should change it and query _priority instead.
+                    case PRIORITY:
+                    case TOOL_EXTERNAL_EXECUTOR_ID:
+                    case PARENT_ID:
+                    case DRY_RUN:
+                    case INTERNAL_KILL_JOB_REQUESTED:
 //                    case START_TIME:
 //                    case END_TIME:
 //                    case OUTPUT_ERROR:

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java

@@ -563,6 +563,7 @@ protected Bson parseQuery(Query query, Document extraQuery, String user)
                     case RELEASE:
                     case VERSION:
                     case INTERNAL_REGISTRATION_USER_ID:
+                    case MANAGER_ID:
                     case TYPE:
                     case DRAFT:
                         addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList);

opencga-catalog/src/main/resources/catalog-indexes.txt

@@ -23,7 +23,6 @@
 {"collections": ["job"], "fields": {"tool.externalExecutor.id": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"userId": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"parentId": 1, "studyUid": 1}, "options": {}}
-{"collections": ["job"], "fields": {"dependsOn.id": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"outDir.uid": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"input.uid": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"output.uid": 1, "studyUid": 1}, "options": {}}
@@ -34,8 +33,6 @@
 {"collections": ["job"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"internal.status.id": 1, "_priority": 1, "_creationDate": 1}, "options": {}}
 {"collections": ["job"], "fields": {"internal.killJobRequested": 1, "studyUid": 1}, "options": {}}
-{"collections": ["job"], "fields": {"_priority": 1, "_creationDate": 1, "studyUid": 1}, "options": {}}
-{"collections": ["job"], "fields": {"_priority": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}}
 {"collections": ["job"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}}

opencga-client/src/main/R/R/Job-methods.R

@@ -21,9 +21,9 @@
 #' | -- | :-- | --: |
 #' | updateAcl | /{apiVersion}/jobs/acl/{members}/update | members[*], action[*], body[*] |
 #' | create | /{apiVersion}/jobs/create | study, body[*] |
-#' | distinct | /{apiVersion}/jobs/distinct | study, otherStudies, id, uuid, toolId, toolType, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field[*] |
+#' | distinct | /{apiVersion}/jobs/distinct | study, otherStudies, id, uuid, toolId, toolType, tool.externalExecutor.id, parentId, dryRun, internal.killJobRequested, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field[*] |
 #' | retry | /{apiVersion}/jobs/retry | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, study, body[*] |
-#' | search | /{apiVersion}/jobs/search | include, exclude, limit, skip, count, study, otherStudies, id, uuid, toolId, toolType, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted |
+#' | search | /{apiVersion}/jobs/search | include, exclude, limit, skip, count, study, otherStudies, id, uuid, toolId, toolType, tool.externalExecutor.id, parentId, dryRun, internal.killJobRequested, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted |
 #' | buildTool | /{apiVersion}/jobs/tool/build | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] |
 #' | runTool | /{apiVersion}/jobs/tool/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] |
 #' | top | /{apiVersion}/jobs/top | limit, study, internalStatus, priority, userId, toolId |
@@ -68,6 +68,10 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi
         #' @param uuid Comma separated list of job UUIDs up to a maximum of 100.
         #' @param toolId Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.
         #' @param toolType Tool type executed by the job [OPERATION, ANALYSIS].
+        #' @param tool.externalExecutor.id Id of the external executor. This field is only applicable for jobs executed by an external executor.
+        #' @param parentId Job id that generated this job (if any).
+        #' @param dryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.
+        #' @param internal.killJobRequested Flag indicating that the user requested to kill the job.
         #' @param userId User that created the job.
         #' @param priority Priority of the job.
         #' @param status Filter by status.
@@ -110,6 +114,10 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi
         #' @param uuid Comma separated list of job UUIDs up to a maximum of 100.
         #' @param toolId Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.
         #' @param toolType Tool type executed by the job [OPERATION, ANALYSIS].
+        #' @param tool.externalExecutor.id Id of the external executor. This field is only applicable for jobs executed by an external executor.
+        #' @param parentId Job id that generated this job (if any).
+        #' @param dryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.
+        #' @param internal.killJobRequested Flag indicating that the user requested to kill the job.
         #' @param userId User that created the job.
         #' @param priority Priority of the job.
         #' @param status Filter by status.

opencga-client/src/main/R/R/Workflow-methods.R

@@ -21,9 +21,9 @@
 #' | -- | :-- | --: |
 #' | updateAcl | /{apiVersion}/workflows/acl/{members}/update | study, members[*], action[*], body[*] |
 #' | create | /{apiVersion}/workflows/create | include, exclude, study, includeResult, body[*] |
-#' | distinct | /{apiVersion}/workflows/distinct | study, id, name, uuid, tags, draft, internal.registrationUserId, type, creationDate, modificationDate, acl, release, snapshot, deleted, field[*] |
+#' | distinct | /{apiVersion}/workflows/distinct | study, id, name, uuid, tags, draft, internal.registrationUserId, manager.id, type, creationDate, modificationDate, acl, release, snapshot, deleted, field[*] |
 #' | run | /{apiVersion}/workflows/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] |
-#' | search | /{apiVersion}/workflows/search | include, exclude, limit, skip, count, study, id, name, uuid, tags, draft, internal.registrationUserId, type, creationDate, modificationDate, acl, release, snapshot, deleted |
+#' | search | /{apiVersion}/workflows/search | include, exclude, limit, skip, count, study, id, name, uuid, tags, draft, internal.registrationUserId, manager.id, type, creationDate, modificationDate, acl, release, snapshot, deleted |
 #' | update | /{apiVersion}/workflows/{workflowId}/update | include, exclude, workflowId[*], study, includeResult, body |
 #' | acl | /{apiVersion}/workflows/{workflows}/acl | workflows[*], study, member, silent |
 #' | delete | /{apiVersion}/workflows/{workflows}/delete | study, workflows[*] |
@@ -67,6 +67,7 @@ setMethod("workflowClient", "OpencgaR", function(OpencgaR, members, workflowId,
         #' @param tags Comma separated list of tags.
         #' @param draft Boolean field indicating whether the workflow is a draft or not.
         #' @param internal.registrationUserId UserId that created the workflow.
+        #' @param manager.id Id of the workflow system (Allowed values: NEXTFLOW).
         #' @param type Workflow type. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER].
         #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.
         #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.
@@ -106,6 +107,7 @@ setMethod("workflowClient", "OpencgaR", function(OpencgaR, members, workflowId,
         #' @param tags Comma separated list of tags.
         #' @param draft Boolean field indicating whether the workflow is a draft or not.
         #' @param internal.registrationUserId UserId that created the workflow.
+        #' @param manager.id Id of the workflow system (Allowed values: NEXTFLOW).
         #' @param type Workflow type. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER].
         #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.
         #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.

opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java

@@ -95,6 +95,11 @@ public RestResponse<Job> create(JobCreateParams data, ObjectMap params) throws C
      *       toolId: Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g.
      *            '~value' for case sensitive, '~/value/i' for case insensitive search.
      *       toolType: Tool type executed by the job [OPERATION, ANALYSIS].
+     *       tool.externalExecutor.id: Id of the external executor. This field is only applicable for jobs executed by an external executor.
+     *       parentId: Job id that generated this job (if any).
+     *       dryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters
+     *            and prerequisites are correctly set for successful execution, but the job will not actually run.
+     *       internal.killJobRequested: Flag indicating that the user requested to kill the job.
      *       userId: User that created the job.
      *       priority: Priority of the job.
      *       status: Filter by status.
@@ -154,6 +159,11 @@ public RestResponse<Job> retry(JobRetryParams data, ObjectMap params) throws Cli
      *       toolId: Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g.
      *            '~value' for case sensitive, '~/value/i' for case insensitive search.
      *       toolType: Tool type executed by the job [OPERATION, ANALYSIS].
+     *       tool.externalExecutor.id: Id of the external executor. This field is only applicable for jobs executed by an external executor.
+     *       parentId: Job id that generated this job (if any).
+     *       dryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters
+     *            and prerequisites are correctly set for successful execution, but the job will not actually run.
+     *       internal.killJobRequested: Flag indicating that the user requested to kill the job.
      *       userId: User that created the job.
      *       priority: Priority of the job.
      *       status: Filter by status.

opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java

@@ -99,6 +99,7 @@ public RestResponse<Workflow> create(WorkflowCreateParams data, ObjectMap params
      *       tags: Comma separated list of tags.
      *       draft: Boolean field indicating whether the workflow is a draft or not.
      *       internal.registrationUserId: UserId that created the workflow.
+     *       manager.id: Id of the workflow system (Allowed values: NEXTFLOW).
      *       type: Workflow type. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER].
      *       creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.
      *       modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.
@@ -156,6 +157,7 @@ public RestResponse<Job> run(NextFlowRunParams data, ObjectMap params) throws Cl
      *       tags: Comma separated list of tags.
      *       draft: Boolean field indicating whether the workflow is a draft or not.
      *       internal.registrationUserId: UserId that created the workflow.
+     *       manager.id: Id of the workflow system (Allowed values: NEXTFLOW).
      *       type: Workflow type. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER].
      *       creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.
      *       modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805.