From b15fc09beff203a81d622c49707a175ce75a92bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 25 Jan 2024 21:36:15 +0100 Subject: [PATCH 001/128] catalog: fix clinical analysis creation, #TASK-5538, #TASK-5516 --- .../managers/ClinicalAnalysisManager.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 351ff17a184..5fb26ec5367 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -342,14 +342,20 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis // Validate users Set userIds = new HashSet<>(); for (ClinicalAnalyst analyst : clinicalAnalysis.getAnalysts()) { - userIds.add(analyst.getId()); + if (StringUtils.isNotEmpty(analyst.getId())) { + userIds.add(analyst.getId()); + } } - Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userIds); - OpenCGAResult result = userDBAdaptor.get(query, userInclude); - if (result.getNumResults() < userIds.size()) { - throw new CatalogException("Some clinical analysts could not be found."); + if (CollectionUtils.isNotEmpty(userIds)) { + Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userIds); + OpenCGAResult result = userDBAdaptor.get(query, userInclude); + if (result.getNumResults() < userIds.size()) { + throw new CatalogException("Some clinical analysts could not be found."); + } + userList = result.getResults(); + } else { + userList = userDBAdaptor.get(userId, userInclude).getResults(); } - userList = result.getResults(); } List clinicalAnalystList = new ArrayList<>(userList.size()); for (User user : userList) { @@ -663,7 +669,7 @@ public ClinicalAnalysisLoadResult load(String studyStr, Path filePath, String to counter++; } catch (Exception e) { logger.error("Error loading clinical analysis" + (clinicalAnalysis != null ? (": " + clinicalAnalysis.getId()) : "") - + ": " + e.getMessage()); + + ": " + e.getMessage()); result.getFailures().put(clinicalAnalysis.getId(), e.getMessage()); } } @@ -2328,8 +2334,8 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List updateAnnotations(String studyStr, String clinicalStr, String annotationSetId, - Map annotations, ParamUtils.CompleteUpdateAction action, - QueryOptions options, String token) throws CatalogException { + Map annotations, ParamUtils.CompleteUpdateAction action, + QueryOptions options, String token) throws CatalogException { if (annotations == null || annotations.isEmpty()) { throw new CatalogException("Missing array of annotations."); } From 23224a3fafd41d9cdf940518c2c37f7a95552b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 25 Jan 2024 22:03:10 +0100 Subject: [PATCH 002/128] pom: workaround for SonarCloud issue on CI/CD, #TASK-5516 --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 0de0a7fdc99..8079976c15f 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,13 @@ 2.5-20081211 2.23.0 + + + true From 32710deff1a0b8ed8826273bf7781d8e18bed831 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 8 Mar 2024 17:07:25 +0100 Subject: [PATCH 003/128] Prepare cicd release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86bcb471976..93b011f556f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ jobs: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: build with: - cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ needs.build.outputs.version }} + cli: python3 ./build/cloud/docker/docker-build.py push --images base,init secrets: inherit deploy-python: From 3e75a862e879ff7e225b8d65d215459f0335e08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 19 Apr 2024 13:37:48 +0100 Subject: [PATCH 004/128] storage: Implement HBase table autoscale #TASK-5448 --- .../org/opencb/opencga/server/RestServer.java | 3 + .../core/variant/VariantStorageBaseTest.java | 5 + .../opencga-storage-hadoop-compat-api/pom.xml | 6 + .../storage/hadoop/HBaseCompatApi.java | 5 + .../pom.xml | 6 + .../opencga/storage/hadoop/HBaseCompat.java | 9 + .../pom.xml | 6 + .../opencga/storage/hadoop/HBaseCompat.java | 9 + .../pom.xml | 6 + .../opencga/storage/hadoop/HBaseCompat.java | 8 + .../opencga-storage-hadoop-compat/pom.xml | 6 + .../opencga/storage/hadoop/app/HBaseMain.java | 216 ++++++++++-------- .../storage/hadoop/utils/HBaseManager.java | 146 +++++++++++- .../variant/HadoopVariantStorageOptions.java | 2 + .../variant/HadoopVariantStoragePipeline.java | 1 + .../variant/archive/ArchiveRowKeyFactory.java | 21 ++ .../variant/archive/ArchiveTableHelper.java | 41 ++-- .../index/sample/SampleIndexBuilder.java | 1 + .../index/sample/SampleIndexDBAdaptor.java | 36 ++- .../index/sample/SampleIndexDBLoader.java | 1 + .../variant/AutoScaleHBaseTableTest.java | 116 ++++++++++ 21 files changed, 537 insertions(+), 113 deletions(-) create mode 100644 opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java index 28bdffc8016..2c431ea36cb 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java @@ -16,7 +16,10 @@ package org.opencb.opencga.server; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.webapp.WebAppContext; import java.io.IOException; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java index fe5313bdc80..23ece394c79 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java @@ -374,6 +374,11 @@ public static StoragePipelineResult runDefaultETL(URI inputUri, VariantStorageEn return storagePipelineResult; } + public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, + ObjectMap params) throws StorageEngineException, FileFormatException, IOException { + return runETL(variantStorageManager, inputUri, outputUri, params, true, true, true); + } + public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, ObjectMap params, boolean doExtract, diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml index deac888af82..9173d1853fa 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -43,6 +43,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java index 4bfb6cd45fd..8ffcf75747f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java @@ -1,9 +1,12 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.List; public abstract class HBaseCompatApi { @@ -28,4 +31,6 @@ public static HBaseCompatApi getInstance() { public abstract void available(Configuration configuration) throws IOException; public abstract boolean isSolrTestingAvailable(); + + public abstract List getServerList(Admin admin) throws IOException; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml index 661d9416a8f..890234d2dac 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -49,6 +49,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index f3f9170f6f7..8691e5721b0 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -1,11 +1,15 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class HBaseCompat extends HBaseCompatApi { @@ -24,4 +28,9 @@ public PhoenixCompatApi getPhoenixCompat() { return new PhoenixCompat(); } + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterStatus().getServers()); + } + } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml index 358590f69f6..7760b351e8f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -49,6 +49,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index 451238f6da5..f249baffc3c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -1,11 +1,15 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class HBaseCompat extends HBaseCompatApi { @@ -22,4 +26,9 @@ public boolean isSolrTestingAvailable() { public PhoenixCompatApi getPhoenixCompat() { return new PhoenixCompat(); } + + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterMetrics().getServersName()); + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml index 394e6c90e16..35c8abdfa23 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -49,6 +49,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index 93f61922b9f..062adc91618 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -1,11 +1,15 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class HBaseCompat extends HBaseCompatApi { @@ -24,4 +28,8 @@ public PhoenixCompatApi getPhoenixCompat() { return new PhoenixCompat(); } + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterMetrics().getServersName()); + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml index 58aac8414c5..42338f95864 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -45,6 +45,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.phoenix phoenix-core diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java index e16371af737..154ac5be9b3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java @@ -3,18 +3,19 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.tools.ant.types.Commandline; import org.opencb.commons.ProgressLogger; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; @@ -78,9 +79,15 @@ public void run(String[] args) throws Exception { case MOVE_TABLE_REGIONS: moveTableRegions(args); break; - case BALANCE_TABLE_REGIONS: - balanceTableRegions(getArg(args, 1), getArgsMap(args, 2, "maxMoves")); + case BALANCE_TABLE_REGIONS: { + ObjectMap argsMap = getArgsMap(args, 2, "maxMoves", "dryRun", "ignoreExceptions", "maxRetries"); + balanceTableRegions(getArg(args, 1), + argsMap.getInt("maxMoves", 50000), + argsMap.getBoolean("dryRun", false), + argsMap.getBoolean("ignoreExceptions", false), + argsMap.getInt("maxRetries", 1)); break; + } case "tables": case LIST_TABLES: print(listTables(getArg(args, 1, "")).stream().map(TableName::getNameWithNamespaceInclAsString).iterator()); @@ -197,8 +204,10 @@ public void run(String[] args) throws Exception { System.out.println(" (see " + CHECK_TABLES_WITH_REGIONS_ON_DEAD_SERVERS + ") by creating a temporary snapshot"); System.out.println(" " + MOVE_TABLE_REGIONS + " "); System.out.println(" Move all regions from selected tables to new random nodes."); -// System.out.println(" " + BALANCE_TABLE_REGIONS + " [--maxMoves N]"); // FIXME -// System.out.println(" " + REGIONS_PER_TABLE + " "); // FIXME + System.out.println(" " + BALANCE_TABLE_REGIONS + " [--maxMoves N]" + + " [--maxRetries 1] [--ignoreExceptions]" + + " [--dryRun]"); + System.out.println(" " + REGIONS_PER_TABLE + " "); // FIXME System.out.println(" " + CLONE_TABLES + " " + "[--keepSnapshots] [--dryRun] [--snapshotSuffix ]"); System.out.println(" Clone all selected tables by creating an intermediate snapshot."); @@ -207,8 +216,8 @@ public void run(String[] args) throws Exception { + "[--skipTablesWithSnapshot]"); System.out.println(" " + SNAPSHOT_TABLE + " [--dryRun] [--snapshotSuffix ] " + "[--skipTablesWithSnapshot]"); - System.out.println(" " + DELETE_SNAPSHOTS + " [--dryRun] [--skipMissing]"); System.out.println(" Create a snapshot for all selected tables."); + System.out.println(" " + DELETE_SNAPSHOTS + " [--dryRun] [--skipMissing]"); System.out.println(" " + CLONE_SNAPSHOTS + " [--dryRun] " + "[--tablePrefixChange :] " + "[--onExistingTables [fail|skip|drop] ]"); @@ -326,47 +335,38 @@ private void exportSnapshot(String storageConfigurationPath, String snapshot, St } private void regionsPerTable(String tableNameStr) throws Exception { -// TableName tableName = getTable(tableNameStr); -// hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { -// List servers = new ArrayList<>(admin.getClusterStatus().getServers()); -// Map regionsPerServer = new HashMap<>(); -// -// List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); -// -// System.out.println("#REGION\tSERVER\tSTART_KEY\tEND_KEY"); -// for (Pair pair : tableRegionsAndLocations) { -// RegionInfo region = pair.getFirst(); -// ServerName server = pair.getSecond(); -// regionsPerServer.merge(server.getServerName(), 1, Integer::sum); -// -// System.out.println(region.getEncodedName() -// + "\t" + server.getServerName() -// + "\t" + Bytes.toStringBinary(region.getStartKey()) -// + "\t" + Bytes.toStringBinary(region.getEndKey())); -// } -// -// System.out.println(""); -// System.out.println("#SERVER\tREGIONS"); -// for (ServerName server : servers) { -// System.out.println(server.getServerName() + "\t" + regionsPerServer.getOrDefault(server.getServerName(), 0)); -// } -// -// -// -// return null; -// }); - } - -// private List> getTableRegionsAndLocations(TableName tableName, Admin admin) throws IOException { -// List> tableRegionsAndLocations; -//// try (ZooKeeperWatcher zkw = new ZooKeeperWatcher(admin.getConfiguration(), "hbase-main", null)) { -//// tableRegionsAndLocations = MetaTableAccessor -//// .getTableRegionsAndLocations(zkw, admin.getConnection(), tableName); -//// } -// tableRegionsAndLocations = MetaTableAccessor -// .getTableRegionsAndLocations(admin.getConnection(), tableName); -// return tableRegionsAndLocations; -// } + TableName tableName = getTable(tableNameStr); + hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { + List servers = new ArrayList<>(admin.getClusterStatus().getServers()); + Map regionsPerServer = new HashMap<>(); + + List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); + + System.out.println("#REGION\tSERVER\tSTART_KEY\tEND_KEY"); + for (Pair pair : tableRegionsAndLocations) { + RegionInfo region = pair.getFirst(); + ServerName server = pair.getSecond(); + regionsPerServer.merge(server.getServerName(), 1, Integer::sum); + + System.out.println(region.getEncodedName() + + "\t" + server.getServerName() + + "\t" + Bytes.toStringBinary(region.getStartKey()) + + "\t" + Bytes.toStringBinary(region.getEndKey())); + } + + System.out.println(""); + System.out.println("#SERVER\tREGIONS"); + for (ServerName server : servers) { + System.out.println(server.getServerName() + "\t" + regionsPerServer.getOrDefault(server.getServerName(), 0)); + } + + return null; + }); + } + + private List> getTableRegionsAndLocations(TableName tableName, Admin admin) throws IOException { + return MetaTableAccessor.getTableRegionsAndLocations(admin.getConnection(), tableName); + } private void reassignTablesWithRegionsOnDeadServers(String[] args) throws Exception { String tableNameFilter = getArg(args, 1); @@ -415,47 +415,81 @@ private void moveTableRegions(String[] args) throws Exception { }); } - private void balanceTableRegions(String tableNameStr, ObjectMap options) throws Exception { -// TableName tableName = getTable(tableNameStr); -// -// int regionCount = hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { -// int maxMoves = options.getInt("maxMoves", 50000); -// List servers = new ArrayList<>(admin.getClusterStatus().getServers()); -// List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); -// int expectedRegionsPerServer = (tableRegionsAndLocations.size() / servers.size()) + 1; -// Map regionsPerServer = new HashMap<>(); -// servers.forEach(s -> regionsPerServer.put(s.getServerName(), 0)); -// for (Pair pair : tableRegionsAndLocations) { -// regionsPerServer.merge(pair.getSecond().getServerName(), 1, Integer::sum); -// } -// -// for (Pair pair : tableRegionsAndLocations) { -// if (maxMoves < 0) { -// System.out.println("Reached max moves!"); -// break; -// } -// -// String sourceHost = pair.getSecond().getServerName(); -// if (regionsPerServer.get(sourceHost) > expectedRegionsPerServer) { -// Collections.shuffle(servers); -// Optional targetOptional = servers.stream() -// .filter(s -> regionsPerServer.get(s.getServerName()) < expectedRegionsPerServer).findAny(); -// if (!targetOptional.isPresent()) { -// break; -// } -// String testHost = targetOptional.get().getServerName(); -// regionsPerServer.merge(sourceHost, -1, Integer::sum); -// regionsPerServer.merge(testHost, 1, Integer::sum); -// System.out.println("Move region '" + pair.getFirst().getEncodedName() + "' from " + sourceHost + " to " + testHost); -// StopWatch stopWatch = StopWatch.createStarted(); -// admin.move(pair.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(testHost)); -// System.out.println("Moved in "+TimeUtils.durationToString(stopWatch)); -// -// maxMoves--; -// } -// } -// return tableRegionsAndLocations.size(); -// }); + private void balanceTableRegions(String tableNameStr, int maxMoves, boolean dryRun, boolean ignoreExceptions, int maxRetries) + throws Exception { + TableName tableName = getTable(tableNameStr); + + LOGGER.info("Balancing table " + tableName.getNameAsString() + " with maxMoves=" + maxMoves + ", dryRun=" + dryRun + + ", ignoreExceptions=" + ignoreExceptions + ", maxRetries=" + maxRetries); + int regionCount = hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { + List servers = HBaseCompat.getInstance().getServerList(admin); + List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); + int expectedRegionsPerServer = (tableRegionsAndLocations.size() / servers.size()) + 1; + + Map regionsPerServer = new HashMap<>(); + servers.forEach(s -> regionsPerServer.put(s.getServerName(), 0)); + for (Pair pair : tableRegionsAndLocations) { + regionsPerServer.merge(pair.getSecond().getServerName(), 1, Integer::sum); + } + + // Shuffle the regions to avoid hotspots + Collections.shuffle(tableRegionsAndLocations); + + int moves = 0; + for (Pair pair : tableRegionsAndLocations) { + if (moves > maxMoves) { + LOGGER.info("Reached max moves!"); + break; + } + + ServerName sourceHost = pair.getSecond(); + // If the source host has more regions than expected, move one to another server + if (regionsPerServer.get(sourceHost.getServerName()) > expectedRegionsPerServer) { + // Iterate over the servers in a random order + Collections.shuffle(servers); + Optional targetOptional = servers.stream() + .filter(s -> !s.equals(sourceHost)) + .filter(s -> regionsPerServer.get(s.getServerName()) < expectedRegionsPerServer) + .findAny(); + if (!targetOptional.isPresent()) { + break; + } + ServerName targetHost = targetOptional.get(); + LOGGER.info("Move region '" + pair.getFirst().getEncodedName() + "' from " + sourceHost + " to " + targetHost); + StopWatch stopWatch = StopWatch.createStarted(); + int attempts = 0; + while (attempts < maxRetries) { + try { + if (dryRun) { + LOGGER.info("[DRY-RUN]: admin.move('" + pair.getFirst().getEncodedName() + "," + targetHost + ")"); + } else { + admin.move(pair.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(targetHost.getServerName())); + } + break; + } catch (Exception e) { + LOGGER.info("Error moving region: " + e.getMessage()); + attempts++; + if (attempts < maxRetries) { + LOGGER.info("Retrying... " + attempts + "/" + maxRetries); + } else if (!ignoreExceptions) { + throw e; + } else { + LOGGER.info("Ignoring exception. Unable to move region after " + attempts + " attempts."); + break; + } + } + } + LOGGER.info("Moved in " + TimeUtils.durationToString(stopWatch)); + + regionsPerServer.merge(sourceHost.getServerName(), -1, Integer::sum); + regionsPerServer.merge(targetHost.getServerName(), 1, Integer::sum); + + moves++; + } + } + return tableRegionsAndLocations.size(); + }); + System.out.println("#Balanced regions for table '" + tableNameStr + "' . Total regions: " + regionCount); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java index f9bfa4d0a3f..3eef1fe3ced 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java @@ -17,24 +17,25 @@ package org.opencb.opencga.storage.hadoop.utils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.core.common.ExceptionUtils; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.hadoop.auth.HBaseCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import static org.opencb.opencga.storage.hadoop.utils.PersistentResultScanner.isValid; @@ -388,6 +389,145 @@ public boolean createTableIfNeeded(String tableName, byte[] columnFamily, return createTableIfNeeded(getConnection(), tableName, columnFamily, preSplits, compressionType); } + public boolean splitAndMove(Admin admin, TableName name, byte[] expectedSplit) throws IOException { + return splitAndMove(admin, name, expectedSplit, 3, true); + } + + public boolean splitAndMove(Admin admin, TableName tableName, byte[] expectedSplit, int retries, boolean ignoreExceptions) + throws IOException { + StopWatch stopWatch = StopWatch.createStarted(); + int count = 0; + while (count < retries) { + count++; + try { + // Check if split point exists + RegionInfo regionInfo = getRegionInfo(admin, tableName, expectedSplit); + + if (regionInfo == null) { + LOGGER.info("Splitting table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + admin.split(tableName, expectedSplit); + regionInfo = getRegionInfo(admin, tableName, expectedSplit); + int getRegionInfoAttempts = 10; + while (regionInfo == null) { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + regionInfo = getRegionInfo(admin, tableName, expectedSplit); + getRegionInfoAttempts--; + if (getRegionInfoAttempts == 0) { + throw new IOException("Split point not found after creation"); + } + } + } else if (count == 1) { + // First try and split point exists. Skip moving region + LOGGER.info("Split point {} already exists. Nothing to do.", Bytes.toStringBinary(expectedSplit)); + return false; + } + LOGGER.info("Moving region '{}' to another region server", regionInfo.getRegionNameAsString()); + admin.move(regionInfo.getEncodedNameAsBytes()); + LOGGER.info("New region created '{}' in {}", regionInfo.getRegionNameAsString(), TimeUtils.durationToString(stopWatch)); + return true; + } catch (IOException | RuntimeException e) { + if (ignoreExceptions) { + LOGGER.warn("Error splitting table {} at {}. Retry {}/{}", tableName, + Bytes.toStringBinary(expectedSplit), count, retries, e); + } else { + throw e; + } + } + try { + // Wait before retry + LOGGER.info("Waiting before retrying split table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + LOGGER.warn("Unable to split table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + return false; + } + + public static RegionInfo getRegionInfo(Admin admin, TableName name, byte[] expectedSplit) throws IOException { + return admin.getRegions(name) + .stream() + .filter(region -> Bytes.equals(region.getStartKey(), expectedSplit)) + .findFirst() + .orElse(null); + } + + public int expandTableIfNeeded(String tableName, int batch, + Function> batchSplitsGenerator, + int extraBatches, Function batchPlaceholderSplitGenerator) throws IOException { + return expandTableIfNeeded(tableName, Collections.singletonList(batch), batchSplitsGenerator, extraBatches, + batchPlaceholderSplitGenerator); + } + public int expandTableIfNeeded(String tableName, Collection batches, + Function> batchSplitsGenerator, + int extraBatches, Function batchPlaceholderSplitGenerator) throws IOException { + if (batches.isEmpty()) { + throw new IllegalArgumentException("No batches provided"); + } + // Get the expected splits for these batches + Collection expectedSplits = new LinkedHashSet<>(); + for (Integer batch : batches) { + expectedSplits.addAll(batchSplitsGenerator.apply(batch)); + } + int lastBatch = batches.stream().max(Integer::compareTo).get(); + + // Add some split placeholders for the extra batches + for (int i = lastBatch + 1; i < lastBatch + 1 + extraBatches; i++) { + expectedSplits.add(batchPlaceholderSplitGenerator.apply(i)); + } + // Shuffle the splits + List shuffledSplits = new ArrayList<>(expectedSplits); + Collections.shuffle(shuffledSplits); + + // Ensure that the table is split at least until the next expected split + return act(tableName, (table, admin) -> { + int newSplits = 0; + byte[][] existingSplits = table.getRegionLocator().getStartKeys(); + + int expectedNewSplits = 0; + for (byte[] expectedSplit : expectedSplits) { + boolean found = false; + for (byte[] split : existingSplits) { + if (Bytes.compareTo(expectedSplit, split) == 0) { + found = true; + break; + } + } + if (!found) { + expectedNewSplits++; + LOGGER.info("Missing split point '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + } + } + if (expectedNewSplits == 0) { + return 0; + } else { + LOGGER.info("Found {} missing split points. Splitting table '{}'", expectedNewSplits, tableName); + } + + for (byte[] expectedSplit : expectedSplits) { + boolean found = false; + for (byte[] split : existingSplits) { + if (Bytes.compareTo(expectedSplit, split) == 0) { + found = true; + break; + } + } + if (!found) { + if (splitAndMove(admin, table.getName(), expectedSplit)) { + newSplits++; + } + } + } + return newSplits; + }); + } + + /** * Create default HBase table layout with one column family. * diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java index ab6dfaf87ef..272daaa4d7e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java @@ -78,6 +78,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { ///////////////////////// ARCHIVE_TABLE_COMPRESSION("storage.hadoop.archive.table.compression", Compression.Algorithm.GZ.getName()), ARCHIVE_TABLE_PRESPLIT_SIZE("storage.hadoop.archive.table.preSplit.splitsPerBatch", 500), + ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.archive.table.preSplit.extraSplits", 3), ARCHIVE_CHUNK_SIZE("storage.hadoop.archive.table.chunkSize", 1000), ARCHIVE_FILE_BATCH_SIZE("storage.hadoop.archive.table.fileBatchSize", 1000), @@ -92,6 +93,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { ///////////////////////// SAMPLE_INDEX_TABLE_COMPRESSION("storage.hadoop.sampleIndex.table.compression", Compression.Algorithm.SNAPPY.getName()), SAMPLE_INDEX_TABLE_PRESPLIT_SIZE("storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit", 15), + SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.sampleIndex.table.preSplit.extraSplits", 5), SAMPLE_INDEX_BUILD_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.build.maxSamplesPerMR", 2000), SAMPLE_INDEX_ANNOTATION_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.annotation.maxSamplesPerMR", 2000), SAMPLE_INDEX_FAMILY_MAX_TRIOS_PER_MR("storage.hadoop.sampleIndex.family.maxTriosPerMR", 1000), diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java index 9b86cba3f6a..c3f5ebf550e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java @@ -170,6 +170,7 @@ public URI preLoad(URI input, URI output) throws StorageEngineException { try { if (YesNoAuto.parse(getOptions(), LOAD_ARCHIVE.key()).orYes().booleanValue()) { ArchiveTableHelper.createArchiveTableIfNeeded(getOptions(), getArchiveTable(), dbAdaptor.getConnection()); + ArchiveTableHelper.expandTableIfNeeded(getOptions(), getArchiveTable(), getFileId(), dbAdaptor.getHBaseManager()); } else { logger.info("Skip archive table"); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java index 43e9cf97cf7..d20e5eee924 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java @@ -143,6 +143,27 @@ public String generateBlockId(int fileId) { return sb.toString(); } + /** + * Generates a Row key based on Chromosome and position adjusted for the + * Chunk size.
+ *
    + *
  • Using {@link Region#normalizeChromosome(String)} to get standard chromosome + * name + *
  • Using {@link #getSliceId(long)} to return slice position + *
+ * e.g. using chunk size 100, separator _ with chr2 and 1234 would result in + * 2_12 + * + * @param fileBatch File batch + * @return {@link String} Row key string + */ + public String generateBlockIdFromBatch(int fileBatch) { + StringBuilder sb = new StringBuilder(FILE_BATCH_PAD + 1); + sb.append(StringUtils.leftPad(String.valueOf(fileBatch), FILE_BATCH_PAD, '0')); + sb.append(getSeparator()); + return sb.toString(); + } + public String generateBlockIdFromSlice(int fileId, String chrom, long slice) { return generateBlockIdFromSliceAndBatch(getFileBatch(fileId), chrom, slice); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java index 7c831e3cc81..3a0e168d96e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java @@ -55,7 +55,7 @@ public class ArchiveTableHelper extends GenomeHelper { public static final String REF_COLUMN_SUFIX = "_R"; public static final byte[] REF_COLUMN_SUFIX_BYTES = Bytes.toBytes(REF_COLUMN_SUFIX); - private final Logger logger = LoggerFactory.getLogger(ArchiveTableHelper.class); + private static Logger logger = LoggerFactory.getLogger(ArchiveTableHelper.class); private final AtomicReference meta = new AtomicReference<>(); private final ArchiveRowKeyFactory keyFactory; private final byte[] nonRefColumn; @@ -187,17 +187,13 @@ public static String getRefColumnName(int fileId) { public static boolean createArchiveTableIfNeeded(Configuration conf, String tableName) throws IOException { try (Connection con = ConnectionFactory.createConnection(conf)) { - return createArchiveTableIfNeeded(conf, tableName, con); + Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( + conf.get(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); + final List preSplits = generateArchiveTableBootPreSplitHuman(conf); + return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); } } - public static boolean createArchiveTableIfNeeded(Configuration conf, String tableName, Connection con) throws IOException { - Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( - conf.get(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); - final List preSplits = generateArchiveTableBootPreSplitHuman(conf); - return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); - } - public static boolean createArchiveTableIfNeeded(ObjectMap conf, String tableName, Connection con) throws IOException { Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( conf.getString(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); @@ -205,6 +201,20 @@ public static boolean createArchiveTableIfNeeded(ObjectMap conf, String tableNam return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); } + public static void expandTableIfNeeded(ObjectMap options, String archiveTable, int fileId, HBaseManager hBaseManager) + throws IOException { + int splitsPerBatch = options.getInt(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), ARCHIVE_TABLE_PRESPLIT_SIZE.defaultValue()); + int extraBatches = options.getInt(ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.key(), ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); + ArchiveRowKeyFactory rowKeyFactory = new ArchiveRowKeyFactory(options); + int thisBatch = rowKeyFactory.getFileBatch(fileId); + int newRegions = hBaseManager.expandTableIfNeeded(archiveTable, thisBatch, + batch -> generateBatchSplitsHuman(rowKeyFactory, splitsPerBatch, batch), + extraBatches, batch -> Bytes.toBytes(rowKeyFactory.generateBlockIdFromBatch(batch))); + if (newRegions > 0) { + logger.info("Table '" + archiveTable + "' expanded with " + newRegions + " new regions for batch " + thisBatch); + } + } + public static List generateArchiveTableBootPreSplitHuman(Configuration conf) { final ArchiveRowKeyFactory rowKeyFactory = new ArchiveRowKeyFactory(conf); int nSplits = conf.getInt(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), ARCHIVE_TABLE_PRESPLIT_SIZE.defaultValue()); @@ -225,15 +235,18 @@ private static List generateArchiveTableBootPreSplitHuman(ArchiveRowKeyF final List preSplits = new ArrayList<>(nSplits * expectedNumBatches); for (int batch = 0; batch <= expectedNumBatches; batch++) { - int finalBatch = batch; - preSplits.addAll(generateBootPreSplitsHuman(nSplits, (chr, position) -> { - long slice = rowKeyFactory.getSliceId(position); - return Bytes.toBytes(rowKeyFactory.generateBlockIdFromSliceAndBatch(finalBatch, chr, slice)); - })); + preSplits.addAll(generateBatchSplitsHuman(rowKeyFactory, nSplits, batch)); } return preSplits; } + public static List generateBatchSplitsHuman(ArchiveRowKeyFactory rowKeyFactory, int nSplits, int batch) { + return generateBootPreSplitsHuman(nSplits, (chr, position) -> { + long slice = rowKeyFactory.getSliceId(position); + return Bytes.toBytes(rowKeyFactory.generateBlockIdFromSliceAndBatch(batch, chr, slice)); + }); + } + public VariantFileMetadata getFileMetadata() { return meta.get(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java index 8b18a5e9a9f..8047fb49cf1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java @@ -85,6 +85,7 @@ public void buildSampleIndex(List samples, ObjectMap options, boolean ov } sampleIndexDBAdaptor.createTableIfNeeded(studyId, schema.getVersion(), options); + sampleIndexDBAdaptor.expandTableIfNeeded(studyId, schema.getVersion(), sampleIds, options); if (finalSamplesList.size() < 20) { logger.info("Run sample index build on samples " + finalSamplesList); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java index 06a3ca79b9a..72884815476 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java @@ -2,10 +2,7 @@ import com.google.common.collect.Iterators; import org.apache.commons.collections4.CollectionUtils; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.util.Bytes; import org.opencb.biodata.models.core.Region; @@ -796,8 +793,11 @@ public boolean createTableIfNeeded(int studyId, int version, ObjectMap options) int preSplitSize = options.getInt( HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.defaultValue()); + int sampleIndexExtraSplits = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); - int splits = samples / preSplitSize; + int splits = (samples / preSplitSize) + sampleIndexExtraSplits; ArrayList preSplits = new ArrayList<>(splits); for (int i = 0; i < splits; i++) { preSplits.add(SampleIndexSchema.toRowKey(i * preSplitSize)); @@ -815,6 +815,32 @@ public boolean createTableIfNeeded(int studyId, int version, ObjectMap options) } } + public void expandTableIfNeeded(int studyId, int version, List sampleIds, ObjectMap options) { + String sampleIndexTable = getSampleIndexTableName(studyId, version); + int preSplitSize = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.defaultValue()); + int sampleIndexExtraBatches = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); + Set batches = new HashSet<>(); + for (Integer sampleId : sampleIds) { + batches.add((sampleId) / preSplitSize); + } + + try { + int newRegions = hBaseManager.expandTableIfNeeded(sampleIndexTable, batches, + batch -> Collections.singletonList(SampleIndexSchema.toRowKey(batch * preSplitSize)), + sampleIndexExtraBatches, batch -> SampleIndexSchema.toRowKey(batch * preSplitSize)); + if (newRegions != 0) { + // Log number of new regions + logger.info("Created {} new regions in table '{}'", newRegions, sampleIndexTable); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public void updateSampleIndexSchemaStatus(int studyId, int version) throws StorageEngineException { StudyMetadata studyMetadata = metadataManager.getStudyMetadata(studyId); if (studyMetadata.getSampleIndexConfiguration(version).getStatus() diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java index f2700831e94..23c94cf0c28 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java @@ -188,6 +188,7 @@ public String toString() { public boolean open() { super.open(); dbAdaptor.createTableIfNeeded(studyId, schema.getVersion(), options); + dbAdaptor.expandTableIfNeeded(studyId, schema.getVersion(), sampleIds, options); return true; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java new file mode 100644 index 00000000000..457e25bc240 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java @@ -0,0 +1,116 @@ +package org.opencb.opencga.storage.hadoop.variant; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.ExternalResource; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.hadoop.utils.HBaseManager; +import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; + +import static org.junit.Assert.assertEquals; + +@Category(LongTests.class) +public class AutoScaleHBaseTableTest extends VariantStorageBaseTest implements HadoopVariantStorageTest { + + @ClassRule + public static ExternalResource externalResource = new HadoopExternalResource(); + + private VariantHadoopDBAdaptor dbAdaptor; + private SampleIndexDBAdaptor sampleIndexDBAdaptor; + private HadoopVariantStorageEngine engine; + + @Before + public void before() throws Exception { + clearDB(DB_NAME); + engine = getVariantStorageEngine(); + dbAdaptor = engine.getDBAdaptor(); + sampleIndexDBAdaptor = engine.getSampleIndexDBAdaptor(); + + } + + @Test + public void testAutoScaleTables() throws Exception { + + int archiveSplitsPerBatch = 10; + int samplesPerSplit = 2; + int extraSplits = 3; + + ObjectMap params = new ObjectMap() + .append(VariantStorageOptions.STUDY.key(), STUDY_NAME) + .append(VariantStorageOptions.ANNOTATE.key(), false) + .append(VariantStorageOptions.STATS_CALCULATE.key(), false) + .append(HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), samplesPerSplit) + .append(HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), extraSplits) + .append(HadoopVariantStorageOptions.ARCHIVE_TABLE_PRESPLIT_SIZE.key(), archiveSplitsPerBatch) + .append(HadoopVariantStorageOptions.ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.key(), extraSplits) + .append(HadoopVariantStorageOptions.ARCHIVE_FILE_BATCH_SIZE.key(), 2) + .append(HadoopVariantStorageOptions.EXPECTED_SAMPLES_NUMBER.key(), 1) + .append(HadoopVariantStorageOptions.EXPECTED_FILES_NUMBER.key(), 1); + + + // -- Batch 1 + int batches = 1; + runETL(engine, getPlatinumFile(1), outputUri, params); + // Each batch starts with one extra split, expect the first batch. So, -1 + // Then, a fixed number of extra splits + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 2 + // First batch has 1 fewer elements than the rest of the batches. + batches = 2; + runETL(engine, getPlatinumFile(2), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + runETL(engine, getPlatinumFile(3), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 3 + batches = 3; + runETL(engine, getPlatinumFile(4), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + runETL(engine, getPlatinumFile(5), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 4 + batches = 4; + runETL(engine, getPlatinumFile(6), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + +// VariantHbaseTestUtils.printVariants(dbAdaptor, newOutputUri()); + } + + private void checkArchiveTableSplits(int expectedSplits) throws Exception { + int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); + + String archiveTableName = engine.getArchiveTableName(studyId); + HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); + int archiveNumRegions = hBaseManager.act(archiveTableName, table -> table.getRegionLocator().getStartKeys().length); + // numRegions == numSplits + 1 + assertEquals(archiveTableName, expectedSplits + 1, archiveNumRegions); + } + + private void checkSampleIndexTableSplits(int expectedSplits) throws Exception { + int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); + + String sampleIndexTableName = sampleIndexDBAdaptor.getSampleIndexTableName(studyId, 1); + HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); + int sampleIndexNumRegions = hBaseManager.act(sampleIndexTableName, table -> table.getRegionLocator().getStartKeys().length); + // numRegions == numSplits + 1 + assertEquals(sampleIndexTableName, expectedSplits + 1, sampleIndexNumRegions); + } + + +} From 3a77b5994ad9836f4f3f5d0c541b7c8a08fb24c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 16 May 2024 09:49:08 +0100 Subject: [PATCH 005/128] storage: Fix hbase compatibility for table autoscale. #TASK-5448 --- .../opencga/storage/hadoop/HBaseCompatApi.java | 3 +++ .../opencb/opencga/storage/hadoop/HBaseCompat.java | 12 ++++++++++++ .../opencb/opencga/storage/hadoop/HBaseCompat.java | 14 ++++++++++++++ .../opencb/opencga/storage/hadoop/HBaseCompat.java | 6 ++++++ .../opencga/storage/hadoop/utils/HBaseManager.java | 5 +++-- .../hadoop/variant/AutoScaleHBaseTableTest.java | 7 +++++-- 6 files changed, 43 insertions(+), 4 deletions(-) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java index 8ffcf75747f..48ab8846f65 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java @@ -3,6 +3,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; @@ -33,4 +34,6 @@ public static HBaseCompatApi getInstance() { public abstract boolean isSolrTestingAvailable(); public abstract List getServerList(Admin admin) throws IOException; + + public abstract byte[][] getTableStartKeys(Admin admin, Table table) throws IOException; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index 8691e5721b0..bb74dabf4c6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -4,6 +4,9 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; @@ -33,4 +36,13 @@ public List getServerList(Admin admin) throws IOException { return new ArrayList<>(admin.getClusterStatus().getServers()); } + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + List regions = admin.getRegions(table.getName()); + regions.sort((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())); + byte[][] startKeys = new byte[regions.size()][]; + for (int i = 0; i < regions.size(); i++) { + startKeys[i] = regions.get(i).getStartKey(); + } + return startKeys; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index f249baffc3c..f7cf534508a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -4,6 +4,9 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; @@ -31,4 +34,15 @@ public PhoenixCompatApi getPhoenixCompat() { public List getServerList(Admin admin) throws IOException { return new ArrayList<>(admin.getClusterMetrics().getServersName()); } + + @Override + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + List regions = admin.getRegions(table.getName()); + regions.sort((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())); + byte[][] startKeys = new byte[regions.size()][]; + for (int i = 0; i < regions.size(); i++) { + startKeys[i] = regions.get(i).getStartKey(); + } + return startKeys; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index 062adc91618..194b47b7794 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -4,6 +4,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Table; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; @@ -32,4 +33,9 @@ public PhoenixCompatApi getPhoenixCompat() { public List getServerList(Admin admin) throws IOException { return new ArrayList<>(admin.getClusterMetrics().getServersName()); } + + @Override + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + return table.getRegionLocator().getStartKeys(); + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java index 3eef1fe3ced..8dc812e127f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.auth.HBaseCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -426,7 +427,7 @@ public boolean splitAndMove(Admin admin, TableName tableName, byte[] expectedSpl return false; } LOGGER.info("Moving region '{}' to another region server", regionInfo.getRegionNameAsString()); - admin.move(regionInfo.getEncodedNameAsBytes()); + admin.move(regionInfo.getEncodedNameAsBytes(), null); LOGGER.info("New region created '{}' in {}", regionInfo.getRegionNameAsString(), TimeUtils.durationToString(stopWatch)); return true; } catch (IOException | RuntimeException e) { @@ -487,7 +488,7 @@ public int expandTableIfNeeded(String tableName, Collection batches, // Ensure that the table is split at least until the next expected split return act(tableName, (table, admin) -> { int newSplits = 0; - byte[][] existingSplits = table.getRegionLocator().getStartKeys(); + byte[][] existingSplits = HBaseCompat.getInstance().getTableStartKeys(admin, table); int expectedNewSplits = 0; for (byte[] expectedSplit : expectedSplits) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java index 457e25bc240..ae3ddabd3fb 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java @@ -9,6 +9,7 @@ import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; @@ -97,7 +98,8 @@ private void checkArchiveTableSplits(int expectedSplits) throws Exception { String archiveTableName = engine.getArchiveTableName(studyId); HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); - int archiveNumRegions = hBaseManager.act(archiveTableName, table -> table.getRegionLocator().getStartKeys().length); + int archiveNumRegions = hBaseManager.act(archiveTableName, + (table, admin) -> HBaseCompat.getInstance().getTableStartKeys(admin, table).length); // numRegions == numSplits + 1 assertEquals(archiveTableName, expectedSplits + 1, archiveNumRegions); } @@ -107,7 +109,8 @@ private void checkSampleIndexTableSplits(int expectedSplits) throws Exception { String sampleIndexTableName = sampleIndexDBAdaptor.getSampleIndexTableName(studyId, 1); HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); - int sampleIndexNumRegions = hBaseManager.act(sampleIndexTableName, table -> table.getRegionLocator().getStartKeys().length); + int sampleIndexNumRegions = hBaseManager.act(sampleIndexTableName, + (table, admin) -> HBaseCompat.getInstance().getTableStartKeys(admin, table).length); // numRegions == numSplits + 1 assertEquals(sampleIndexTableName, expectedSplits + 1, sampleIndexNumRegions); } From 28d9ecd2f3ab301a237879a7f98f7b2fcc3575b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 16 May 2024 13:58:26 +0100 Subject: [PATCH 006/128] storage: Add variant-setup operation. #TASK-5861, #TASK-5448 --- .../manager/VariantStorageManager.java | 8 + .../VariantSetupOperationManager.java | 138 ++++++++++++++++++ .../manager/VariantOperationsTest.java | 20 ++- .../catalog/managers/StudyManager.java | 25 ++++ .../opencga/catalog/managers/UserManager.java | 19 +++ .../StudyVariantEngineConfiguration.java | 16 +- .../core/models/study/VariantSetupResult.java | 74 ++++++++++ .../models/variant/VariantSetupParams.java | 126 ++++++++++++++++ .../core/variant/VariantStorageEngine.java | 8 + .../storage/hadoop/utils/HBaseManager.java | 2 +- .../variant/HadoopVariantStorageEngine.java | 45 ++++++ .../variant/HadoopVariantStorageOptions.java | 8 +- .../variant/HadoopVariantStorageTest.java | 7 +- 13 files changed, 485 insertions(+), 11 deletions(-) create mode 100644 opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 847a2bb1ac6..565bb8048a4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -71,7 +71,9 @@ import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.VariantPruneParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.core.tools.ToolParams; @@ -474,6 +476,12 @@ public void aggregate(String studyStr, VariantAggregateParams params, String tok }); } + public VariantSetupResult variantSetup(String studyStr, VariantSetupParams params, String token) + throws CatalogException, StorageEngineException { + return secureOperation("variant-setup", studyStr, params.toObjectMap(), token, + engine -> new VariantSetupOperationManager(this, engine).setup(studyStr, params, token)); + } + public ObjectMap configureProject(String projectStr, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperationByProject("configure", projectStr, params, token, engine -> { DataStore dataStore = getDataStoreByProjectId(projectStr, token); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java new file mode 100644 index 00000000000..fd94b8d8f91 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -0,0 +1,138 @@ +package org.opencb.opencga.analysis.variant.manager.operations; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.IOUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; + +public class VariantSetupOperationManager extends OperationManager { + + + public VariantSetupOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine variantStorageEngine) { + super(variantStorageManager, variantStorageEngine); + } + + public VariantSetupResult setup(String studyStr, VariantSetupParams params, String token) throws CatalogException { + // Copy params to avoid modifying input object + params = new VariantSetupParams(params); + check(studyStr, params, token); + + VariantSetupResult result = new VariantSetupResult(); + result.setDate(TimeUtils.getTime()); + result.setUserId(catalogManager.getUserManager().getUserIdContextStudy(studyStr, token)); + result.setParams(params.toObjectMap()); + result.setStatus(VariantSetupResult.Status.READY); + + inferParams(params); + + ObjectMap options = variantStorageEngine.inferConfigurationParams(params); + result.setOptions(options); + + catalogManager.getStudyManager().setVariantEngineSetupOptions(studyStr, result, token); + + return result; + } + + /** + * Infer some parameters from others. + * - averageFileSize inferred from fileType + * - samplesPerFile inferred from dataDistribution or expectedSamplesNumber and expectedFilesNumber + * - numberOfVariantsPerSample inferred from fileType + * @param params params to infer + */ + private static void inferParams(VariantSetupParams params) { + if (params.getFileType() != null) { + switch (params.getFileType()) { + case GENOME_gVCF: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize(IOUtils.fromHumanReadableToByte("1GiB")); + } + if (params.getNumberOfVariantsPerSample() == null) { + params.setNumberOfVariantsPerSample(5000000); + } + break; + case GENOME_VCF: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize(IOUtils.fromHumanReadableToByte("400MiB")); + } + if (params.getNumberOfVariantsPerSample() == null) { + params.setNumberOfVariantsPerSample(5000000); + } + break; + case EXOME: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize(IOUtils.fromHumanReadableToByte("50MiB")); + } + if (params.getNumberOfVariantsPerSample() == null) { + params.setNumberOfVariantsPerSample(50000); + } + break; + default: + throw new IllegalArgumentException("Unknown fileType " + params.getFileType()); + } + } + // Unable to tell. Use a default value for numberOfVariantsPerSample + if (params.getNumberOfVariantsPerSample() == null) { + params.setNumberOfVariantsPerSample(5000000); + } + + if (params.getSamplesPerFile() == null) { + if (params.getDataDistribution() == null) { + params.setSamplesPerFile(params.getExpectedSamplesNumber().floatValue() / params.getExpectedFilesNumber().floatValue()); + } else { + switch (params.getDataDistribution()) { + case SINGLE_SAMPLE_FILES: + params.setSamplesPerFile(1f); + break; + case MULTI_SAMPLE_FILES: + params.setSamplesPerFile(params.getExpectedSamplesNumber().floatValue() / params.getExpectedFilesNumber().floatValue()); + break; + case MULTIPLE_FILE_PER_SAMPLE: + // Hard to tell. Let's assume 2 samples per file + params.setSamplesPerFile(2f); + break; + case MULTI_SAMPLE_FILES_SPLIT_BY_CHROMOSOME: + case MULTI_SAMPLE_FILES_SPLIT_BY_REGION: + params.setSamplesPerFile(params.getExpectedSamplesNumber().floatValue()); + break; + default: + throw new IllegalArgumentException("Unknown dataDistribution " + params.getDataDistribution()); + } + } + } + } + + private void check(String studyStr, VariantSetupParams params, String token) throws CatalogException { + Study study = catalogManager.getStudyManager().get(studyStr, + new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token) + .first(); + if (study.getInternal() != null + && study.getInternal().getConfiguration() != null + && study.getInternal().getConfiguration().getVariantEngine() != null) { + VariantSetupResult setup = study.getInternal().getConfiguration().getVariantEngine().getSetup(); + if (setup != null && setup.getStatus() == VariantSetupResult.Status.READY) { + throw new IllegalArgumentException("Study " + studyStr + " is already setup"); + // TODO: Allow double setup if no files are loaded + } + } + + if (params.getExpectedFilesNumber() == null || params.getExpectedFilesNumber() <= 0) { + throw new IllegalArgumentException("Missing expectedFilesNumber"); + } + if (params.getExpectedSamplesNumber() == null || params.getExpectedSamplesNumber() <= 0) { + throw new IllegalArgumentException("Missing expectedSamplesNumber"); + } + + if (params.getAverageFileSize() == null && params.getFileType() == null) { + throw new IllegalArgumentException("Missing averageFileSize or fileType"); + } + } + +} diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index d92fe79abfc..8ca2342c51a 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -55,7 +55,9 @@ import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; +import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.VariantIndexParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; @@ -71,6 +73,7 @@ import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.slf4j.LoggerFactory; import java.nio.file.Path; import java.nio.file.Paths; @@ -160,8 +163,8 @@ public void tearDown() { if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) engine.getDBAdaptor()), Paths.get(opencga.createTmpOutdir("_hbase_print_variants_AFTER")).toUri()); } - } catch (Exception ignore) { - ignore.printStackTrace(); + } catch (Exception e) { + LoggerFactory.getLogger(getClass()).error("Ignoring exception printing variants", e); } hadoopExternalResource.after(); @@ -313,6 +316,19 @@ public void setUpCatalogManager() throws Exception { } + @Test + public void testSetup() throws Exception { + VariantSetupParams setupParams = new VariantSetupParams(); + setupParams + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTI_SAMPLE_FILES) + .setExpectedFilesNumber(20) + .setExpectedSamplesNumber(100); + VariantSetupResult result = variantStorageManager.variantSetup(STUDY, setupParams, token); + System.out.println("result = " + result); + System.out.println(JacksonUtils.getDefaultObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(result)); + } + @Test public void testVariantFileReload() throws Exception { try { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 848cb8e5e2d..04eff764dcb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -1820,6 +1820,31 @@ public void setVariantEngineConfigurationOptions(String studyStr, ObjectMap opti getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); } + public void setVariantEngineSetupOptions(String studyStr, VariantSetupResult variantSetupResult, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), + StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key())), + tokenPayload); + + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + StudyVariantEngineConfiguration configuration = study.getInternal().getConfiguration().getVariantEngine(); + if (configuration == null) { + configuration = new StudyVariantEngineConfiguration(); + } + configuration.setSetup(variantSetupResult); + if (configuration.getOptions() == null) { + configuration.setOptions(new ObjectMap()); + } + configuration.getOptions().putAll(variantSetupResult.getOptions()); + + ObjectMap parameters = new ObjectMap(StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key(), configuration); + getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); + } + public void setVariantEngineConfigurationSampleIndex(String studyStr, SampleIndexConfiguration sampleIndexConfiguration, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index 3708e64c9ed..daa122a6ece 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -31,6 +31,7 @@ import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.CatalogIOManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -1366,4 +1367,22 @@ private String getValidUserId(String userId, JwtPayload payload) throws CatalogE } } + public String getUserId(String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + return tokenPayload.getUserId(); + } + + public String getUserIdContextProject(String projectStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + return tokenPayload.getUserId(organizationId); + } + + public String getUserIdContextStudy(String studyStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + return tokenPayload.getUserId(organizationId); + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java index 3bd986b2fb4..a0a47596cd1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.models.study; +import org.opencb.commons.annotations.DataField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; @@ -8,6 +9,9 @@ public class StudyVariantEngineConfiguration { private ObjectMap options; private SampleIndexConfiguration sampleIndex; + @DataField(description = "Variant setup run", since = "3.2.0") + private VariantSetupResult setup; + public StudyVariantEngineConfiguration() { } @@ -34,11 +38,21 @@ public StudyVariantEngineConfiguration setSampleIndex(SampleIndexConfiguration s return this; } + public VariantSetupResult getSetup() { + return setup; + } + + public StudyVariantEngineConfiguration setSetup(VariantSetupResult setup) { + this.setup = setup; + return this; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("StudyVariantEngineConfiguration{"); - // sb.append("options=").append(options != null ? options.toJson() : ""); + sb.append("options=").append(options != null ? options.toJson() : ""); sb.append(", sampleIndex=").append(sampleIndex); + sb.append(", setup=").append(setup); sb.append('}'); return sb.toString(); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java new file mode 100644 index 00000000000..a1127f873f8 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java @@ -0,0 +1,74 @@ +package org.opencb.opencga.core.models.study; + +import org.opencb.commons.annotations.DataField; +import org.opencb.commons.datastore.core.ObjectMap; + +public class VariantSetupResult { + + @DataField(description = "User ID that started the setup run") + private String userId; + @DataField(description = "Date when the variant setup was executed") + private String date; + @DataField(description = "Variant setup status") + private Status status; + @DataField(description = "Input params for the variant setup") + private ObjectMap params; + + @DataField(description = "Generated variant storage configuration options given the input params.") + private ObjectMap options; + + public enum Status { + READY, + NOT_READY + } + + public VariantSetupResult() { + } + + public String getUserId() { + return userId; + } + + public VariantSetupResult setUserId(String userId) { + this.userId = userId; + return this; + } + + public String getDate() { + return date; + } + + public VariantSetupResult setDate(String date) { + this.date = date; + return this; + } + + public Status getStatus() { + return status; + } + + public VariantSetupResult setStatus(Status status) { + this.status = status; + return this; + } + + public ObjectMap getParams() { + return params; + } + + public VariantSetupResult setParams(ObjectMap params) { + this.params = params; + return this; + } + + public ObjectMap getOptions() { + return options; + } + + public VariantSetupResult setOptions(ObjectMap options) { + this.options = options; + return this; + } + + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java new file mode 100644 index 00000000000..76870051795 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -0,0 +1,126 @@ +package org.opencb.opencga.core.models.variant; + +import org.opencb.opencga.core.tools.ToolParams; + +public class VariantSetupParams extends ToolParams { + + private Integer expectedSamplesNumber; + + private Integer expectedFilesNumber; + + private FileType fileType; + + private Long averageFileSize; + + private Integer numberOfVariantsPerSample; + + private Float samplesPerFile; + + private DataDistribution dataDistribution; + + public VariantSetupParams(VariantSetupParams params) { + this.expectedSamplesNumber = params.expectedSamplesNumber; + this.expectedFilesNumber = params.expectedFilesNumber; + this.fileType = params.fileType; + this.averageFileSize = params.averageFileSize; + this.numberOfVariantsPerSample = params.numberOfVariantsPerSample; + this.samplesPerFile = params.samplesPerFile; + this.dataDistribution = params.dataDistribution; + } + + public VariantSetupParams() { + } + + public enum DataDistribution { + // Single sample VCF files. One file per sample. + // e.g. + // - Platinum gVCF + // - Cancer germline + // - RD germline without family calling + SINGLE_SAMPLE_FILES, + // Multi samples VCF files. One file with multiple samples. + // e.g. + // - Corpasome + // - RD germline with family calling + MULTI_SAMPLE_FILES, + // Multiple files per sample. Each file might have multiple samples. + // e.g. + // - Somatic study with multiple callers + MULTIPLE_FILE_PER_SAMPLE, + // Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. + // e.g. + // - 1000 genomes + MULTI_SAMPLE_FILES_SPLIT_BY_CHROMOSOME, + // Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific region. + MULTI_SAMPLE_FILES_SPLIT_BY_REGION, + } + + public enum FileType { + GENOME_VCF, + GENOME_gVCF, + EXOME + } + + public Integer getExpectedSamplesNumber() { + return expectedSamplesNumber; + } + + public VariantSetupParams setExpectedSamplesNumber(Integer expectedSamplesNumber) { + this.expectedSamplesNumber = expectedSamplesNumber; + return this; + } + + public Integer getExpectedFilesNumber() { + return expectedFilesNumber; + } + + public VariantSetupParams setExpectedFilesNumber(Integer expectedFilesNumber) { + this.expectedFilesNumber = expectedFilesNumber; + return this; + } + + public FileType getFileType() { + return fileType; + } + + public VariantSetupParams setFileType(FileType fileType) { + this.fileType = fileType; + return this; + } + + public Long getAverageFileSize() { + return averageFileSize; + } + + public VariantSetupParams setAverageFileSize(Long averageFileSize) { + this.averageFileSize = averageFileSize; + return this; + } + + public Integer getNumberOfVariantsPerSample() { + return numberOfVariantsPerSample; + } + + public VariantSetupParams setNumberOfVariantsPerSample(Integer numberOfVariantsPerSample) { + this.numberOfVariantsPerSample = numberOfVariantsPerSample; + return this; + } + + public Float getSamplesPerFile() { + return samplesPerFile; + } + + public VariantSetupParams setSamplesPerFile(Float samplesPerFile) { + this.samplesPerFile = samplesPerFile; + return this; + } + + public DataDistribution getDataDistribution() { + return dataDistribution; + } + + public VariantSetupParams setDataDistribution(DataDistribution dataDistribution) { + this.dataDistribution = dataDistribution; + return this; + } +} diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index a3ab7e539b5..74d075f2fc3 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -31,6 +31,7 @@ import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngine; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -1404,6 +1405,13 @@ public VariantAggregationExecutor getVariantAggregationExecutor(Query query, Que throw new VariantQueryException("No VariantAggregationExecutor found to run the query. " + messages).setQuery(query); } + public ObjectMap inferConfigurationParams(VariantSetupParams params) { + ObjectMap options = new ObjectMap(); + + + return options; + } + @Override public void close() throws IOException { cellBaseUtils = null; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java index 8dc812e127f..e38099c1060 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java @@ -427,7 +427,7 @@ public boolean splitAndMove(Admin admin, TableName tableName, byte[] expectedSpl return false; } LOGGER.info("Moving region '{}' to another region server", regionInfo.getRegionNameAsString()); - admin.move(regionInfo.getEncodedNameAsBytes(), null); + admin.move(regionInfo.getEncodedNameAsBytes(), (byte[]) null); LOGGER.info("New region created '{}' in {}", regionInfo.getRegionNameAsString(), TimeUtils.durationToString(stopWatch)); return true; } catch (IOException | RuntimeException e) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index ff93b19fe54..eac6daddc60 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -32,6 +32,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.DatabaseCredentials; @@ -39,6 +40,7 @@ import org.opencb.opencga.core.config.storage.StorageEngineConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.StoragePipelineException; @@ -1084,6 +1086,49 @@ protected List initVariantAggregationExecutors() { return executors; } + @Override + public ObjectMap inferConfigurationParams(VariantSetupParams params) { + ObjectMap options = super.inferConfigurationParams(params); + ObjectMap configuredOptions = getOptions(); + + long expectedHBaseRegionSize = IOUtils.fromHumanReadableToByte("7.5GiB"); + + options.put(EXPECTED_SAMPLES_NUMBER.key(), params.getExpectedSamplesNumber()); + options.put(EXPECTED_FILES_NUMBER.key(), params.getExpectedFilesNumber()); + + // Variant pre-split + int defaultVariantPreSplit = configuredOptions + .getInt(VARIANT_TABLE_PRESPLIT_SIZE.key(), VARIANT_TABLE_PRESPLIT_SIZE.defaultValue()); + float variantsFileToHBaseMultiplier = 1.3f; + float variantsTableSize = params.getExpectedFilesNumber() * params.getAverageFileSize() * variantsFileToHBaseMultiplier; + int variantPreSplit = (int) (variantsTableSize / expectedHBaseRegionSize); + options.put(VARIANT_TABLE_PRESPLIT_SIZE.key(), Math.max(defaultVariantPreSplit, variantPreSplit)); + + // Archive pre-split + Long averageFileSize = params.getAverageFileSize(); + int filesPerBatch = configuredOptions + .getInt(ARCHIVE_FILE_BATCH_SIZE.key(), ARCHIVE_FILE_BATCH_SIZE.defaultValue()); + float archiveFileToHBaseMultiplier = 1.2f; + float archiveTableSize = filesPerBatch * averageFileSize.floatValue() * archiveFileToHBaseMultiplier; + int archiveTablePreSplit = (int) (archiveTableSize / expectedHBaseRegionSize); + options.put(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), archiveTablePreSplit); + + // SampleIndex pre-split + long averageSizePerVariant; + if (params.getNumberOfVariantsPerSample() > 3500000) { + // With this many variants per sample, most of them won't have much data + averageSizePerVariant = IOUtils.fromHumanReadableToByte("13B"); + } else { + // With a small number of variants per sample, most of them will have a lot of data + averageSizePerVariant = IOUtils.fromHumanReadableToByte("25B"); + } + long sampleIndexSize = params.getNumberOfVariantsPerSample() * averageSizePerVariant; + int samplesPerSplit = (int) (expectedHBaseRegionSize / sampleIndexSize); + options.put(SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), samplesPerSplit); + + return options; + } + @Override public void close() throws IOException { super.close(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java index 272daaa4d7e..bd867c80be3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java @@ -8,7 +8,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { HADOOP_LOAD_FILES_IN_PARALLEL("storage.hadoop.load.filesInParallel", 1), HBASE_NAMESPACE("storage.hadoop.hbase.namespace"), - EXPECTED_FILES_NUMBER("expected_files_number", 5000), + EXPECTED_FILES_NUMBER("expected_files_number", 50), EXPECTED_SAMPLES_NUMBER("expected_samples_number"), DBADAPTOR_PHOENIX_FETCH_SIZE("storage.hadoop.phoenix.fetchSize", -1), DBADAPTOR_PHOENIX_QUERY_COMPLEXITY_THRESHOLD("storage.hadoop.phoenix.queryComplexityThreshold", 250), @@ -63,7 +63,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Variant table configuration ///////////////////////// VARIANT_TABLE_COMPRESSION("storage.hadoop.variant.table.compression", Compression.Algorithm.SNAPPY.getName()), - VARIANT_TABLE_PRESPLIT_SIZE("storage.hadoop.variant.table.preSplit.numSplits", 500), + VARIANT_TABLE_PRESPLIT_SIZE("storage.hadoop.variant.table.preSplit.numSplits", 50), // Do not create phoenix indexes. Testing purposes only VARIANT_TABLE_INDEXES_SKIP("storage.hadoop.variant.table.indexes.skip"), VARIANT_TABLE_LOAD_REFERENCE("storage.hadoop.variant.table.load.reference", false), @@ -77,7 +77,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Archive table configuration ///////////////////////// ARCHIVE_TABLE_COMPRESSION("storage.hadoop.archive.table.compression", Compression.Algorithm.GZ.getName()), - ARCHIVE_TABLE_PRESPLIT_SIZE("storage.hadoop.archive.table.preSplit.splitsPerBatch", 500), + ARCHIVE_TABLE_PRESPLIT_SIZE("storage.hadoop.archive.table.preSplit.splitsPerBatch", 10), ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.archive.table.preSplit.extraSplits", 3), ARCHIVE_CHUNK_SIZE("storage.hadoop.archive.table.chunkSize", 1000), @@ -92,7 +92,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Sample index table configuration ///////////////////////// SAMPLE_INDEX_TABLE_COMPRESSION("storage.hadoop.sampleIndex.table.compression", Compression.Algorithm.SNAPPY.getName()), - SAMPLE_INDEX_TABLE_PRESPLIT_SIZE("storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit", 15), + SAMPLE_INDEX_TABLE_PRESPLIT_SIZE("storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit", 200), SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.sampleIndex.table.preSplit.extraSplits", 5), SAMPLE_INDEX_BUILD_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.build.maxSamplesPerMR", 2000), SAMPLE_INDEX_ANNOTATION_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.annotation.maxSamplesPerMR", 2000), diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java index 787ea7c4412..9d1725e5356 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java @@ -232,8 +232,8 @@ public void before() throws Exception { Configurator.setLevel(MapTask.class.getName(), Level.WARN); Configurator.setLevel(TableInputFormatBase.class.getName(), Level.WARN); - utility.set(new HBaseTestingUtility()); - Configuration conf = utility.get().getConfiguration(); + HBaseTestingUtility testingUtility = new HBaseTestingUtility(); + Configuration conf = testingUtility.getConfiguration(); HadoopVariantStorageTest.configuration.set(conf); @@ -277,7 +277,8 @@ public void before() throws Exception { } //org.apache.commons.configuration2.Configuration - utility.get().startMiniCluster(1); + testingUtility.startMiniCluster(1); + utility.set(testingUtility); // MiniMRCluster miniMRCluster = utility.startMiniMapReduceCluster(); // MiniMRClientCluster miniMRClientCluster = MiniMRClientClusterFactory.create(HadoopVariantStorageManagerTestUtils.class, 1, configuration); From 17955cf6f2ba7f18dc8eb92b2771d03d344f0dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 16 May 2024 16:06:24 +0100 Subject: [PATCH 007/128] storage: Make variant-setup mandatory for any operation. #TASK-5861, #TASK-5448 --- .../manager/VariantStorageManager.java | 27 ++++++++--- .../VariantSetupOperationManager.java | 48 ++++++++++++++----- .../analysis/variant/VariantAnalysisTest.java | 4 ++ .../manager/VariantOperationsTest.java | 41 ++++++++++++++-- .../manager/VariantStorageManagerTest.java | 15 ++++++ .../AbstractVariantOperationManagerTest.java | 5 ++ .../storage/hadoop/utils/HBaseManager.java | 24 +++------- .../variant/HadoopVariantStorageEngine.java | 2 +- 8 files changed, 126 insertions(+), 40 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 565bb8048a4..efc92cd62f2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -478,8 +478,14 @@ public void aggregate(String studyStr, VariantAggregateParams params, String tok public VariantSetupResult variantSetup(String studyStr, VariantSetupParams params, String token) throws CatalogException, StorageEngineException { - return secureOperation("variant-setup", studyStr, params.toObjectMap(), token, - engine -> new VariantSetupOperationManager(this, engine).setup(studyStr, params, token)); + return secureOperation(VariantSetupOperationManager.ID, studyStr, params.toObjectMap(), token, + engine -> new VariantSetupOperationManager(this, engine).setup(getStudyFqn(studyStr, token), params, token)); + } + + public boolean hasVariantSetup(String studyStr, String token) throws CatalogException { + Study study = catalogManager.getStudyManager().get(studyStr, + new QueryOptions(INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token).first(); + return VariantSetupOperationManager.hasVariantSetup(study); } public ObjectMap configureProject(String projectStr, ObjectMap params, String token) throws CatalogException, StorageEngineException { @@ -1181,7 +1187,7 @@ private interface VariantOperationFunction { private R secureOperationByProject(String operationName, String project, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineByProject(project, params, token)) { - return secureTool(operationName, true, params, token, variantStorageEngine, operation); + return secureTool(operationName, true, null, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1190,7 +1196,7 @@ private R secureOperationByProject(String operationName, String project, Obj private R secureOperation(String operationName, String study, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) { - return secureTool(operationName, true, params, token, variantStorageEngine, operation); + return secureTool(operationName, true, study, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1199,7 +1205,7 @@ private R secureOperation(String operationName, String study, ObjectMap para private R secureAnalysis(String operationName, String study, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) { - return secureTool(operationName, false, params, token, variantStorageEngine, operation); + return secureTool(operationName, false, study, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1221,7 +1227,7 @@ private R secureOperationByProject(String operationName, String projectStr, return secureOperationByProject(operationName, projectStr, params, token, operation); } - private R secureTool(String toolId, boolean isOperation, ObjectMap params, String token, + private R secureTool(String toolId, boolean isOperation, String study, ObjectMap params, String token, VariantStorageEngine variantStorageEngine, VariantOperationFunction operation) throws CatalogException, StorageEngineException { @@ -1241,6 +1247,15 @@ private R secureTool(String toolId, boolean isOperation, ObjectMap params, S throw new StorageEngineException("Unable to execute operation '" + toolId + "'. " + "The storage engine is in mode=" + storageConfiguration.getMode()); } + if (isOperation && study != null && !VariantSetupOperationManager.ID.equals(toolId)) { + // Ensure that the variant setup has been executed + // do not check for the setup operation itself + // Project level operations can not be checked for setup. + if (!hasVariantSetup(study, token)) { + throw new StorageEngineException("Unable to execute operation '" + toolId + "'. " + + "The variant storage has not been setup for study '" + study + "'"); + } + } result = operation.apply(variantStorageEngine); return result; } catch (CatalogException | StorageEngineException e) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java index fd94b8d8f91..02ed3d916ff 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -10,23 +10,28 @@ import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.VariantSetupParams; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; public class VariantSetupOperationManager extends OperationManager { + public static final String ID = "variant-setup"; + public VariantSetupOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine variantStorageEngine) { super(variantStorageManager, variantStorageEngine); } - public VariantSetupResult setup(String studyStr, VariantSetupParams params, String token) throws CatalogException { + public VariantSetupResult setup(String studyFqn, VariantSetupParams params, String token) + throws CatalogException, StorageEngineException { // Copy params to avoid modifying input object params = new VariantSetupParams(params); - check(studyStr, params, token); + check(studyFqn, params, token); VariantSetupResult result = new VariantSetupResult(); result.setDate(TimeUtils.getTime()); - result.setUserId(catalogManager.getUserManager().getUserIdContextStudy(studyStr, token)); + result.setUserId(catalogManager.getUserManager().getUserIdContextStudy(studyFqn, token)); result.setParams(params.toObjectMap()); result.setStatus(VariantSetupResult.Status.READY); @@ -35,7 +40,7 @@ public VariantSetupResult setup(String studyStr, VariantSetupParams params, Stri ObjectMap options = variantStorageEngine.inferConfigurationParams(params); result.setOptions(options); - catalogManager.getStudyManager().setVariantEngineSetupOptions(studyStr, result, token); + catalogManager.getStudyManager().setVariantEngineSetupOptions(studyFqn, result, token); return result; } @@ -47,7 +52,7 @@ public VariantSetupResult setup(String studyStr, VariantSetupParams params, Stri * - numberOfVariantsPerSample inferred from fileType * @param params params to infer */ - private static void inferParams(VariantSetupParams params) { + private void inferParams(VariantSetupParams params) { if (params.getFileType() != null) { switch (params.getFileType()) { case GENOME_gVCF: @@ -109,19 +114,23 @@ private static void inferParams(VariantSetupParams params) { } } - private void check(String studyStr, VariantSetupParams params, String token) throws CatalogException { + private void check(String studyStr, VariantSetupParams params, String token) throws CatalogException, StorageEngineException { Study study = catalogManager.getStudyManager().get(studyStr, new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token) .first(); - if (study.getInternal() != null - && study.getInternal().getConfiguration() != null - && study.getInternal().getConfiguration().getVariantEngine() != null) { - VariantSetupResult setup = study.getInternal().getConfiguration().getVariantEngine().getSetup(); - if (setup != null && setup.getStatus() == VariantSetupResult.Status.READY) { - throw new IllegalArgumentException("Study " + studyStr + " is already setup"); - // TODO: Allow double setup if no files are loaded + + VariantStorageMetadataManager metadataManager = variantStorageEngine.getMetadataManager(); + if (metadataManager.studyExists(studyStr)) { + int studyId = metadataManager.getStudyId(studyStr); + if (!metadataManager.getIndexedFiles(studyId).isEmpty()) { + throw new IllegalArgumentException("Unable to execute variant-setup on study '" + studyStr + "'. " + + "It already has indexed files."); } } + if (hasVariantSetup(study)) { + // TODO: Allow double setup if no files are loaded? + throw new IllegalArgumentException("Study " + studyStr + " is already setup"); + } if (params.getExpectedFilesNumber() == null || params.getExpectedFilesNumber() <= 0) { throw new IllegalArgumentException("Missing expectedFilesNumber"); @@ -135,4 +144,17 @@ private void check(String studyStr, VariantSetupParams params, String token) thr } } + public static boolean hasVariantSetup(Study study) { + boolean hasSetup = false; + if (study.getInternal() != null + && study.getInternal().getConfiguration() != null + && study.getInternal().getConfiguration().getVariantEngine() != null) { + VariantSetupResult setup = study.getInternal().getConfiguration().getVariantEngine().getSetup(); + if (setup != null && setup.getStatus() == VariantSetupResult.Status.READY) { + hasSetup = true; + } + } + return hasSetup; + } + } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index 59d2ba43d5d..6119a5a8ea0 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -44,6 +44,7 @@ import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool; @@ -178,6 +179,9 @@ public void setUp() throws Throwable { setUpCatalogManager(); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, STUDY, token); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, CANCER_STUDY, token); + file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token); variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index 8ca2342c51a..f59be01a3da 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.variant.manager; import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.*; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -32,6 +33,7 @@ import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; import org.opencb.opencga.analysis.variant.operations.*; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; @@ -231,6 +233,8 @@ private void loadDataset() throws Throwable { solrExternalResource.configure(variantStorageManager.getVariantStorageEngineForStudyOperation(STUDY, new ObjectMap(), token)); } + dummyVariantSetup(variantStorageManager, STUDY, token); + file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token); // variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token); toolRunner.execute(VariantIndexOperationTool.class, STUDY, @@ -292,6 +296,15 @@ private void loadDataset() throws Throwable { } } + public static void dummyVariantSetup(VariantStorageManager variantStorageManager, String study, String token) + throws CatalogException, StorageEngineException { + variantStorageManager.variantSetup(study, new VariantSetupParams() + .setAverageFileSize(100L) + .setExpectedFilesNumber(5) + .setExpectedSamplesNumber(5) + .setNumberOfVariantsPerSample(1000), token); + } + public void setUpCatalogManager() throws Exception { catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), opencga.getAdminToken()); @@ -324,9 +337,31 @@ public void testSetup() throws Exception { .setDataDistribution(VariantSetupParams.DataDistribution.MULTI_SAMPLE_FILES) .setExpectedFilesNumber(20) .setExpectedSamplesNumber(100); - VariantSetupResult result = variantStorageManager.variantSetup(STUDY, setupParams, token); - System.out.println("result = " + result); - System.out.println(JacksonUtils.getDefaultObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(result)); + String study2 = "study2"; + catalogManager.getStudyManager().create(PROJECT, study2, null, "Phase 1", "Done", null, null, null, null, null, token); + + try { + toolRunner.execute(VariantIndexOperationTool.class, study2, + new VariantIndexParams() + .setFile(file.getId()) + .setAnnotate(false) + .setLoadHomRef(YesNoAuto.YES.name()), + Paths.get(opencga.createTmpOutdir("_index")), "index", token); + fail("Should have thrown an exception"); + } catch (ToolException e) { + MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); + } + + VariantSetupResult result = variantStorageManager.variantSetup(study2, setupParams, token); + assertEquals(VariantSetupResult.Status.READY, result.getStatus()); + + try { + variantStorageManager.variantSetup(STUDY, setupParams, token); + fail("Should fail"); + } catch (Exception e) { + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Unable to execute variant-setup on study")); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("It already has indexed files.")); + } } @Test diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java index 0a5ac63293d..4aeedde871f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java @@ -17,12 +17,15 @@ package org.opencb.opencga.analysis.variant.manager; import org.apache.commons.lang3.RandomStringUtils; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.manager.operations.AbstractVariantOperationManagerTest; +import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.models.file.File; @@ -64,6 +67,18 @@ public void testConfigure() throws CatalogException, StorageEngineException { variantManager.configureStudy(studyFqn, expectedStudyConfiguration1, sessionId); variantManager.configureStudy(studyId2, expectedStudyConfiguration2, sessionId); + + try { + Study study = catalogManager.getStudyManager().create(projectId, "s_no_setup", "s_no_setup", "s_no_setup", + "Study 1", null, null, null, Collections.singletonMap(VariantStatsAnalysis.STATS_AGGREGATION_CATALOG, getAggregation()), null, sessionId) + .first(); + // Variant setup mandatory for configuring study + variantManager.configureStudy(study.getFqn(), expectedStudyConfiguration1, sessionId); + fail("Expect exception. Study not setup"); + } catch (Exception e) { + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); + } + ObjectMap configuration = variantManager.getDataStoreByProjectId(projectId, sessionId).getOptions(); assertEquals(expectedConfiguration, configuration); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java index d1dcd2e9353..f0d9ba16e96 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java @@ -27,6 +27,7 @@ import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.analysis.variant.operations.OperationTool; import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; @@ -170,6 +171,10 @@ public final void setUpAbstract() throws Exception { true, null, QueryOptions.empty(), sessionId).first().getId(); files = Arrays.asList(new File[5]); + + + VariantOperationsTest.dummyVariantSetup(variantManager, studyFqn, sessionId); + VariantOperationsTest.dummyVariantSetup(variantManager, studyId2, sessionId); } @After diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java index e38099c1060..5152ea92c11 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java @@ -32,11 +32,13 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.stream.Collectors; import static org.opencb.opencga.storage.hadoop.utils.PersistentResultScanner.isValid; @@ -488,18 +490,13 @@ public int expandTableIfNeeded(String tableName, Collection batches, // Ensure that the table is split at least until the next expected split return act(tableName, (table, admin) -> { int newSplits = 0; - byte[][] existingSplits = HBaseCompat.getInstance().getTableStartKeys(admin, table); + Set existingSplits = Arrays.stream(HBaseCompat.getInstance().getTableStartKeys(admin, table)) + .map(ByteBuffer::wrap) + .collect(Collectors.toSet()); int expectedNewSplits = 0; for (byte[] expectedSplit : expectedSplits) { - boolean found = false; - for (byte[] split : existingSplits) { - if (Bytes.compareTo(expectedSplit, split) == 0) { - found = true; - break; - } - } - if (!found) { + if (!existingSplits.contains(ByteBuffer.wrap(expectedSplit))) { expectedNewSplits++; LOGGER.info("Missing split point '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); } @@ -511,14 +508,7 @@ public int expandTableIfNeeded(String tableName, Collection batches, } for (byte[] expectedSplit : expectedSplits) { - boolean found = false; - for (byte[] split : existingSplits) { - if (Bytes.compareTo(expectedSplit, split) == 0) { - found = true; - break; - } - } - if (!found) { + if (!existingSplits.contains(ByteBuffer.wrap(expectedSplit))) { if (splitAndMove(admin, table.getName(), expectedSplit)) { newSplits++; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index eac6daddc60..80e3ebf5cc6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -1111,7 +1111,7 @@ public ObjectMap inferConfigurationParams(VariantSetupParams params) { float archiveFileToHBaseMultiplier = 1.2f; float archiveTableSize = filesPerBatch * averageFileSize.floatValue() * archiveFileToHBaseMultiplier; int archiveTablePreSplit = (int) (archiveTableSize / expectedHBaseRegionSize); - options.put(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), archiveTablePreSplit); + options.put(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), Math.max(1, archiveTablePreSplit)); // SampleIndex pre-split long averageSizePerVariant; From 55d56558718d278dadf0a310fbd94a0e0101e483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 17 May 2024 17:04:02 +0100 Subject: [PATCH 008/128] storage: Change some default variant-setup values. #TASK-5861, #TASK-5448 --- .../manager/operations/VariantSetupOperationManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java index 02ed3d916ff..989ee262807 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -65,7 +65,7 @@ private void inferParams(VariantSetupParams params) { break; case GENOME_VCF: if (params.getAverageFileSize() == null) { - params.setAverageFileSize(IOUtils.fromHumanReadableToByte("400MiB")); + params.setAverageFileSize(IOUtils.fromHumanReadableToByte("500MiB")); } if (params.getNumberOfVariantsPerSample() == null) { params.setNumberOfVariantsPerSample(5000000); @@ -73,10 +73,10 @@ private void inferParams(VariantSetupParams params) { break; case EXOME: if (params.getAverageFileSize() == null) { - params.setAverageFileSize(IOUtils.fromHumanReadableToByte("50MiB")); + params.setAverageFileSize(IOUtils.fromHumanReadableToByte("100MiB")); } if (params.getNumberOfVariantsPerSample() == null) { - params.setNumberOfVariantsPerSample(50000); + params.setNumberOfVariantsPerSample(100000); } break; default: From b49d9e16ffedeedf576530359c5fa3e4af662759 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 20 May 2024 19:06:22 +0200 Subject: [PATCH 009/128] pom: Release 3.1.0 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-compat-api/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.0/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.2/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.4/pom.xml | 2 +- .../opencga-storage-hadoop-compat/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.13/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdi5.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 2 +- 27 files changed, 27 insertions(+), 27 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index d43f173b4ff..a82fde15ccc 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index 67bd1fa34e0..6dcf364e559 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 11535022d1b..ff10c269d7f 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index c9461e9c4de..ce253ddc6ae 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index adb57573b45..215d392ce1c 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index 0153566efea..432db00c834 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 3cbbbb2fa89..eda14c31ea1 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 804b83e792b..7632ed6d65f 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index a31567cb3bc..cc692c6fcea 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index ff97332afff..bae503e0e91 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 12c0f24a27b..0dffe6806ff 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml index e63a1dfcc78..deac888af82 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml index 85caa7da48e..661d9416a8f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml index 719b5cf072a..358590f69f6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml index a8a16269d93..394e6c90e16 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml index a2d228c120d..58aac8414c5 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index eab3b7f565e..8c00149ba90 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml index 1292ba96965..45370142854 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml index 4b4ab70de2b..82a3d1f0fdc 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml index b9f0fcc7696..c4839e1c646 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml index d48ef593c4d..8d09832b15a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml index cddf51ec925..0326f2479f3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 943b2ff3eb8..7d97161a5e3 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 891225e9311..054ab3bc9de 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index f89a8783b7f..c3f5166b7fd 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 70f7e126a02..12981fefe10 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 1b5cc6907e7..09ddd09943d 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.1.1-SNAPSHOT + 3.1.0-SNAPSHOT pom OpenCGA From 0e9dda91f9397049ae683f047498eb6fe02d42fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 22 May 2024 16:02:09 +0100 Subject: [PATCH 010/128] app: Add migration VariantSetupMigration. Add migration test. #TASK-5861, #TASK-5448 --- .../variant/OpenCGATestExternalResource.java | 32 +++- opencga-app/pom.xml | 44 +++++ .../app/migrations/StorageMigrationTool.java | 6 +- .../v3_2_0/VariantSetupMigration.java | 47 ++++++ .../app/migrations/MigrationsTest.java | 84 ++++++++++ .../v3.0.0/mongodb/opencga/audit.json.gz | Bin 0 -> 3707 bytes .../v3.0.0/mongodb/opencga/file.json.gz | Bin 0 -> 1038 bytes .../v3.0.0/mongodb/opencga/migration.json.gz | Bin 0 -> 372 bytes .../v3.0.0/mongodb/opencga/note.json.gz | Bin 0 -> 464 bytes .../mongodb/opencga/organization.json.gz | Bin 0 -> 560 bytes .../v3.0.0/mongodb/opencga/project.json.gz | Bin 0 -> 506 bytes .../v3.0.0/mongodb/opencga/study.json.gz | Bin 0 -> 3896 bytes .../v3.0.0/mongodb/opencga/user.json.gz | Bin 0 -> 397 bytes .../opencga/v3.0.0/mongodb/test/audit.json.gz | Bin 0 -> 12501 bytes .../v3.0.0/mongodb/test/cohort.json.gz | Bin 0 -> 451 bytes .../opencga/v3.0.0/mongodb/test/file.json.gz | Bin 0 -> 2659 bytes .../v3.0.0/mongodb/test/individual.json.gz | Bin 0 -> 897 bytes .../v3.0.0/mongodb/test/migration.json.gz | Bin 0 -> 368 bytes .../v3.0.0/mongodb/test/organization.json.gz | Bin 0 -> 556 bytes .../v3.0.0/mongodb/test/project.json.gz | Bin 0 -> 662 bytes .../v3.0.0/mongodb/test/sample.json.gz | Bin 0 -> 2154 bytes .../opencga/v3.0.0/mongodb/test/study.json.gz | Bin 0 -> 5240 bytes .../opencga/v3.0.0/mongodb/test/user.json.gz | Bin 0 -> 565 bytes .../catalog/db/mongodb/MongoBackupUtils.java | 154 +++++++++++++++--- .../catalog/managers/AbstractManagerTest.java | 4 +- .../CatalogManagerExternalResource.java | 1 + .../dummy/DummyVariantStorageEngine.java | 4 + 27 files changed, 341 insertions(+), 35 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java create mode 100644 opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/migration.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/project.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/user.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/audit.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 1f05dfc5dd3..17356322406 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -24,6 +24,7 @@ import org.opencb.opencga.analysis.StorageManager; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; @@ -39,20 +40,20 @@ import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; +import org.reflections.Reflections; +import org.reflections.scanners.ResourcesScanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.net.URI; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Created on 26/08/15 @@ -72,7 +73,8 @@ public class OpenCGATestExternalResource extends ExternalResource { private ToolRunner toolRunner; - public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource = new HadoopVariantStorageTest.HadoopExternalResource(); + public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource + = new HadoopVariantStorageTest.HadoopExternalResource(); public OpenCGATestExternalResource() { this(false); @@ -236,6 +238,9 @@ public Path isolateOpenCGA() throws IOException { StorageEngineFactory.configure(storageConfiguration); storageEngineFactory = StorageEngineFactory.get(storageConfiguration); + if (storageEngine.equals(DummyVariantStorageEngine.STORAGE_ENGINE_ID)) { + DummyVariantStorageEngine.configure(getStorageEngineFactory(), true); + } // inputStream = StorageEngine.class.getClassLoader().getResourceAsStream("client-configuration-test.yml"); // Files.copy(inputStream, conf.resolve("client-configuration.yml"), StandardCopyOption.REPLACE_EXISTING); @@ -359,6 +364,23 @@ public String createTmpOutdir(String suffix) throws IOException { // return getCatalogManager().getJobManager().createJobOutDir(studyId, "I_tmp_" + date + sufix, sessionId).toString(); } + public void restore(URL resource) throws Exception { + if (resource.getProtocol().equals("jar")) { + Reflections reflections = new Reflections(resource.getPath().replace('/','.'), new ResourcesScanner()); + Set resources = reflections.getResources(x -> true); + for (String file : resources) { + catalogManagerExternalResource.getResourceUri(file.replace('.', '/')); + } + MongoBackupUtils.restore(getCatalogManager(), opencgaHome, opencgaHome + .resolve("resources") + .resolve(resource.getPath()) + .resolve("mongodb")); + } else { + MongoBackupUtils.restore(getCatalogManager(), opencgaHome, Paths.get(resource.toURI()).resolve("mongodb")); + } + catalogManagerExternalResource.resetCatalogManager(); + } + // private class StorageLocalExecutorManager extends LocalExecutorManager { // // public StorageLocalExecutorManager(String sessionId) { diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index 6dcf364e559..def07ce89b2 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -239,12 +239,56 @@ jline-terminal-jna runtime + + org.opencb.opencga opencga-storage-core test-jar test + + org.opencb.opencga + opencga-storage-hadoop-core + test + + + org.opencb.opencga + opencga-storage-hadoop-core + test + test-jar + + + org.opencb.opencga + opencga-storage-hadoop-compat-${opencga-storage-hadoop-compat.id} + ${project.parent.version} + test + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + test + + + org.yaml + snakeyaml + + + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + test + test-jar + + + org.yaml + snakeyaml + + + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java index c9aeee6c7e2..4952f9d9812 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java @@ -46,10 +46,10 @@ protected final VariantStorageEngine getVariantStorageEngineByProject(String pro * @return List of projects * @throws Exception on error */ - protected final List getVariantStorageProjects(String organizationId) throws Exception { + protected final List getVariantStorageProjects() throws Exception { Set projects = new LinkedHashSet<>(); - for (String studyFqn : getVariantStorageStudies(organizationId)) { + for (String studyFqn : getVariantStorageStudies()) { projects.add(catalogManager.getStudyManager().getProjectFqn(studyFqn)); } @@ -61,7 +61,7 @@ protected final List getVariantStorageProjects(String organizationId) th * @return List of projects * @throws Exception on error */ - protected final List getVariantStorageStudies(String organizationId) throws Exception { + protected final List getVariantStorageStudies() throws Exception { Set studies = new LinkedHashSet<>(); VariantStorageManager variantStorageManager = getVariantStorageManager(); for (Study study : catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), new QueryOptions(QueryOptions.INCLUDE, diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java new file mode 100644 index 00000000000..9d02073d099 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java @@ -0,0 +1,47 @@ +package org.opencb.opencga.app.migrations.v3_2_0; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; + +import java.util.LinkedHashSet; + +@Migration(id = "variant_setup", description = "Add a dummy variant setup for studies with data", version = "3.2.0", + domain = Migration.MigrationDomain.STORAGE, date = 20240516) +public class VariantSetupMigration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + VariantStorageManager variantStorageManager = getVariantStorageManager(); + for (String study : getVariantStorageStudies()) { + logger.info("--- Checking study '{}'", study); + if (variantStorageManager.hasVariantSetup(study, token)) { + logger.info("Study '{}' already has a variant setup", study); + continue; + } + + String projectFqn = catalogManager.getStudyManager().getProjectFqn(study); + VariantStorageMetadataManager metadataManager = getVariantStorageEngineByProject(projectFqn).getMetadataManager(); + int studyId = metadataManager.getStudyId(study); + LinkedHashSet indexedFiles = metadataManager.getIndexedFiles(studyId); + if (indexedFiles.isEmpty()) { + logger.info("Study '{}' does not have any indexed files. Skipping variant setup", study); + continue; + } + logger.info("Study '{}' doesn't have a variant setup, but it has {} indexed files. Creating a dummy variant setup", + study, indexedFiles.size()); + logger.info("Creating a dummy variant setup for study '{}'", study); + VariantSetupResult dummy = new VariantSetupResult(); + dummy.setDate(TimeUtils.getTime()); + dummy.setUserId(catalogManager.getUserManager().getUserId(token)); + dummy.setParams(new ObjectMap("executed_from_migration", getId())); + dummy.setStatus(VariantSetupResult.Status.READY); + dummy.setOptions(new ObjectMap()); + catalogManager.getStudyManager().setVariantEngineSetupOptions(study, dummy, token); + } + } +} diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java new file mode 100644 index 00000000000..70ca0549fd3 --- /dev/null +++ b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java @@ -0,0 +1,84 @@ +package org.opencb.opencga.app.migrations; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.app.migrations.v3_2_0.VariantSetupMigration; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageEngine; + +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; + +@Category(LongTests.class) +public class MigrationsTest { + + public OpenCGATestExternalResource opencga; + + @Test + public void testVariantSetupMigration() throws Exception { + setup("v3.0.0"); + VariantStorageMetadataManager mm = DummyVariantStorageEngine.getVariantMetadataManager(); + String studyName = "test@1000G:phase1"; + StudyMetadata studyMetadata = mm.createStudy(studyName); + int fileId = mm.registerFile(studyMetadata.getId(), "folder/file.vcf", Arrays.asList("s1", "s2")); + mm.addIndexedFiles(studyMetadata.getId(), Collections.singletonList(fileId)); + + for (Study study : opencga.getCatalogManager().getStudyManager().searchInOrganization("test", new Query(), new QueryOptions(), opencga.getAdminToken()).getResults()) { + Assert.assertNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } + + runMigration(VariantSetupMigration.class); + + for (Study study : opencga.getCatalogManager().getStudyManager().searchInOrganization("test", new Query(), new QueryOptions(), opencga.getAdminToken()).getResults()) { + if (study.getFqn().equals(studyName)) { + Assert.assertNotNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } else { + Assert.assertNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } + } + } + + @After + public void tearDown() throws Exception { + if (opencga != null) { + opencga.after(); + opencga = null; + } + } + + protected void testMigration(Class migration, String dataset) throws Exception { + setup(dataset); + runMigration(migration); + } + + private void setup(String dataset) throws Exception { + if (opencga != null) { + opencga.after(); + opencga = null; + } + opencga = new OpenCGATestExternalResource(false); + opencga.before(); + URL resource = getClass().getResource("/datasets/opencga/" + dataset + "/"); + opencga.restore(resource); + } + + private void runMigration(Class migration) throws CatalogException { + Migration annotation = migration.getAnnotation(Migration.class); + opencga.getCatalogManager().getMigrationManager() + .runManualMigration(annotation.version(), annotation.id(), opencga.getOpencgaHome(), new ObjectMap(), opencga.getAdminToken()); + } + +} \ No newline at end of file diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..2b44b35ae32b29453ac69b6842aa4c90cc8367f4 GIT binary patch literal 3707 zcmV->4utU^iwFP!000001MM8ya@#oaeZNAbr`avr#ZwgLVQVy!EYp!l4JA48%w$<0 z2}+20C_L=QnfaLgmHn~>z+0qzt&-Gc>`E-7fo?SVLgUisPN!=iyW9D(qnRI|6)W}+ z-Cj?yp=m+ZhCqZp5ttHW<-XAU`w4EIw(bI^$FdMOrwrT2zwi(F#hyBnWb#}enzqGt zN&8?6LlC3DR~|-WK(7aEnUh!sQp6;JVT$7?u1g?bi>_c8VOo?DcgAvO;^{&i%oi8h zk|F^s&Z~#|c(5E(Xb7Vq2`xzBnm!q*%CCc^qN%moj6uw>Z>>SCYL!%n-Q(nfdn@JM z>N|k#$q>r@9>t8}#NK5TOF%(~g0>(8UPQ{DiJ-zY@Xo`n z!NoL2m_k*0USJ~!S*>+e;5%rYq>9Qr1WCMsew?9bAzCA!a`PLm&e_rCV|6$g;GVnj zZVUHen;?qIjxe8uRFp$He|FxdHWDK*8+tL`2zgr^)ecK#%2ac~1m zZrybdqIly`7A_W?m?;8cfy8g%j>6&n@7+I?|NL7BKKQ~C4gU%&s({h|Kz z<6BFQex_LP(-xJ|;Cl>%d8M6#I1Z7S#IQ1od<-S`dkmJ%q8P+UF~&lHi`8&AUM$kS z!!QWzSazl_HStUsP6L1aLzk7<9^dDB5+kx~>YH`mC>xzW|MvMAPTp&Aa*NydRnr8H z#7nF!LHhwGZz9?N0+}~OOLoAXlhec%@ETQl^G7*^Eix-O6sWW&e`(Y6?Ew`%!qNkxiUrx7JV#m^LzeTrwX36V6tdI z?Ad)5@T|>9wtYO^D(bm*j8lf+&E13v!^cHU2e>Yg*0@jtnL+7MVIm!2Hx9m6#~4qCRfQdXKxgA zhpyjU-6$UdWb!||rl^Ee`}M6j(wQ-u4llRY?|wnrhcQ1}Uf*jY zg;Vbbxb2l-4VB((X?;>Z%mnp=!p|<4Gw=GI5qIn5UF2;OW(saVFxlSfV{9D8?S*y! zNj{z5FpG;TbKU)N-XU?)vAjcW@ea9Ri9Zf_a0@;ELV2&5neolID0-~HhhHRLBrHx&*CK@0l}7}gFe!R zXjy{TmnEKaK<~>hH|Dx_IUX+C+=B+>!_L9+a=BDu(_g$S$bD91#6I7Dp>t&E=3Wk7 z@{*fFz6-FQfRCU95;yMTbY8lv(%uI2`L5qLo-!g_$%1M}QH@QeCX z2T=q)(@j_M>(JU@$)pBrLDw}wRO|H#3-!F|tD867oR#`Vb91Z3%}ukhtI&g%1qDWs zIOhR3ZzAISkbwg13= zEr^V1igXLC?kVo3zJ2t<3Zhqw0F?xE?ot=*w1J{=|WDr4Ibejw54w{=FxT!nSIt-F+;>P9=?g7$eB zcvYemGFcdVK9w}^-GUGS(}fEd*V2M*aq^%iF08K%k`Ps!hR}ss{z3QtQVXwc;bBq1 zdeV^{msC=-_HEF%AXY{2Ye~|>yQtl7uvyD~!OhxK_G=Wy;iIxLGdpG5!IyoDXKo^m^Zis7j1; zO~`=?thA_?!|k!94YWJJw>u~%pg8EzkOw<4MhW3I2;&+bifZ7s z)!h2GYD=BXwmFHwwRY|n)wJ&a; z?3b?bA~$-XJoBXbG0N0_{x=ymD84j3`D@GMk-7b^Eu8j=@vkkD_F3hxE0Z)N(|s2| z9s<{`Uw5+8U8!?=4=M}Jc@($YB;QB)Ehn@M5xM(`U}J}ArBB9c>r8op)*Bd+!!3%t z>jX{*GeVk%z^B_+>`B4OJZnY3#SPT5n+s)f(bmZV;&r~=**xAC_vF52`Sx zj@R86hJxo1Y@mawp)9l!{y%#z-o+Rvq|ZTJEmj!q%|O4_me=#~6L1!*#hiM+G1C?U zWvDD#&96luzu>O0D2&BmHlH5q>Y+U2RTE!d6OUv!uPXWaDrwKDUN!Lz!c=P}_o|7n ztBK+aC4!YF5NWZI1I~42hM70c6?KIDi0Y;|(ZX4R2f`iMBC(Obj-E2Q@laDo2PvjG z-!o|r2Rb$zy`AC^gdu|AbLipKz<8h!FWL#}FoGboHjhyxg07|NV}Fgxqp-FclM(Q? zE>x<=Xb#E%!`3seB#Hy?96{GEHCMS%aIU0waFe~@!sR8shF@7&43^`GrYpn2l)^W7 zjkF0edfCgfTi_=7@}N5(khRkL0z3oTEfzziIr$(*!e#P!@(5qVa%)`a19dVsrpqQv zzjavVq{HF=JX7JK!R^CrRg>%L1&qRTa?VEcC-a5VXTlmQ=Iotx#Q%ZCTH1}RU>-z> z=>BJvdMTe)e1dkW|8o?k&jriNN5J=kbWwTOgtoypaVg|HgdY>=Tf1ge#kle-9bJ}T zmf@Gg!`k58eQd$u6 z0B540dP|_9<$#1nT`I=aV7f~0?zH*Jp0oHgtVYntR+hG5 zLkrO5pNp5{x=J5;d2yVH?Pg!+=~9=|+B57x_WLUVM5rsNq|oPi|0Q;oB&L^okU44LlGJ4<*by%wD>-jyoL!#vu0%u;^mE zoK1sqJ|G+;x24=Y6j??t@9c9H8`!H43=7%f!l+uftQM|{!cwo`;KHa{xU3dR z9CbFTaBXDM1r0G7)WqXT{Bn%x_;Y;Vnb@s^Br<}~p}~u61(dSIR)r0NklN-a2~&NN zNiXx3wwRq|1vDtY<+FTgAMuzzMXnW5^Qs8S3GwtCZIDwFAMz<2>)wSl(o|pwQwVcl z6oW7}T=L$Umax67sUsG}qK-V`ga)uIxQ&Cb+FMp-2Ywx1Q7x7*$os5<>0|BuDi~!F zhk8KhV`sz`-0X$i+o@MzUkyn55JQ9PWFg{54QvO3I0B399ph2w)PqtAULf}!ut5(?9S`Sq5MjA6#*kLMQu^pU7 zpaOC@{yk{(*)iFoF!Cpwd>!wY7EegyL6IR^LFC~Odthiec5A9rVG*r;YH$9F23W=Z z0>d;+=;JQvXfue5ZlJqsslZl@y%@VHw!#`$`>0zGZ%DvQr>DHK9h!Dy&|79?OZXrs z?M>QwdXQYnM|-n__0$@wa!CPuqOOzHw*_3&S;rkY_%~#z%Nw)%bEZy=gy6rQK=je_kIRXSw z=$jAdwuvb9A>(ko34qDJ>h?*x6?-q$Ew`uJ=9jE+kZHVzcKQP2^cj}Lc!zb{f+%u5 zM-n6xist@Tv*jZH@%S&v9(!qt=3?ugd;B6Vc^hxy-zof0;tuU2k z4mrx#vpwUD^S4cml=j5PXU@z^bowUTMdz2R+u5m<+=OqK!QF;kYoQxEk?~gh^__fW Z%y!b?%OysFz(1H6{TEgRy#F9k006H6Izs>e literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..cb73bece530f11bcae74d3c5f632cf9b8768f962 GIT binary patch literal 1038 zcmV+p1o8VHiwFP!000001MOB_Z`(E$ec!Jz^ppW}^kK_R^5!NEny#@sJH^lqgKEHBdIi5(EmP)vSi7%8}w!C3cuLW;o;@w;pNqN?)97%jh)^_Zywzv(KqplGa3zJ zj7Z|g2qBI_h{oujpP}>PPljGp^jVQbDBlX6o(wHs!H!ctD_z7-?>{3BssNh^4tOI! z7{sH#=bt5g&+`W5kKZ4i`Q!7%4+0#S~ZFYFDGu{n%jc-Sqa|d@*^ux}Lwinl4tar;DkpP$??7a)nG)6DyaZ)LyxP zJ8*4kIq3CPPKgXb***I4p7kbxHo5M4;4L^}Znx-6?U#%Ls=yO|iec@vXeHxPrUtBJ}@eb9(?=IxwP4TIK_ zMujQ5t`zqox@lcU4XGvk-uq)yPA0AR+LU-3{;L2+_0y%`S_)Qc_yCf^R?^Mu67?;b zwTbGaqfEW+;8j8t;H^sUMt%{pD=(ZuxqC=X5WmL&igdGe{ut;^8PTkDecn&VzB_dBZq~ z#@;ya!{K1F8Hp)XTE5tWm??%^+pjP_m}R#PDUb2-iz3bDvuRhO2KlBWYFc~W_I4u| zSMAQSxj>u?ZH2t~4tBn|I7kGKO~-7u)ZF!YWb&PD`#?85P|dX*WfyAqb4#y2BhFS$ z$a03akawC>MvdX%>Cz51is7TVmj-TiIxhIMi#FMIJ~B<@6|@`W>Ot);y`o(P-9u5^ zylR@nQv9~xTbb{=-G|_d!muaChiy7VJ!#hw-7M$j4kh8**J=E+CiGkebiZv_Y5wfB&%MXv7n3{XnA$T$pXNNAkHWhh9- zcnmJDy7ZrHphTkrAA>%$6LnKmQpe#O&!LWT(d2cpRfi`!D5QQpX*Zdu?<@H}1Bf9S zM3^#+*l!K76H+yOqHN1jM?Q0LvemlEbjpkO+Ap`M%2)jw^{YmuwfaM)E>E?38}N9$ zcup2K$rJFtF82R`$p$dQNB$<2ZCV042IAV%(0RUnBk?y_-h(JX)ts( Sfsd8WxqJgTl7QC+3jhEyTB@Z0 literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..7500b56e11e401622b0721d8a41d920955b54608 GIT binary patch literal 464 zcmV;>0Wba^iwFP!000001GSP(i`y^|hVT6qMyD-QAjy_&=az0maS!XVb#vHKj6TM} z#Z8JW2bQq^y_TG;lhBfM>0XRHGw*2T)w2r%RMrLsvmjSLK-|FpO*ri2UbCO40rG}1~XJ2+017CKuFm&#_dI`duH-aB#d!4FxlT0h~z zh~UYE%ku@ZFS$G3_ysWGn6ZMg8INaC@-V)A`KN`i(b-n6JE@&)d=La_6>=(KtRa>- zVwJ%v9K>2P5kVygkHC*#HnOvMwVa;Ewmk$-Z+3rY*RWG|Q*pv>a>lMR^crv(Pv9I} zk}6IrRVh)$uK?}|4gZT?&aR_(U-wV+K6`l?-6Pvyhg;5G#_gng3Af?`x4!`yni(T> G1pokej@~-} literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..57cafc8910ae70bb19d4bd06e9963d379ce43c9c GIT binary patch literal 560 zcmV-00?+*)iwFP!000001BFxDj?*v@edjA8KNUnO$IZ15NM&2G6>M8=S3*Fca*|2v zC5emUK)c<348Ovc5j$-PyOe#2oa31}$7g1ChJltWMn>P5hPM#OaojqX6*;rE$)RQo)~$&kBeHP|0{Pq_U|&FtM>ih}CmE+t*1sk6FT^^KlN=acjZV ztN$!ta0&XqsIpA+@giYqC3G=$u~Vulxq?C-D5ip?t;k*L(R4PNT)Y|7Oh|6m@DN+^ zQl5zFN2^jTwXtvXC^%<#w7pdHCCk?tOSUr*IV;QKRYJ28jJ-;e{41Q)W+@j;u5vBm za!#D44JFiO%D1r9INUAJ{rJ~Un{Qs2xI2McGuREkEEeQ`^6K0B=(>EZvtYAkCpUU7 z#~{7PgPQK;Iny?y$g_d^G@%5?woB?Gwj4Jg9>Lg-lhDH+u^hXHn#Tx8St-SnvzF0P zUg>#*_}rX5r7~w}Lp0hzRJEpte&1h#c*>+Du?+^2r&Y?NR@>64>B}}v(E$hb z1U9oxrnTz-J}`lF(W<;4-*?~r`tEr0Jby28-&=dP=}*k;O+NFZD9ka<3YsHuj5wq* z(p3Wf+jkwk9q8M(PEmhSvYau853mn;Eu`;=?Ff;GvS7vLgeJwD5JHDfunJcR<%B0e z;vyu}c$TF2B^{y~lvxSyPo7L8SfS(w>=48V3khe820Z4(acY%|LSz?tIhst1rl&c| z1`B8_>or>SqBxGyB}ifFp77M!CKD)apME>GeXVNc8E86(8G2N8t>ol)7#M?fTK#cI zsk4V}c<ThBg}U5nU6qt*T=Kt=8R4FUYiqjcs#*rnkR)KisaRMJ=J4w6I`XGg`;B z-+j2+eEoN?C+J-9vC~&q^l~(|plreCyY1DN-SzGLB(<>4oy8$S0tGOVhY{aF%F;wNP7}`C<@kUk+yas w4)Ff`s&96;{d4Au%?Y0~c2C)w$7@c$4L{`me^-7X&U=3I7s<#}4*LTD0KFLa!~g&Q literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..e5aad572d01c577a4b8546921c5d503b4fb09f98 GIT binary patch literal 3896 zcmV-856AEyiwFP!000001MM8!a@#iceZK;umnJ*5Zno?=J3F(gMB8d~DhyBC-l05(jUL?wv6*qBhKiCoufP-_x!NCEb&j^(q>Xp!IG?@HK zT+#7fluk~LJu-2pHILv1Icea@DXE>k@=7lcpn0U5_oKyx@Gc8xFOC#kgG<<3(4eFm zMg~+psnw4rM~BtgtLb62TCK}}wZq!k(ec@&adcd3c&bN$7i0|y_9+h6U@j+EP!_bY z(9P@B`f;^Uuh)(mr;VCwvtS;bQuo$&9tmAem$I2^(k4?J`2vL^@>Z%vNPkzotFkct zsGG4R3EUa}mm=zQN?O1~GOTR)8DO8tWX{+JSqviIM}NqMGtQ!=V(dLMGQ#`Uf=H=` zfU6XRgm=8Si5akjeu1iR6}ZGZ=WLNhnn9xo+6RlQK&5NUDW3RbL_|vBrru9L?`Fst zv*WTkN^Q~%mnt4r@4~yPK5c1ia7{Z z%n3@_FN5tyvbK^1`Xrc%`M#nrw(Gx=O~2Yn)NhSsl?ah?xoEONl9w_f9?)!0qV)`qyF0iR4#8ihaF`*r9QZ&P3f+|AGmyXG-a^VAn_Z_btBGU6;PgoBZA#I zI`4GHmJK!zqf5^LVvyBf$YAGGIgoN0AxpNEb;!j5s4EZ{8ZQ_p&Z*@BZ=KOF;MKE@@ML1$7Txhqc-r9e?78mQYUl-P_C1@wF`jp}d#L zLbprsTRZc{ZWF*TgZ?QeLK#99Ed4svONX6;t|6wAz`ZRK`U1mpa*Yd}_qGNP-A1cl zMy4O&K> zDN-Cm7>XE7DIYJFFo29$VNsdmYodZD2vmuoC_F$^Kx75+t0ozT67SyX) zmHo{Xeb-eX6ZT8tCR?ceBYzbG6h=rH@B^0M@f{_oW1jny-$#giVDHepp5^412S41U z^<*dU^$3K0KWdheEoer7db@;TruP+PjBL{w*DD^dIqtTE7KC~CZjmG5waImnQKmHy z*Dg>DQ`i@2Ma1d4TLT97T&i^~q#Lf!DDY@yNpF6qKJv$!o=STxvE z$rj#qam#HQ4YOr52>g}FT%`#`<+9~mS;*}@5CZmHM#DMSxC#Mh0qpp_GWz@TK54Jq z;vM%%1$&}`ol}0Fbia|i+8iqu*5mhelJ7Z=r(Vu1baA^8jK+4SfAP@Q_>uNo+WSo@ zC}8i2^e9&J-S)Kix}rbE)Ie>w6Qn1f8=wJ%SzUE9v#Il<%7>Ch5_+Z z(enbD4op)v2?@Ujq-mcyTd1+4Fn3lATXZy|*CYVFdi0ul;63+0i}~xHQGS`%{4>ue zU7rTj#r_BaZgpD)f~UiX?@k(vCfTb<@*Rx}9H~Xlr6YQ3Oxb-%{{-NH2$QTlETAZ^2XS1b#yI2R` z|6sMo-*Az->rq_7AtI8+FbSkhz^CD0wSXgA*-CGYJ>`>T%KBSfg=#OK=waV(7xli*Vc%c+Leg%hQ-A7(q@vTRQL{G$v3!lw zDxKXlXI8f#Sm~*{%Yl`?^bYHk+6lA3j{JmgeOo``)C9ETb)`~)EikG4id{Bg6}(*f zmH0r*d>(=!VV}(^0`nOW6%c|rlw4HCR&O}4n_cI8&~00`b7|RDB_wLxuma);6I@Hq zT<1KZXAyK&Ee>BFjn<0TP~%c*@GCsU z^{RJ#QireATI1xbep;_qkG<*Sw0c^vHI9#BW;mJA5a!Np4@$tSMNd6G(Y~OljZ6mq zX5_$$E7gBZE=WK&MOEq##*Wweki`8=8>~*^Dv`Kx= zYYA-DR)MHJw5@S-+!^$@?T}h#*VzR9uR^(}4H4{VJ?-C>Wh^;NZDG7n;%OznqjA$7 z1HX9(o)TeOLwQKPW$;c8yBv86)P!@npAI~^^_9sjhSWPEy{>bzWL&I;E6&g!T-erV zv_?m(I~XZt?9RIDGuY1O@W)uzS(Gv99g%Y){lqr(+A*}6{T9St&uV>KYHePP2dzPG z*t9`!Z>?wjJ~I7|I{l08vsxdUTFoV|XSF^WwPr%zL@IZQu(FaUple;+fjmi{!R{yA zQJyBKz74aopD&Z)2F88Elcz0{Th{icEu7+o@oCGXcvX42GSM#CB&cPYmS=M%aU$cq02rM zB0k<3@m2p#fACXb;B*yy2-v3}8K8{DDnjv4oYR+C*i9*xvxGE?36a;z&0L)Ncy`N* z-|PHrwVl?b)$7RnrB>neENw{48DI4VSGF^}vOD9#iC0<|MhDgqFwkb#>5K+#`0L%Z z-=zfs>0P>Bjjo{Eo2LD4Fn%|*cIY#@8V%LVcY1?Sv(xH~3eB%TAV0!TAPOf=Mi+Ow zDb9G-#MjrvEz!-hO1{2Iiesv0O?(44RT#-VYvSu_A~{0|Ng&Kx9v3%az@gpg0rAdx zr{9Koq;!*)z_V7-I`_F5isY0UzuJ ztft+%EJkRPkYKLvxZQ|R3OX06TfvN`M`3KK(^uI5U7ziCvd3uhXjTDQg=bz-DA?kh z65mTTmpM5;?`S*tKvt$a)>l>1jYV^8T@36_tJziXIqZ?vm{O4#M9&*pvD)Fm`S$d=^ej}dN|2gHM$ch}5a1b!P zsoZWtOSX)B1v%&XW!h@9Dq~#g2Lz=kYSn60y&X%NDR{HnHS;NLe9<5Dnr#R6lY^di z@P)lJw;Kv4ho=XqT9&`A7=JeOXI#`JU1%5w7e9wF%w)9O?Dq#r3OZG_>5#o&kewb* z@2WfQIB%QXE333gr@5Mtn=FBHt08gXou%h^#Kmg+=Tr4KNPbnXh15=r$jh?OiUxu+ z`P+-7Fo^5IfW_Y!?B}=xsGVgo_Z#7U(nEXjJ)H|o^Uc>Mf|mM;j|LjLZO{4{E#iY zO$uW>(+3LeY~hD&A??%ewKG(kC#uTD#Lx-w6$L4ent#(DFD5XpYIj7d&_ey@I9G2&BPNzBX(LljL_dZdc zCIh2@fg4~bFc<1Iur7hU%+y3oib>tlr!H}@>yo8lJUd!k<*^_S&!`q8jLn!?Ff9A& z@2e^z!PR4lA0{JM@a8D;-cG&(<|>ongCLH~WKlVfHhND977>RQJPOj`Hs42UvzbW` z&G@QdwDGw!(eaN1%Ti7k3q6E#YFT)+3Segc2j)(d6^}?$hEWYG6vT=(2DDE_MrRrgveeQ~dU64+gWn{{%MU$483W z7?OLHl&vU!fK<>meYKRWEQab~9Cgb~ri8arQrx7NrW?_vbQCw+TCc1jD;Et|<15>$ z5J4gS{cJ3CYv(?mZI3%|x3@f|`?WM2Fnw82uK3Nqg+{PMxFRVI{U}Th%q6tdYOAny zftgEIV7hp7-uVJR_F2pk26GRLf# z@$~3Nk4HLfEgHFkbSoKX@Qa8I=z3Q@1=VOXk!>^Fw{8VSze3Qh`SKW~Y&#RS`noTr z!+~2sh=VAhjBrY`DbG2H^IoiW`&4Yzg(oyY%lAp^4;Rbp^Q*Iqc?7Mkcc=<#kI5$4 z45m#$XP_RR`v9TypPJpA&lkV<1No2?dg-)%cN@iT{QO7fN0?4BI^oB+SM%%J%jMm0 z-{;uqA)!>}3}FTpP@)HV0_DcJ|zdWplO^&mn zx9vVD4ZJLRZcjL~U94kM>)P!P8dAAW9^#Is(yKA2h*i4ov)rX11gFa`;6C_h!Taad r#c&xFmEf$zLdq#8G%qq#;xu;Hjx9aO3tJdo3?!9W& z{?)aKqF_M&y}&PYG@LfLTYdZbewLJVs6=GFCzh#BHck&|77jso# zy7KW1$`wIMJe*2FwVyv}5lM{L8gy^KNA#*#wbUxGXxn3fkTZ)-iTS3cwSXQ>Kpp`D zj$uV>?@XUry7u+0M(&Byg#aO+TwmUcDF*Oii2rWivB_ zs$AWY3{xrz51*V++eDM@-F^|zd6ut)Hr1CQ748GO?Q7vp5hxu9Aqjdzcgqbbjmw2f zHo?;av<`Pc9%PB`KCIjm--%uKc^7H(-!Yu)+2{U0(bM7JhTCG!nm;4E9Hz6k}-T( zb9cSGEsg`elBIU3EaoJT*K8tyDU~?p^m`sd6{RZBw)MVJgR054jVlAahd6SuTNbL~ z8tb=86XJYGP|+lF`gaw>Ss^7}VkuoDEmrF6)bsb32ShM8JWN6}C2D$Mbyn#4PZU`` z;$Tc=sdhyCX!w<4V)9J4zk_h556V@w?i<&3hYLnYdrACN#+6;w%0XWIS3sqq6u2z(PyEp){u7T>QRi4m zNNnCN73iIt!QDf(khGv2Owb@uCQ6@5cw`VEyk_BJy12}pGb@mUKKHMayfR<}|e331pT zk-|S4&S-b^ZIjagPFBKN^GzPV2o>ux?V0P1Ut4*=4iC?$dD-&=<(kh|LouD(kxOCeECLHfo0g~5#SPH7~s;uh}{6gK0^jZXEw zBIvgnq-vy<585-q<#RqR=UL?S5)j1WRf_528;{F!e6MQuzcHp#5lu#8`{!98dkV`V z)Hxa|q`HpuoAb$dag`ea;iuPX_k9oxpO$Y$W(#7=jb4>>;SnbVencz+1C1?X%XyWA z&kzX~0yC+{pdgf3tw7|FRiu;K z&N3;==|ZbIivQNiblnrBJ96V9Z9jGXNlg|1oqm6*rO>E!kB&KcB_C-6Hm9PEbrrShcHN#9x+Uk#1Douxo)y^>r+A-w8e>5(2KUB7^trg29BCMSGojo$x=#PV;}`}VK9W#yG$0v3F`hz zf(5lyzS~6X5awn8*yOMyG*o!Y!b=i}-tZk?J}w9IdhUxw zHKA)|z4c3Ua)Tym9wz_v-F2+VtF&7IxV;M>OlrW=E1j>Gi60UWF9%6)Auh>n#}^x> zXcko0TRzr3wG{VkU>Uj@9QyPynJAQAm>QII1uNw8-IZR1y0Q~g4B(6(N!xV&=HF#= z1AAY}W9~(y!9E=zn3-rIR^ushg6W6Xdcay=v==W%DSHs^rMr7-$X4m(LR=+@5+c}-M{Y0(i&&?>bw=K2c3r;~_1&7AQas~nq{zWeQ{I@GLf+24O6zFu4Eb|=Sa*}u4&EgZ{LOz8e;>r_q5VpF!x@5~pdB1{;iN!iAH za_d9x2rXEYh#S`E#280)@GSbdnbEcKK8@I`L;FO!{auwXUu>i11m6sqrpUF-n}i5S zZoHUQ3Ox|fHnx+e5$zf#xtg3c=T(9y$sCrs4VAd_=a5r3LxdTo4o$sG%`QMx^>DkU zUaY3oAD~JVziFt79jru0<}^X`7^$;D>gX-_MpLHpH}zrnH-8O61^RPN?g!B3E~w%2 z*%tq)|HZ2f)=T--XQSBvzZd1Q`;^R?))P# zWgH^OMbL8VgPZAez~j-YY`MBm1r{#4+giQuuF{NDh&yR=7g`lDZI^+~;BWwT7U~)b zfHW1cK>vao+%`ZPl(fvAR}gO@n?3vc<0f@3lIk(A{2~y2lt}#8(TDkaAbh(`@JZqF zO1#cazSO8{lzo&jQpC>d&d&B;5+PJ>DsRVn_k%hL$@V<_2S zYY9vNf{D4UVZz=8dVIvMKXw8JPM6W6c(%zFE?h^7H=@^QTt(15xIN`OjXD0FM|WXe zK#X#a5C?Jz$*@?R#8h}PrF)$h)yu1;+@+(^rwX>7jhff0K{;ob=~`q5N3kKjo7Z$? zG7V;^JOnYl)OO|_dB>~|7P4LDAdRZFIc0=x$8*fftCyCRB38YZtT^z|DyAE>aM?{P+D*_W5!lkL9|M#qzmZu z6WQXw__}bkTvN-L)YyFjG^T_$K=4$LtTL*lj!9q#dcPhg9lp57_kdRS+)NOy5rPcB z7<-AJCZv*xI4B851=W9GBtB{zClz0sME^lwcgpsvGe+cxeFDP^rM_14tDF3DZlz(R zoPkyHphhu{%20wQ4#E86(nT!LOjoz5+uQ>qiq*@W0M!?epl2!vX43egZ5_N08XY)= z6wWSqN1E-W2D(j&p%GsxrV=VHzxv2nDNN4l2R`o%t1*MAudNcUf8e-v1;-TXx&%CI z$y;s7lYU*`ZR@IPrQj-i%xZhgs#=idtWd+PK38At zON53COLs{hamDW7aW~NdJ#VGxiFA&puP|{*wC53I(yD95#57H73dBdhdHB`3h|XUu zZ(=L%@m$G~jF+&>(|fxab95ibpM3%x<(Xd`^MLbW#I)6g}QOm2=6Nt7U$NQOGU*yA&#bc?EnUfN5fXAMRy;*+plVB4TM65_;3 z<#cYKZX6S-Avc_wMZ>$Eo3R z2krVQ-y!<+LT~ZAqB@c}8Vka(O}Qq~RKdkk{NYFhxEUqt+I&|U#j3g>Ldq82PpWWm1_DsqS`z}!fp`68aTRW!)(nD0H zG0vasrQarhFqA=gK(Lq+%$f+zMPL%5*-zR##d;o$^p-_$ZDshpE&)S=cu4QAGj8>> zqky&Y&%iO{$L4?jUOKe;mL3|n&1+^Xb~mRyQA|stf5c_lHB_1&Zr*XyEApft&ecsP znxy$ocEV``$nTWazC>_R^H0oc^(&=>AI@F5KVuiJ5seVv7p{}#o%~U45F;;sGbK1u zl=C$1(at|{bu~gX6w`u1)E=E(6-*fjy=|qYDWteobB)3jk0+9Mgp%`-dJQGC#d0G_ z+4#NXX#7#D!9OjE`kTLf+`PH4%faWltYYHItd8JK!LVHnvnu}MFS1OQo*2Fo*#<^ z{?0nvCLYcCr&3ZF?BgsfCLW3(+>FONx3hz`E%AW4TWq2*Ww|rDzhLo@8UvMJ#RNoQ z#xN6^M(uxHl;_Y9ZAjuu4~M&?b|QbA6o>*sBLZxx8BGLlRl#0Ff3J&=R`t?PlkP`6 z#_bMQBkc;1Kf5qZHRcNjV+kFQxof@~IOmcQCjDs136{*3Ru&6MsIprnXRP+s}Q37bM>N%{HBz_=>qt`<1`teF_jbY3~Ww#d&ZLj>p3$?%#ON|JQe#zYr(VAi^3J&Bsq zNOySkkY>m8OjTa_8B2fuTKtUT404(yPi$7Kaj4@Xh=mJx1ccKYouLpg6@yPAY)ofJ z;CGHQgnvx}J2+@6tvx=ft z^8L#?O6cevFE;}D7RY81IYd8egpA<(d;QJs`mv8bXIDmBWwloGSpV9y7DUzw#i5W! z+Lg7W)FW$T)Fr#76SWoCG~zy@!AJAwvp9a5LeyhJmt*^dHh9h-zobWwP#n%EZ%-m) zNSfVkn%X}Ayc2Uqp12O(%NylA2chp;GNNU&>|73t%jB$_TSEeE zW`hN-3*k^+kh2vGOIS~D)Ud1N7fz|#t22$E#~8UxdNu!S%qQbf-6-m`VM#l*m5xOB z_T>AT@N#1T;dG?E&Tx?h_s9qA^-GZLCS&-L_62TgaTcrivya32Zgcl=JdP18R1M9~ zO^uTY-x(_uyLk^0hNzTek;p|Q5e;FzD2#-}_x8NRF1hG}{pn{m>#XxV+W}(v%oIbI zh)e6y3H~+{X&NvvF1U{?XZ^bCS3a1Kn_o zA5LxpV?065Z6qw|8aR||r{>FrY8$7{9Jr^^wG$VXi>H^Q#QvLWEGXh8^JKiT0Wj-7{rVT><=3MKw3sNVL#leLuE)*UY!vS`>J#5H zj8IUt3tdd+0MoP!>|m%i8#?b_&a-kihNK3fEn#Kv6?4-=s=(0G#Cz+q!>jfPq()oL z+HOefRn@wh3vq5-9wW#)Wd&4pj+vSQ*3H6ls=t72%U9Oa7#$le1Tx=VMz)zFiHtwg z{$+rvWf|$XHwT}SD?>81UVdy|RD*uY5ISWu0o-!FFkRJ*qyuNV6iV~z)Ehp8A^GbLsIF^sxLRXY^QVdy*_DP<>1<#u zW?_Nxc1r9&ycv@%Ffyd;X%Cg-!qF>eM&~yWN~(&O+Sa~gy5Dh6iHuGagFaRQ^Fm)0 zXFYLS-Dfem9k^yB)mHcZe&&#Y1(Ofu2kj#>bFJ<*k}4z#lEY*VY2UT&e)z#_Z5Dpf zalH~4`LCFIaqkxkvW7E`KP#=->P8s0i(nHA2;S4Y9#bGI7(JKOpT7Aj3F7Ug0q(@WyhD4 zVdSt&OdpVavi0ODr>;C}%fo8*4AQH1Mi!CVJa2BAe)~j93u`maLNP7;_hthwbywCe z=s7GQ7xDd)g~5x^0>eUcMoU&OKI&xNb{1!V72JlIiy6#|Gl5jOeo2$!+!NHXLE!q7MUI1DXo^mn9wmuy?rPW`>*l zQRd7K;o8U+4C$v7zR(w9|JI8&;zb8HU~qE*B& zlv@Iq3!0k0>rZs)R*HBO41N?eq7 zm&bVu^ln4hXuzPkUa9XRDjr=LPpZ&M$hAV~3X`lgs|!URg!JUNwT&-^0*%I*|32|o z?l8c!g|u9ICFl#1uj^6|(r}J(6<8j4%Np1066YF9yZOhgH2qX$2q;Jk@{X$86ds2P z;kmdAz?i~cP%i&6rMBj>jE>&@QEjs6SQdqu-k=f1J|$RBv?Y%hvEa=`qkoDBFS{v5?Z0eJNSv9o7(gha3x{>}C;fQjt;Gyk6or%dP+j^>WI?6Abve<|RNz+T?$l{YsB?#4qtc#O5 z#wt^!U$TGRE7-RtKJi<7jwSp|u@_iuA9UIs2&rgS4A2(|Z~PNe7v^mTiiylWGa#jB0+ATJUxfK(SecZa zhKm1eu?2a0lL$R&(q4M{Fxvb&62)%?%)o1vS&8OZTZbukMe&DqP+j{QwjWfs5eiH- zF&92o=reYC3}2STjf>JCgoJ3i@KFDb@^{75F`$#UBCDk*Q@&v!Ku2OukZlIqsxgQF z1tCiCPtZObm|C8kJ2>4pU1}Jy0u{KtjaRTQ)sk42n1s`^=-pV2IcXNx5S&ZG z(yZZ>qa@YNf0NmXB#Qc}Rg#cMOc{P6PZ@2;TI&tx7`GN@6EBy7w`tbi1uB9Eomh%j za~Ij`-yH_^T|ac%7cbj}Sl#8emZ6Ll+pK0#&y%O`&Y21#qjX_?pqF7^(pbiD==^#Q z=Mum56XF`=Jw@y_2!0e#UY&6=U=#hG4P|n!YkSmr|Ka_LCCwc#&#NGisBb^o1FDyX z!GP!HJqubas(nQHGE7$K=k@Y76n=I<_WI3zqlPSst}!Nws}iQ|d_Md29AMa=+dAK7OYE#BGJ=SvA-pGke>Ye~KRHV}Ni0D#{b#uS0owa*Aq{dQss&lK1vuRk! z$<{ciIWfv=Mvw7Y;^b|EeF5)b8$aLjTc^~k|9-uL1@seHrW>>|=+ewFWh+8 zWxT!LCXM%&_2457(%p%vK;^NZ--;qUMqk>wdaiw~%0a*0&NO|@A=#IA-r_Hp`wY_W zgk_f4$SFTfF^?wlg=F?!s_AjTzBMqI*vA|4ThY}~5M>4bNfyaU@zZ!2zP3#~hTILi zx|F|so;}X}Q1eHn0y>eH@YSC0phJ2)gHtonk#4x@>qz{% zVI?@oF}sh$wM>KS`R;6)&w`k_ZH!3cTC&6ghg!pQl1b@<;mo571UgYV-8J<9x9<=6 zu>?(qy%r3-@+fLP-YL10N_Z%46^mKbW;;SavBYM9aBb(ZCPnM#SRw)S#Jw4 z-xQBqvP1k&s6^RjuUB`19>6zY2OoSv0T>(twhRJOdIi8E$;L*O>T%wv!Dv?I@#Imv zt#g?d!Gn43P=*5Bv2G$`k3n{>ycd+mB%LK%_LYYxL7m!1?>YClGIXuHCs0VmsI!`m zmul;hgnjYZ`6WW3?kiulMWekXL@{gZ-uiEcOAtHL%w*65 zf~J7xy^8FkH*eqlyx>q&o!+C_=<)CB(Y0qM@#!j+wexrU6i?qu;#ohCP?8T_CKdNp z40tleWD#rlUxGKxcxARz=Xw?4*kJ}zQ6kv#pEB@4ZOrh`K(_Fhg!JR3ZI&;)NCR%y zP=gyjmJ2pYbLQ%SdME%cyw(2_MC{NTFN0N#0yNYPET(ELMCY+}1mXuf{Z$ijq@Pf{ zV>_=baS>Uq`Rx2bn+w@n>U`G6J@1rr@_^Z9QW zH;e4vEd13reeBK*fU&+@ygpEG6aG^INx-9rW!K9B!$~fl~HbU#OLGF(w z8;!LbF(A3YjD9oGHySgPc%7>*J#R7D6B{Ut{wxsjtZ6bJr36IlE)%QIYX{p))g;d< zhc<-5qX|(1uI^zH;bmFr$Lq-f{~|_G=hq!19Y0!Ek6WaJZSBf^pBe zq0VCvl2~w#ZW^Hq(9Ot=h|yF`PwtZpx=8?*h7J@su6dI2#_EX%(xDVMssBq3S?Rxv z7;sWwS)6gGQY6zu@#QysERe0;u5j9Rg z7fk@Nk3i9T^ueyq`Z4Mm0DbUK{*&JC#zU}Fuh=j{r48WFtobH_XO5X*53BoS)bTIo?cs*K?Axp30KrTo4P9X zYEgW6f+F(u8WBlDAy|C=B@+8+Q$TN|O14e$=C(7-(>Q<%bMEf_D}{m9@U>J#WP~Y;+ClEv5v4sgkr~36 z5cM!F4q`_S4j+I_9EHNxmJ|g7)Fc=S)n> zLkj1&l9f#egu)y7V3}WY7>x}}LVBfazk8cZn|ZqGRTOMXHLM-rl;eCZ2$Q7B`iyFV z*^bt^RaKVq<)DcdtP~Juv;6%FsJg%V`f6QLyaekVjt+JBh^cUt;>qdN7w-2KwJIls34W*Y8AS`>m-JJ~9$yKra5c5_f% zuzR~{+n@E6+d4>;>izy=DUedtmDUI?L=;YFHxV~P@-*iUFA15J??HjtT@jq^a*jF;6FfpemY){q~49a zm8!w*0>ysT>^^rvBV88xfyKb@PLS+P8tY$+Lv3byiz#%tlB1IL!W!3uRwYFsp%PWc zAc4`4feQNPk9{CZ(O_tRtRf4p`VS9_!A77iOTY%)9D-0Vp9Gsk|1=^)+|Lq)#j9sd zS0`y0%v2W%5N3J!+aG|OVG!Ja(SXw;Di9D7y7WUV-KnElvG(D1@ zvx2xAyuet(DKcQ-2*&{cMRTu1bs8s4lSz!Hj5N^EUCTejUZxjGpY(`k~jQ03NTlJXw1)mUo$ z8}j?p?u$Quv$A6~rIu>C%;6lM-dZ1|>#;DX1TIsV;(W+8t2cIrm)YvHG*Sww)iU=i zStVG$79UU(x{a~k$?%6bks5i%6UIG;K_f=xok1%`CCf_J=sv63BW7BmF}d*X4O|sx zND?OIWrRqNz+7B2mFwhX1t|E;Vcs}>%Q0FDRdpNHSHe@V?iQa|!t%UkV3~lDZsBmKk&zh9g=x|aL*EIX+OX5^#Fh~VKcT;`p3y=lS~Qv z4Yf0sS&aB1?xsx|jLIi{RnWQZOOtif}0^$aID)ey8dH z=(Uq3&W|@QTX~;+&k^B2UT?6~m_mKF@pdu3T^Fn|G37g~e>_nEG>rIjL>|Xa56f@& zrW82rLS(l#EuKV#+vu^~;(G{lQ)t$jKK-V2Uk$)V=9rz{;kLKloDgwoH+EmgGp3ow zy!8A^Zz;Yy9;`Aj^_ce#wB=)&!jgXY+>^gfKTi0PfN>fdX>sZ%wz^4c!E4C?m5FyO zh&7Hdq=LY-EV5JP#fxkFh#4*@azepQmk0kD9%!Yew27cPCEvxgA8{JmF{8IRPqFj{x#8&BbK{t!{z_RtEX8HN2j(_c5`jjxF*hp{qOU5hz&1zXfHiby) zwlQjI-K4{3T$3?s6 zqeA#6%m+j>)R#X^0{eWFe=-hNFzKW<`HTiF(uaRnp)HsJq7cH1(Ic%(A%9XuVTdKT{&Pzw{Agb6$ySm3w(Pdb=TaYo;T`CW+3UA3T_@( z141sU_bD_KHw&&z-bl}(Ef=iB=S~4 zBENCBp-QZeUYuXI%Ogg-@QO7i)mH6awj09{s70xvN+DW}HqQ=aab@A2i*r#v!#tH+ zUV3U7UkW?V1TtUFP=Wx?DD=Z4rzQvj)LLVPBH_>!@-r|@6+0%r6{=aa&E7<+3A?yt rMb=5mW%rjDZ2c_G`~{{|+IL>r?u()fDm!k{ouBmIH7NcUP>}xtSZi!t literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..fff41de312634cd13520fcdd178539ab286147a6 GIT binary patch literal 451 zcmV;!0X+U6iwFP!000001C5j2YU3~vh41$%jIUCti!`>Zq<5uTXdtzhP4=eC5@c&q zRBXrSr%*!QeQi13kk+&d#>jKNk!Fs@had>IN`}F8kQdKb+NGR@X_`nZB%=}q!<31F zBNc>&7ndFG20U|hfqJ!Xo@Pnxg&`h_stWs0W1W2c^%3@LBWm=tN1YpUNlN2+i4mdm zT|@|B{fkCW@I?tYUjUXq!@jmd{%yWm-Q8}MdFJD#Mq$;y`7Z3B1A&Av02(vSxM%9U zR6A9E$w@S&R&9@+N;RTxD>QxJcW{_;0~Qy-+vT(tlTS3-X~7@wj;q(FbK*Xvu|i>d zt#oi#Sm))#$@=i4wUbTAvFos}rb}-u>PA%Kc9WZDd3JYxz`w}nhfG-eg4&EQTv51U z(K@^NvA)kY!!_>fT3IibJV7xRI{`!{DRfIhS-JoYgd}oTaKeGIBtEvH@ktr0zr5T^ z82jJ{0p%1k>?W+$s&E!ZGluY190$PP*7n6bi1r*%=Y*O--jC{Xo%hR`uO#XwA7De_ tn(^*=9Dcp+)ad?CUW>X`rnmg3>!#|aZL4G7U4_Bhx4+LieKH3G003G|;+p^f literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..72820a3d31c9a6575bd331a0cd11f2be9fe9afbf GIT binary patch literal 2659 zcmV-p3Y_&HiwFP!000001MS^!bDKCC0Py$sDLj7Fo4LdY1em-!Z{u#gN$g(YZf5W1 z#v>%*(8b^d+-;iacRwMF&EF)>Z4%==Z(3Ujfdq)(BV=;d>2zl<>vm2$6Z;FN@rb?a z8U|+EW*TCg=-kkWZE|GUr2F@Oi0rF0`*^jmd3YZ9^LM&TPm{ad>re9A-m&5nd$>nb{ z*$Ki0iRFUNt;@&V{bb=eQm}>GgUy!Gia`5E|G) zk1Us7FIaM)&Pn$F5hXG4f_X0{;he|4m`Cvu6KTEa@Z!_ta&SKTFgZURUd}!aFNeK| zN0A%&Q7;JRk<8p%hC$NsQ4jTSPwv-}#5dcv47tZiBrA;MUWc5-qKs1#r>&qWDpqt= zMVfBvx?E=wFxPSEgZUVbXy`7JY%+^HZk;d);*Jw|jE6E8{}%JmCti}pAzsAgA$GsZ zViox%aN<8nC=>2-2P|vVuRM&_+sKkaz6>Ha4#L}fDh(D3H_oU0$!(FXP}-7&WD#X4 zoge8wE_vuKxF2Wx5OaFt2jb8aHn8CFO~CRg{)&3sr331+>H5UlWG4Q~Av9ivq9yV> z;E^~vr}Al@Yloy*8AgixO~u|73-X9vY?EjyP7c@h3hR}QQkG2OICSlmIEji)@g=ZF zxoa^dbD7~ko&T;+C$U@XgE+tQ$$zYf=f=1113wM}FKffEBy`EPN!MSmcznUdwrNz@ zok*N-rSx=%M9FhMe`p*M#s8u;Wkh5V2Z0w|#U!p>@WCVV%EFtdcC$1PryZGZPRcp= z`TKR+rVLU2EcYVW#eADMxi(5OMcHbiylu)VODgn%n@+3tO6%rNkZw0E%=Nby+FeuO zm&3v7Z^bQalkVS4O%IT}GP~+AQjhhctMTB|)yK)STf88<)Wn?hi9=L{VM7%k6{H)M zYO0EYnPZ!ZDf(@!tt%7qxf{jdgME-EPKFEFUvbvBm&NnY^%?(mZzGK-<6*IlHYt16 zM4PVtNcQcbx%?>Kv#c*5-w$HhkfQ^r`TD})H1L_|F*jA(TSk{XH zCy(usq93X16sae=d7|t8P7yn+6Ve0gb$fS9)78n&W+qD6%$yN| zkc|=Ldht3)w$OGqCv`~wDFS($^pZ+G z3VsE@f?xkHzY2n|cKfyQEfuO#@oU1e-4Sc^GGdX@r+rmZ9EXwO1(9swz!58xYb3i( z2hIX#fwRC_;4E+!I18M8InKrrkK4^z+Q?Z{&)M`D_!CrXl~t=F3oDFK2dQT9y2w3e z|7%^Fo}PQ7+O(dHxXmwarE#-V;2v;qFDwJx1MUI$fO}8mUdWjjIPC@>Zv>yZ)ARSb zxvIKSovVs18}}Jy4yJGC`8$T^?=WBqf(5~XU_r1TSP<;K2)6yt-z}`~#=qpC;wL|UuYHYx z!zx3wOx3vUsE)?j+j;(u;rTmo54Z>1+q=6za1XfGM(&CI;a<8sUx~NkX``7}dRMszpewsd6(JYclT_Adxu0U`Q||7!nM5D3}66w%Cxi=!ZG+3pcA;S8m3r zLqmhm8{#T8M>8Tv+Qan4$)3Ph_)QH3T*s%K4 zu90V9OHYF5==6MaHEk0;3w3@#o`t;o1y3E^3GM`Uf;$ffVBpTfaAzE@YS>AVnO3#( zRP-u6UvQ{XqEkOs4uG9jBRkEQx8zMgC3YEzZrFuWc4?-8 zk)_+j;z(3GJ0GJNl2xcJT2pF@)_A#STQL;Q#WKq&7Omb>G^#0DSIb4Kx@y}NV_2iK zP_%YW(P~Z6hEXmWIhH70Q4B`ZLeco1qP3c$%}QIF4%N7BI~sD#LeYA1AylJ`xxYL< zLx&J$IE_{oEjqBIcPn1x3>AZj1DjpTP~D{3EOozv0~n8n z&;UXM2n`O#$PgN|H#B&;;bXLK?*T|FPQ7%~-G7FH7UH%e?xsJ7!D>01{W=<63xf;< zMH0DIGn@|3htpyGvd^E+Cxg@aWwPMGcs!X7rlZODs-g7B_4t7@-cQEU;dok~V|~Rm ztXse2`f@ZKK4RBo#UIWmi|eir>nh^mfO-+M_p`V6j)98W7^qbLM4AXxn7VtcVpiV9 zi*;u7NpfkLN-0)R4?R|4YD2h^Er5sxA{K~PAYy@t1tJ!RSY9b&Q6XY^vx0_*Wv?R^ z+!V3k>L0~RBNnFZ9<-3kFH|w73RbzIQ(k_%wDtyq7N#}FE!i>%T_ALU&;>#l2wfm_ zfzaiZLKm%dp^H4_9_#LP<{i=4_+wLC{nK+BG%~!q8>vxtqtUks!8Xxw-zm8fzYRAs zd@mid#h^#fBj^$I2zmrPf*xNTwc@n7rN&qE*5VKzlO R+gpS-{{tjh$f-Eq003<|KjZ)a literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..ed972760e0b9c80b3165851c4e280c386cd83e40 GIT binary patch literal 897 zcmV-{1AhD;iwFP!000001MODbkDEpif6rfmc)Cg@f;E_Pp5o@*)@lNk?c|h3m9?3qi2Ij$W+&v|(^TJ89-ym0Oa^=qFA%ToU z0RbGrJi=KF0rq3}n{N$VTX>?&48&_*-Cc!YE3kZ*RwQs6S}IKrZ_m-zN@vE!{ydm1 z{K)fz>%#MWf8P8B9*J<6#RVxKT(ssYT*BBvfz>^ThWFq&&gX_z@-tR}OUe#wA{*`p z=hueEc#ak2RR?r2Ap&HJs5D~@q~=m#_KFi2a&xK7SijMpzb*tMUr!q!q{&K6#+(ZwGx7h zb0-C5Fr;%{8kHl&8j#MpLsDe1AEO`&XI(M^?xzynJ^m#|J0qbeag$VjxmN!$7h*ZNyRXcnj9s zw0XR0v-8YH1)*JlUPQ6^{bD~~`>Cz?J9;&UlFBByjW(0MtZUZQvd`MI!3=QT7+%{V z$8fzsK@oc)4stIhD* zb}NIMkiA>4ZuiS1Nq5WLI!(6Is3c9kpg%3S-K=-3r%l`7p2oH2+^$kuf>`EE{y$Q7 zK0kZ-iaTrZmv>h5=*~i4eP@Mdch(||i675n^QM`d-C5{zXI<{B%boR~y0d2f<<7d? XS(iKO(Vc}%>7V@x8!vTU?+^e0?Y6=? literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..432b409764136e6ce1ac828d4fd1f48a652532fb GIT binary patch literal 368 zcmV-$0gwJ4iwFP!000001HG0@OT$1Afbaf_u%`;9>?@nrn@|seL=ZiRlsrb8l5R?} zsY3C;n{3IYSy7W^k29IILEpsMpy>O7v)Io8<>>a@!Ds(+%BAoc4}vrC}5FXj6Tk&po?j@p)qE_v}@yKoJOdTI@Rc7(Btju zIa=LBPssbaIQ|1nIsn6c;_qVF#D$R$8O<$hkuE6x2EPw$#07H#n z$OJIXHj3KYD4+%~Ce=&<6F37C*aO1{V2oVwc*Rg>ilO!t1HzzSW>gFt3x;7=VERrr zz$}IX^O38KuRrjdPrFXz7m6`we3(5x!%GeS;Zj3KgK=gWX3;dmOO5f@G{H2?nQ7QB Oruhb*G?#$}3jhFq(ysde literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..9290eaeeff1501327299bb064c284de6a4ce7795 GIT binary patch literal 556 zcmV+{0@M8;iwFP!000001BFv zQ$rI6#~ItI{&!-hFX{x{7whi1m*?($w{aYIA-L=GojCakslMSIHw*&~2`88XqYzS- zM1Xre>b|^e+*@x~L#17#*YyrzD8z(zLl5EZ(nAQP=E0tiP%mW6=RN4!h=S!{Bj)=Y z@=pQELak&l#&c0r2K?T@DjOL6Qw0h)<`aOp&JPojDqvd3V#IWtpCA$-N(lC8M2XFs zOD>ioJ>wTxzfwsvjvc>EU&y9sqZjphZTV!e6j`loGGo}7vTMD9LhmSIC9*>oo5S$p z>3A}HJ1~vthZ3G*E7{EDK;+8Oq&iSL+qND9=i-r_7>z;8UvR8vok=D1Dz_xpGeSe# zU}XK}Z@AVH);GL=X!2GKo+`Fb#aSZAL*RDy^Q~r8TSc2fI(e z|F=iaRhf&dB^uvDS|7f$S-G|rrnM4Dz1zHIm@VAg>`>Nb=mwtDc>?OaEV52O?U62? u-oni+He$!t=ppJ6GRLG(qdq0y+QVM_>o(5B1x=i@eeo9zE)=Ch1ONawEet0B literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..41e2ae77fa880537b8e49d22878c320da42dac5e GIT binary patch literal 662 zcmV;H0%`ppiwFP!000001MODZPTVjMeeYLTd7@Hxv7I={?o&~z0JQ}KytGw?Ch=r( zm0N9xiX#5KiIZJ!;Z~_ds(4A_9l7R(VIVP)K9dLn zLjg&IY?g5E^_!O7^z_}PNKk*Q%EcOUcn^DwQo7|i;=uHPhRmlyG@&fcCha`yA90w) zGhkVeW#CqnP@r3}ud6N@0!h_aVyXhk0?YPg954G(I+{ge-(GH*Y6PpQauaMHhEpC; z!8!9qBm&1NDycG+KFE`3QmyKa<|q!V^a4s{iVit%$0k)++AiMiGV|@KsEC2Oo#=B- ztD>mN)x5*N7%Y9sSHBieo`jF8IL%nfy|PKMx2SjAIh$Ap^ViJc0~@xesUdz^3Kj_Yi(?$ zaaD{{&S?GRKuvV+2u`6N^AKz@_ox{EEbIolwp<=@M5#w_a%|-B~*W~e7TR*Ul zT)aAeg8GBI)X%S~zb@(#?K2QJwNFtdaYj?dqS?6iKS9~HdC@i_8KB$RUv785^2H;S zFGkxkR~Zt}mUJDT=W5YqWX_Dm;>8r6(-g$Bn}T?RDTse=3gQnHD)A{{!{l32ejy*k5;m7n#4v{J)d=Z$x7>9{LLa04*L(vj6}9 literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..f8a2ba4f5fed06fd632711d52c78872d3f2af887 GIT binary patch literal 2154 zcmV-w2$lCAiwFP!000001MQt#Z{s!^hVT0;3|&o&I*=$)vU0UX1zR<>M89yA!tf3Hrqb+BS9>nh6Vy$K?c;`i}j@_7kT{4y8_OUDuh*Mqt78#lwL z^+r%t#o~sgNET<5tCSfUmvqY#zW*^wzD<+Qwkw1C@4WTH8=@x>X}Az9B_Vzc!maiA zuS#Mi%_Pf3V4^C?s$>pDX=D#YQB>kT(ooh^MbfFkbe?7BZGDr+?|eDfN@CV*#Sc#w z4@}?LgpRj91Rh>apSQl7J`BFbj~m}Jc@Q}3hh6?rh_yAtOPh-ySQz?_u?=~U$DEri*13B#KB0kz}e-HLWbqa%{&unXaaft)dsrIKXCZz>^;IDJoOk zM0xL>wdH(vtf+?bT_i5k)GF5QC(jRyVL?=p+g)Qd{S_SAKCLom9dduo_RV`V^SjCT z=08OXhoGm;9I3fns|e&uB3E>HKO29ze>-0YyB_5v?Jt;`!r6$~jEI(^lIUMiqN+|b z5{*fgZD^uKB&x{5uKgdJAoNeRoy&soebUa{s!QWPuxgzMv-x5=5tx8-?Kn6>DWzgrDZK#ohd!P;%qL+sUlpM-uQ%N7Br4 z#>!quSF~jR?PPlUc2V#k3wovxy9^U!m0C!lqTs=7{Jz){Nx(C$r$J zM3v{3>HR$Z_f0Y~3`II5WFe7js$Em{zbUjWkIgmbytm>0$_Zlk?zYh^INogBY$V&4 zdq2NaVQ(m5Y!bdUPv7Cb9K|l99w#rA%b5s%6Iala_=`Y^j4P zhP0((2vkfrRE$ez5lg;UxVoKnDP zH{jIj#ImebOT0>vc-6+lt5>!oXc9F>nrcu*v_QNdUJx&c7sLzVeJ1ge0_KqOiB}$7 zj(Dq1RERe+*T$l3#gi9Ul&$!1ihI>da&N_Yj(gRYfI*WvlAFI7Vwq1WmNTGSC*V;x~sws(jU zkxZROey9p`1at&+1at&+L|=5o<+pd#3yD+PJCqB4cjOEup&a}0$c5L=pO6vlterR3 z5L#OrLW73rfQHberrSD0T)BA!koI@Kg<={n#Pa0)mDoB~e!0;k2!PrJK$V*UYV9YRWvvLPdSgK4av4x5m2JgVMSDbS z04e|#fC@kbpnl7MS|tKtKnoZ^PhtR7siNwpXj85jmPl}O126y>a1sN^tHXe}nb>vI zEaA27xfb`6|Ci#G^z+pw{!xGvzzHwT3Ee!OgsUBrG4SQjDB)^{o}E$BD<4J0B?d`_ z;A)44S38u1j)JQlAao#fAao#fAauUD?WDN-;XI|&(N8hp?uXvXoJ-yPP{LjncRv7o zfxW<9U@x#2>-)GIqHE}j%OQG?zU_R4tOUj*Tn+&+1{ed30mcAhtOw)vi0+|rKW>k} z=IsL*H)L3byCcA`U|29L7#0kR#a&!3(H(Z}$K?`W*Oy(?r9~jNYd>m$(~JOO0P%j@ zlKTO~0&`sfSPU!%7UN7~XV4gD8hf_mf$`ID+_%$v=c$0c)k;ITW8+LC&WFz1g@C*0_`~rSE0l(c@>&N-V&H!*f z&NpHsOoO%lzkMSNCn7e&0FD92C*fGR92{$Z2OLKM5&-$N0kQ@_?l>T0GfYSDxF4Hg zx`D^qW$;*0o{z_dcp8uUw&~+M9niO0X(o7-+Ix(!6$XL_g6A~}9s2Ag620)ua6e}UBi08!gGjQ{`u literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..0ea0384acdc22a60fe901ed3b991c415086f5d57 GIT binary patch literal 5240 zcmY+9Wmpq#xW**~X=xFLz^EaO)Ch^uB?yd=W~4|+2n>)=y1SHtv~Pu{l|Y9GN8pJ!%vq&hiIa5GyCKd{8& zNOW;folqvRgD6L_Wu0j{PawOH%hLTh`zdj_k5j4=InOci=4b03xoWx6{*s$yDkWPf z(D+4_**yaX+!D+;H~6C9_7K=HEva_fCv%9`Zq>o_wT~}$k7(ZIyP6E0iUWyA(dRk5 z37}ozkv*bQEld%V4__3MY;nk$WJo>v{h@8q`d&oSIC?O0rgfoc3Pn({97nw_0wIoI z`$0a(a>wh`gBMb(q@JZM44x!(7T;&m&&2 zSIx%sEpaFd%bl|#G*bFUiId+>v8`eL3f!}(H^J29Ste+(DRgh+E%vTL57~<5$}w!=9c^@Z=d2krAT zhGBY5tT~FdFS36z+m0aDgmgYPuYNDK;`#ixQEiIIgS3OtLxvfPV?@7RgmZekKY^1o z6tBR!BiiZ~xvYreBLA%8U3QbrD~`&~!wH++=g{VL+;R6Y`<80IqAG63=bau}__S0Z zmv9Zq{hx%d2=>+kvwnx}E}WR8{BXnc$}(;5yem0Pat*zBtNFt7FlZY4CB#kneCvQ= z5Mlc`o;s|%|1#ABL7w)J}15ZO11K}+vcl~ zT)tr}dCI#BHoKR{{SG7{2|OHm2tJJPG{SNkOeiY>%ampNh<|Bd1dj$1i*A9tJJ`O9n{BY=&XEkep_$Dxj`;&sH070|qk0b`1t5Tb`COw=kp_tFh0H zLvaVKRC1qBEhVa((XgC-`dpnA@;KKif2OVbU9oCybCGJ|@|MD@Po@O{EF{xQiOZiT zi;WfxNb&3?8=SP&uE!Tt_{9B*A_SF7Y9<)ITR4HqY_YzJgTo(NahaCh77OCjf0|6+ zcpzBmL~b4D9tnzbc%49{pR%rk?;2>TDf{Jc;EaO#3>cfS98uT~N zmN?9EkSh=a!E1gf!SCHT6qRR|$HPe;HL#K3S6=2J{3&5wpG|7cmZ-_x{60!Uj${eC z`tR=TZlZAOPdUnSwrMB1g6yVyP4irBVj ztuwB={{smLei^5K= zV4)#1sLw5?a*Qq*cLwI)a;j{pX&=j#UTIH*QA+Jg9mhd+!F~wDMJ(((UtXplNJH&W${Y z0Fq4wG|?2*uT|XluI)Ct7q!>8GozEKjThyWWRax_VrX;rKb=9;?a<`AsNHuGt0ir0WGhjCPR%QsY(e9mTF){ zJx|uA@)cNhLdM?u3pPl>@eF~KxH-x`wY^*lxL5KuX3z)H>C3-J`8fWSME#<0h?wA0 z1RbrmUSveGQ0@n>2g2WLJPcaV(YX?!Fko|S<$e*4pGdjnATHYH20C;R0Ko{CV}K}5 zk~ue?;Au+!MvN3Af3=|cx_p7E<~5|sM*Vx+(>z`b;k^9Yp(31cek*9Z&h?umj7g&T zc4tv#6eSnt0{?;cp0=*m$D|w>GLojmT*LS}%!7s(1wLtul0UFyNaRERV&EUCHW=%- zS$*2F011&gHuNpf88Y7%yy@8zGa5z~?c(-$GBdSmylX9P$Qug7O^euUq90)AkjX8Z zy#^$WoHA&>ij|6(v*gqX%WD_OXARcM4<5&kvbNnp73`uDi^#Dqq$TYca>QcE%l}#N^!7MyTU^q6Ba!%rtjvQ` zV&#upGDp)Vx*qCV7D+GvC`kubHLK;SElAy)o&`Ttm-qRSFF_6j|IT?oeP4mJ@!vB< z>WnWA3oFKpZKvtX7uDxVdM+cF*Kt?_<^df`@vSw!XG zW2RVY<1xnPT>fWWF=Yx|7&e~@dw)eHxX_ug)5THv?|>kvObOR(bH+Ze3{zNZRm02* zq81u(FT@{LJg2f#}=A5bm4HVpu=uY+PXk*q9^0iLfyKy!0~Sb z=K0a9N%^r~=lo03f#u-R4lS?RP@-e2Vf2 z9cu%IVTWrE!6cyY>NfcTFxdm_p!3f@)PzOs7o!XB7=7LLz~SAjP@+j z7O@~f6o0hUdECM8zg5mX9n}%o?nu4LUP>%!f32kOF!o8*g;%W-W3yJtfOh7NljAK% zfic&Wo&MIRU(?1;t>Z#db}b^A&!>)M(E+m}`(kRc=DmKYFz^$Qu1|?WS1Y<#v$g<) zbARApZ?UJw1pHhMUy(TB9{ zDAlQER$vAxC@Jm+yiA33C=x<1|c$F?ieb^b=$+H;zw)pX|Q`PlA0Ggni84 z1}t><=g^snYE;9Lr_?CFCLn&BdE)N?FH}1K~K3CUJ`teUAo??Kqh3UQqp2Vw}Ii7V>`p^PH!6l@TP8|1onzUQFflF*}|S=W1{BpMWA6F4H!51hHPdWVtmdYJjQd zbQ>+(!nli=tCARg>YZ$AS;YS-FmS-%G~3SfQq9`}Uvh^{_ePN5*vsyb^4CusZ8OQFGxUAo6{zrsWenE+Q!tf!jnJ%1 zd(dhe_=7bTwBnDV~-k@#iwqSBJCYdp&V zscL12$k-Uka_FZqum0V~tm);+@ELZpZr^qV*YmECI;+*AwIH}azs8vZ51DVoV(z}z zHtbHXl<wwOZr)qLb9Z_ZGJUtk1=UNaEeAP=N->l2D?KuUly7fe z1s|^GIv=Gr>}lOmrB-xv`i5f{@NgUk3dXHTTI3uwE}QX|ZGstPcO7%zsvE$xoW(VI zD9ud4N978;M0baQ_!AOQfV2mE%;BlYu01jw+-tVndd6dm2|yT>sQ zu-G9!oD~y@!M-n&h7Ow>3QbXb7u_6)bjIth#Vf|{F#j?+sv;LKPgG1BlfskTwpoj~ zG@LEJ%(8Nr^1<(d?RT8+@s;~T_@#9H4!U@;p1~e(i-pUldWy4xH$M?}iR}YD3~Icc z-zaw;{30Ph+)Qf1H)E7$)2t~W)nW97KYUi5y{i_`;=GxS`&x_6k1mS2s`%Fg*iIV$89(N&wY(xWQe zk!&RBUvrKVn?Oz5Y8^&6mXY}W8&S3^5e|F&{D>ya6T8fWihI%lztbyE&>EhiwCRC3 zFeeBy!xD0R>*H=u!=gaex|}#sN|>C$HerR z;yn87B|$6W&L*jT-h_q1;6E#HYudr#&9S)rNag7c5R{uzmi=bcnn&Mp>IHOdE}Lx7 z_{rFb>JwKJS7_Yu<*{}=JHH}b$GN+;%MCh6Tn+8fJnxiEVMb1Et9_0*y%N15`3f3d z?*AP=`F#o?-(q;uP2gpcHkSRot+OD|kaf(wCdT|o64^N}s-Z@wRU+htm(fb*JM~e~ zN2)c9p^DYy3Ig$;EY12kcl(F?Y_jJ?qpJLkmydLND@(MGqcq1dP}g77;n5sCNM`Af zD5vo3+`dNKS$L_AZ?SIfp0BfQ3rZ^&InEl^CW36xQS!JAF7X?mY1S>M z_^sQ3mg**r>fM*NCDfh+;L}5~8u{4Zw8LugF^WjTly`^oL;FB3?ZBvXez8+>1>RN4 z6<=?1wl7j(@<_Q4jq_bjOb+ruy*UZrlajE88=J00ChvX4JBknNv%?XRBgn^O!anMV z1%u3RJ2gok^}>6AV})Hvna`V$veGsJC@eXO0D2O+KmgsqKx<~Q%wC{uEzFBOLU^!j zIpk85ZxSe~hg1c@`nTF(p3!u{pIzK$9;He>IS#@CmsFzJgJ==lV6EhVmpu5noqu!- z3ir7T6!KptRbmY?`jxnbjTRzt zCn+a)mHJq(Zb_Uod1=I4Lp^@|B}mMom;y9597b0g5>@c zZIH#EMhltP*dV!AZhzBEjH{0jR@TD7qXPUA_!9d+x9BR;%IO$aZBA$rR;|kQmL5^G zz4}ox@njSTD{~LxsF-_7=v#u|C{Ih$o#M5pPj@j@e@aL-PpeH7$S)(_Cq9}avu=gAKG^tJ69j*+-1Fa zqvPf}lVR{mG=Rg2k&TeQN(n>0Z{c#hnQWPArPwqS*@GEbn<_nPsifV4++h~_%)T)A zNcSdPV*t(Ze946z9;~r%*VMGkJXr5!7DcwE2 z?L$NwpR(RRnT~+3nY{=YjlGGq23^GmxH>7y9An5wW5HMPN;1b=N=$!IfY0gQ&jr;m zZsq|nQ{9B8tY#7^UjUfuci6XmsX>(gq2Ozyv8^HSwHdzu<}oY;dH+iO9?J({&Kykp x&&wBw4q29%(WZRI?;S>XH{T;h8yh7$BX2PAZ`M$uw3vG=nTkBIHV&)@{{saXS6Bc5 literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..aefbad4adcc9824b7e685b8668273d2ac4ef1b67 GIT binary patch literal 565 zcmV-50?Pd#iwFP!000001MSw&Z__Xo2k<+8MU)drEJ~c%NjR-qC)lN(7&{<@rWe~O zX8DEfutBx|oy1YMC0&NNK#_clpTB*zUq7m{bsYCfFxPqGth2A2>NEDzB_v`zV<=!8 zFeezYm_VxMc>g4TW2sTae zD3wE@xO3;+nX6Ws%Lbj4LWdbAR!n_{{CTbE$S=vJ_@sE{!7&3FLr$77^4v`T-^>ijnU}EYD&;MgfjOcXyp{ZLMK< zdM4EMrGkw)$x&c>H|?ZS!tJuljS44CJFJJMjjVe&W-6bY&%+t3Tx@o88e+L9x>-I> zf!0!FZAW1gylOzHn_3RTjq@zb8NzvzpcDa1<1kC2EFb|6G45Q_lw^pL?v5|Ee_$EL zmgQ!_N>N=ZE`#w;*_V*B>X&2K{R2AYnec#m?e$-AZe?59zmrX8WutxB;Zw5PZo>DFu2hfsj7{D`tZ8fdpPBvznY^^!t`-0Q D)f5+P literal 0 HcmV?d00001 diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java index 48ee2b0cfc0..84806f9a959 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java @@ -19,18 +19,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; /** * Contains two methods mainly for testing purposes. One to create a dump of the current testing OpenCGA installation and a second one to @@ -89,12 +88,12 @@ public static void dump(CatalogManager catalogManager, Path opencgaHome) throws } public static void restore(CatalogManager catalogManager, Path opencgaHome) - throws CatalogDBException, IOException, CatalogIOException, URISyntaxException { - StopWatch stopWatch = StopWatch.createStarted(); + throws Exception { try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory())) { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) .getMongoDataStore().getMongoClient(); + MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); Map databaseNames = new HashMap<>(); try (MongoCursor mongoIterator = dumpDatabase.getCollection("summary") @@ -107,25 +106,68 @@ public static void restore(CatalogManager catalogManager, Path opencgaHome) } } - List organizationIds = dbAdaptorFactory.getOrganizationIds(); - for (String organizationId : organizationIds) { + restore(catalogManager, opencgaHome, dumpDatabase, databaseNames.keySet()); + } + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome, URL restoreFolder) + throws Exception { + /* + dataset=v3.0.0 + mongo test_dump --eval 'db.getCollectionNames()' --quiet | jq .[] -r | grep -v summary | while read i ; do + org=$(echo $i | cut -d "_" -f 1) ; + collection=$(echo $i | cut -d "_" -f 3) ; + mkdir -p opencga-app/src/test/resources/datasets/catalog/$dataset/mongodb/$org ; + mongo test_dump --quiet --eval "db.getCollection(\"$i\").find().forEach(function(d){ print(tojsononeline(d)); })" | gzip > opencga-app/src/test/resources/datasets/opencga/$dataset/mongodb/$org/$collection.json.gz ; + done + */ + if (restoreFolder.getProtocol().equals("file")) { + restore(catalogManager, opencgaHome, Paths.get(restoreFolder.toURI())); + } else if (restoreFolder.getProtocol().equals("jar")) { + throw new UnsupportedOperationException("Cannot restore from a jar file"); + } + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome, Path restoreFolder) + throws Exception { + + List organizationIds = new ArrayList<>(); + try (Stream stream = Files.list(restoreFolder)) { + stream.forEach(file -> organizationIds.add(file.getFileName().toString())); + } + System.out.println("organizationIds = " + organizationIds); + if (organizationIds.isEmpty()) { + throw new CatalogDBException("No organization found in the restore folder '" + restoreFolder + "'"); + } + restore(catalogManager, opencgaHome, restoreFolder, organizationIds); + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome, Object source, Collection organizationIds) + throws Exception { + logger.info("Restore opencga from source " + source + " for organizations " + organizationIds); + StopWatch stopWatch = StopWatch.createStarted(); + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), + catalogManager.getIoManagerFactory())) { + + for (String existingOrganizationId : dbAdaptorFactory.getOrganizationIds()) { // We need to completely remove databases that were not backed up so tests that attempt to create them again don't fail - if (!databaseNames.containsKey(organizationId)) { - logger.info("Completely removing database for organization '{}'", organizationId); - dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(organizationId).deleteCatalogDB(); + if (!organizationIds.contains(existingOrganizationId)) { + logger.info("Completely removing database for organization '{}'", existingOrganizationId); + dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(existingOrganizationId).deleteCatalogDB(); } } // First restore the main admin database String adminDBName = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) .getMongoDataStore().getDatabaseName(); - restoreDatabase(catalogManager, opencgaHome, ParamConstants.ADMIN_ORGANIZATION, adminDBName, dbAdaptorFactory); + logger.info("Restoring database for organization '{}'", ParamConstants.ADMIN_ORGANIZATION); + restoreDatabase(catalogManager, opencgaHome, ParamConstants.ADMIN_ORGANIZATION, adminDBName, dbAdaptorFactory, source); - for (Map.Entry entry : databaseNames.entrySet()) { - String organizationId = entry.getKey(); + for (String organizationId : organizationIds) { if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { - String databaseName = entry.getValue(); - restoreDatabase(catalogManager, opencgaHome, organizationId, databaseName, dbAdaptorFactory); + logger.info("Restoring database for organization '{}'", organizationId); + String databaseName = catalogManager.getCatalogDatabase(organizationId); + restoreDatabase(catalogManager, opencgaHome, organizationId, databaseName, dbAdaptorFactory, source); } } } @@ -133,10 +175,9 @@ public static void restore(CatalogManager catalogManager, Path opencgaHome) } private static void restoreDatabase(CatalogManager catalogManager, Path opencgaHome, String organizationId, String databaseName, - MongoDBAdaptorFactory dbAdaptorFactory) - throws IOException, CatalogIOException, CatalogDBException, URISyntaxException { + MongoDBAdaptorFactory dbAdaptorFactory, Object source) + throws Exception { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION).getMongoDataStore().getMongoClient(); - MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); Bson emptyBsonQuery = new Document(); MongoDatabase database = mongoClient.getDatabase(databaseName); @@ -152,10 +193,9 @@ private static void restoreDatabase(CatalogManager catalogManager, Path opencgaH for (String collection : OrganizationMongoDBAdaptorFactory.COLLECTIONS_LIST) { MongoCollection dbCollection = database.getCollection(collection); - MongoCollection dumpCollection = dumpDatabase.getCollection(organizationId + "__" + collection); dbCollection.deleteMany(emptyBsonQuery); - try (MongoCursor iterator = dumpCollection.find(emptyBsonQuery).noCursorTimeout(true).iterator()) { + try (CloseableIterator iterator = documentIterator(source, organizationId, collection)) { List documentList = new LinkedList<>(); while (iterator.hasNext()) { Document document = iterator.next(); @@ -198,6 +238,72 @@ private static void restoreDatabase(CatalogManager catalogManager, Path opencgaH } } + private static CloseableIterator documentIterator(Object source, String organizationId, String collection) + throws IOException { + if (source instanceof MongoDatabase) { + MongoDatabase dumpDatabase = (MongoDatabase) source; + + MongoCollection dumpCollection = dumpDatabase.getCollection(organizationId + "__" + collection); + logger.info("Restoring {}:{} from database - {}", organizationId, collection, dumpCollection.getNamespace().getFullName()); + return new CloseableIterator<>(dumpCollection.find(new Document()).noCursorTimeout(true).iterator()); + } else if (source instanceof Path) { + Path dir = (Path) source; + java.io.File file = dir.resolve(organizationId).resolve(collection + ".json.gz").toFile(); + if (!file.exists()) { +// logger.info("File {} not found", file); + return new CloseableIterator<>(null, Collections.emptyIterator()); + } + logger.info("Restoring {}:{} from file - {}", organizationId, collection, file); + // Read file lines + Stream stream = new BufferedReader( + new InputStreamReader( + new GZIPInputStream( + Files.newInputStream(file.toPath())))).lines(); + + Iterator iterator = stream + .filter(s -> !s.isEmpty()) +// .peek(System.out::println) + .map(Document::parse) + .iterator(); + return new CloseableIterator<>(stream, iterator); + } else { + throw new IllegalArgumentException("Unknown restore source type " + source.getClass()); + } + } + + private static class CloseableIterator implements Iterator, AutoCloseable { + + private final AutoCloseable closeable; + private final Iterator iterator; + + public CloseableIterator(AutoCloseable closeable, Iterator iterator) { + this.closeable = closeable; + this.iterator = iterator; + } + + public & Closeable> CloseableIterator(CI c) { + this.closeable = c; + this.iterator = c; + } + + @Override + public void close() throws Exception { + if (closeable != null) { + closeable.close(); + } + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public T next() { + return iterator.next(); + } + } + private static void createFile(IOManager ioManager, Document document) throws IOException, CatalogIOException { String type = document.getString("type"); if (File.Type.FILE.name().equals(type)) { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java index 9044dd2f7c0..7c65f65cb89 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java @@ -50,8 +50,6 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -182,7 +180,7 @@ public void setUp() throws Exception { setUpCatalogManager(catalogManager); } - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException, URISyntaxException { + public void setUpCatalogManager(CatalogManager catalogManager) throws Exception { if (!firstExecutionFinished) { createDummyData(catalogManager); MongoBackupUtils.dump(catalogManager, catalogManagerResource.getOpencgaHome()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java index e8259db2c4b..09c94ee33d3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java @@ -124,6 +124,7 @@ public CatalogManager getCatalogManager() { public CatalogManager resetCatalogManager() throws CatalogException { catalogManager.close(); catalogManager = new CatalogManager(configuration); + adminToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); return catalogManager; } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java index 55866e24160..55903d221f2 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java @@ -84,6 +84,10 @@ public static void configure(StorageEngineFactory factory, boolean clear) { } } + public static VariantStorageMetadataManager getVariantMetadataManager() { + return new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + } + @Override public String getStorageEngineId() { return STORAGE_ENGINE_ID; From 5e31a8a6dbf86350a1fdd30d84a8b7434dc597e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 22 May 2024 16:25:00 +0100 Subject: [PATCH 011/128] storage: Change param names from VariantSetupParams #TASK-5861, #TASK-5448 --- .../VariantSetupOperationManager.java | 39 +++++++------- .../manager/VariantOperationsTest.java | 12 ++--- .../opencb/opencga/core/common/IOUtils.java | 16 +++++- .../models/variant/VariantSetupParams.java | 54 +++++++++---------- .../variant/HadoopVariantStorageEngine.java | 12 ++--- 5 files changed, 73 insertions(+), 60 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java index 989ee262807..c19ab26d0f1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -5,7 +5,6 @@ import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.VariantSetupResult; @@ -57,26 +56,26 @@ private void inferParams(VariantSetupParams params) { switch (params.getFileType()) { case GENOME_gVCF: if (params.getAverageFileSize() == null) { - params.setAverageFileSize(IOUtils.fromHumanReadableToByte("1GiB")); + params.setAverageFileSize("1GiB"); } - if (params.getNumberOfVariantsPerSample() == null) { - params.setNumberOfVariantsPerSample(5000000); + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); } break; case GENOME_VCF: if (params.getAverageFileSize() == null) { - params.setAverageFileSize(IOUtils.fromHumanReadableToByte("500MiB")); + params.setAverageFileSize("500MiB"); } - if (params.getNumberOfVariantsPerSample() == null) { - params.setNumberOfVariantsPerSample(5000000); + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); } break; case EXOME: if (params.getAverageFileSize() == null) { - params.setAverageFileSize(IOUtils.fromHumanReadableToByte("100MiB")); + params.setAverageFileSize("100MiB"); } - if (params.getNumberOfVariantsPerSample() == null) { - params.setNumberOfVariantsPerSample(100000); + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(100000); } break; default: @@ -84,28 +83,28 @@ private void inferParams(VariantSetupParams params) { } } // Unable to tell. Use a default value for numberOfVariantsPerSample - if (params.getNumberOfVariantsPerSample() == null) { - params.setNumberOfVariantsPerSample(5000000); + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); } - if (params.getSamplesPerFile() == null) { + if (params.getAverageSamplesPerFile() == null) { if (params.getDataDistribution() == null) { - params.setSamplesPerFile(params.getExpectedSamplesNumber().floatValue() / params.getExpectedFilesNumber().floatValue()); + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); } else { switch (params.getDataDistribution()) { case SINGLE_SAMPLE_FILES: - params.setSamplesPerFile(1f); + params.setAverageSamplesPerFile(1f); break; case MULTI_SAMPLE_FILES: - params.setSamplesPerFile(params.getExpectedSamplesNumber().floatValue() / params.getExpectedFilesNumber().floatValue()); + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); break; case MULTIPLE_FILE_PER_SAMPLE: // Hard to tell. Let's assume 2 samples per file - params.setSamplesPerFile(2f); + params.setAverageSamplesPerFile(2f); break; case MULTI_SAMPLE_FILES_SPLIT_BY_CHROMOSOME: case MULTI_SAMPLE_FILES_SPLIT_BY_REGION: - params.setSamplesPerFile(params.getExpectedSamplesNumber().floatValue()); + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue()); break; default: throw new IllegalArgumentException("Unknown dataDistribution " + params.getDataDistribution()); @@ -132,10 +131,10 @@ private void check(String studyStr, VariantSetupParams params, String token) thr throw new IllegalArgumentException("Study " + studyStr + " is already setup"); } - if (params.getExpectedFilesNumber() == null || params.getExpectedFilesNumber() <= 0) { + if (params.getExpectedFiles() == null || params.getExpectedFiles() <= 0) { throw new IllegalArgumentException("Missing expectedFilesNumber"); } - if (params.getExpectedSamplesNumber() == null || params.getExpectedSamplesNumber() <= 0) { + if (params.getExpectedSamples() == null || params.getExpectedSamples() <= 0) { throw new IllegalArgumentException("Missing expectedSamplesNumber"); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index f59be01a3da..22e23ff997f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -299,10 +299,10 @@ private void loadDataset() throws Throwable { public static void dummyVariantSetup(VariantStorageManager variantStorageManager, String study, String token) throws CatalogException, StorageEngineException { variantStorageManager.variantSetup(study, new VariantSetupParams() - .setAverageFileSize(100L) - .setExpectedFilesNumber(5) - .setExpectedSamplesNumber(5) - .setNumberOfVariantsPerSample(1000), token); + .setAverageFileSize("100B") + .setExpectedFiles(5) + .setExpectedSamples(5) + .setVariantsPerSample(1000), token); } public void setUpCatalogManager() throws Exception { @@ -335,8 +335,8 @@ public void testSetup() throws Exception { setupParams .setFileType(VariantSetupParams.FileType.GENOME_VCF) .setDataDistribution(VariantSetupParams.DataDistribution.MULTI_SAMPLE_FILES) - .setExpectedFilesNumber(20) - .setExpectedSamplesNumber(100); + .setExpectedFiles(20) + .setExpectedSamples(100); String study2 = "study2"; catalogManager.getStudyManager().create(PROJECT, study2, null, "Phase 1", "Done", null, null, null, null, null, token); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java index a14652c67cd..e37374e76ea 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java @@ -375,6 +375,17 @@ public static String humanReadableByteCount(long bytes, boolean si) { * @return number of bytes */ public static long fromHumanReadableToByte(String value) { + return fromHumanReadableToByte(value, false); + } + + /** + * Get Bytes numbers from a human-readable string + * + * @param value Human-readable value + * @param assumeBinary Use Binary Units (power of 2) + * @return number of bytes + */ + public static long fromHumanReadableToByte(String value, boolean assumeBinary) { if (value.endsWith("B")) { value = value.substring(0, value.length() - 1); } @@ -385,8 +396,11 @@ public static long fromHumanReadableToByte(String value) { } else { si = true; } + if (assumeBinary) { + si = false; + } int unit = si ? 1000 : 1024; - int exp = (si ? "kMGTPE" : "KMGTPE").indexOf(value.charAt(value.length() - 1)) + 1; + int exp = "KMGTPE".indexOf(value.toUpperCase().charAt(value.length() - 1)) + 1; if (exp > 0) { value = value.substring(0, value.length() - 1); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java index 76870051795..ed6aa1801a0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -4,27 +4,27 @@ public class VariantSetupParams extends ToolParams { - private Integer expectedSamplesNumber; + private Integer expectedSamples; - private Integer expectedFilesNumber; + private Integer expectedFiles; private FileType fileType; - private Long averageFileSize; + private String averageFileSize; - private Integer numberOfVariantsPerSample; + private Integer variantsPerSample; - private Float samplesPerFile; + private Float averageSamplesPerFile; private DataDistribution dataDistribution; public VariantSetupParams(VariantSetupParams params) { - this.expectedSamplesNumber = params.expectedSamplesNumber; - this.expectedFilesNumber = params.expectedFilesNumber; + this.expectedSamples = params.expectedSamples; + this.expectedFiles = params.expectedFiles; this.fileType = params.fileType; this.averageFileSize = params.averageFileSize; - this.numberOfVariantsPerSample = params.numberOfVariantsPerSample; - this.samplesPerFile = params.samplesPerFile; + this.variantsPerSample = params.variantsPerSample; + this.averageSamplesPerFile = params.averageSamplesPerFile; this.dataDistribution = params.dataDistribution; } @@ -61,21 +61,21 @@ public enum FileType { EXOME } - public Integer getExpectedSamplesNumber() { - return expectedSamplesNumber; + public Integer getExpectedSamples() { + return expectedSamples; } - public VariantSetupParams setExpectedSamplesNumber(Integer expectedSamplesNumber) { - this.expectedSamplesNumber = expectedSamplesNumber; + public VariantSetupParams setExpectedSamples(Integer expectedSamples) { + this.expectedSamples = expectedSamples; return this; } - public Integer getExpectedFilesNumber() { - return expectedFilesNumber; + public Integer getExpectedFiles() { + return expectedFiles; } - public VariantSetupParams setExpectedFilesNumber(Integer expectedFilesNumber) { - this.expectedFilesNumber = expectedFilesNumber; + public VariantSetupParams setExpectedFiles(Integer expectedFiles) { + this.expectedFiles = expectedFiles; return this; } @@ -88,30 +88,30 @@ public VariantSetupParams setFileType(FileType fileType) { return this; } - public Long getAverageFileSize() { + public String getAverageFileSize() { return averageFileSize; } - public VariantSetupParams setAverageFileSize(Long averageFileSize) { + public VariantSetupParams setAverageFileSize(String averageFileSize) { this.averageFileSize = averageFileSize; return this; } - public Integer getNumberOfVariantsPerSample() { - return numberOfVariantsPerSample; + public Integer getVariantsPerSample() { + return variantsPerSample; } - public VariantSetupParams setNumberOfVariantsPerSample(Integer numberOfVariantsPerSample) { - this.numberOfVariantsPerSample = numberOfVariantsPerSample; + public VariantSetupParams setVariantsPerSample(Integer variantsPerSample) { + this.variantsPerSample = variantsPerSample; return this; } - public Float getSamplesPerFile() { - return samplesPerFile; + public Float getAverageSamplesPerFile() { + return averageSamplesPerFile; } - public VariantSetupParams setSamplesPerFile(Float samplesPerFile) { - this.samplesPerFile = samplesPerFile; + public VariantSetupParams setAverageSamplesPerFile(Float averageSamplesPerFile) { + this.averageSamplesPerFile = averageSamplesPerFile; return this; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index 80e3ebf5cc6..5c80958e7bf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -1093,19 +1093,19 @@ public ObjectMap inferConfigurationParams(VariantSetupParams params) { long expectedHBaseRegionSize = IOUtils.fromHumanReadableToByte("7.5GiB"); - options.put(EXPECTED_SAMPLES_NUMBER.key(), params.getExpectedSamplesNumber()); - options.put(EXPECTED_FILES_NUMBER.key(), params.getExpectedFilesNumber()); + options.put(EXPECTED_SAMPLES_NUMBER.key(), params.getExpectedSamples()); + options.put(EXPECTED_FILES_NUMBER.key(), params.getExpectedFiles()); // Variant pre-split int defaultVariantPreSplit = configuredOptions .getInt(VARIANT_TABLE_PRESPLIT_SIZE.key(), VARIANT_TABLE_PRESPLIT_SIZE.defaultValue()); float variantsFileToHBaseMultiplier = 1.3f; - float variantsTableSize = params.getExpectedFilesNumber() * params.getAverageFileSize() * variantsFileToHBaseMultiplier; + Long averageFileSize = IOUtils.fromHumanReadableToByte(params.getAverageFileSize()); + float variantsTableSize = params.getExpectedFiles() * averageFileSize * variantsFileToHBaseMultiplier; int variantPreSplit = (int) (variantsTableSize / expectedHBaseRegionSize); options.put(VARIANT_TABLE_PRESPLIT_SIZE.key(), Math.max(defaultVariantPreSplit, variantPreSplit)); // Archive pre-split - Long averageFileSize = params.getAverageFileSize(); int filesPerBatch = configuredOptions .getInt(ARCHIVE_FILE_BATCH_SIZE.key(), ARCHIVE_FILE_BATCH_SIZE.defaultValue()); float archiveFileToHBaseMultiplier = 1.2f; @@ -1115,14 +1115,14 @@ public ObjectMap inferConfigurationParams(VariantSetupParams params) { // SampleIndex pre-split long averageSizePerVariant; - if (params.getNumberOfVariantsPerSample() > 3500000) { + if (params.getVariantsPerSample() > 3500000) { // With this many variants per sample, most of them won't have much data averageSizePerVariant = IOUtils.fromHumanReadableToByte("13B"); } else { // With a small number of variants per sample, most of them will have a lot of data averageSizePerVariant = IOUtils.fromHumanReadableToByte("25B"); } - long sampleIndexSize = params.getNumberOfVariantsPerSample() * averageSizePerVariant; + long sampleIndexSize = params.getVariantsPerSample() * averageSizePerVariant; int samplesPerSplit = (int) (expectedHBaseRegionSize / sampleIndexSize); options.put(SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), samplesPerSplit); From a65f4a2f226ecd72175744a243d95252fed86b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 22 May 2024 17:26:46 +0100 Subject: [PATCH 012/128] app: Fix compilation issue. #TASK-5861, #TASK-5448 --- .../variant/OpenCGATestExternalResource.java | 12 ++++++++++ opencga-app/pom.xml | 22 ++++++------------- .../app/migrations/MigrationsTest.java | 18 +++++++++------ 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 17356322406..6443600c74f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -31,6 +31,7 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; @@ -381,6 +382,17 @@ public void restore(URL resource) throws Exception { catalogManagerExternalResource.resetCatalogManager(); } + public final VariantStorageEngine getVariantStorageEngineByProject(String projectFqn) throws Exception { + DataStore dataStore = getVariantStorageManager().getDataStoreByProjectId(projectFqn, getAdminToken()); + VariantStorageEngine variantStorageEngine = storageEngineFactory + .getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); + if (dataStore.getOptions() != null) { + variantStorageEngine.getOptions().putAll(dataStore.getOptions()); + } + return variantStorageEngine; + } + + // private class StorageLocalExecutorManager extends LocalExecutorManager { // // public StorageLocalExecutorManager(String sessionId) { diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index def07ce89b2..f99551660ff 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -259,22 +259,9 @@ test-jar - org.opencb.opencga - opencga-storage-hadoop-compat-${opencga-storage-hadoop-compat.id} - ${project.parent.version} - test - - - org.opencb.opencga.hadoop.thirdparty - ${opencga-hadoop-shaded.artifactId} - ${opencga.hadoop.thirdparty.version} + org.mockito + mockito-core test - - - org.yaml - snakeyaml - - org.opencb.opencga.hadoop.thirdparty @@ -770,10 +757,15 @@ + Ensure that only one opencga-hadoop-shaded*.jar and opencga-storage-hadoop-compat*.jar libs are included + + + + diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java index 70ca0549fd3..6d72b9149f5 100644 --- a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java +++ b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java @@ -16,7 +16,6 @@ import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageEngine; import java.net.URL; import java.util.Arrays; @@ -29,12 +28,13 @@ public class MigrationsTest { @Test public void testVariantSetupMigration() throws Exception { - setup("v3.0.0"); - VariantStorageMetadataManager mm = DummyVariantStorageEngine.getVariantMetadataManager(); + setup("v3.0.0", false); String studyName = "test@1000G:phase1"; - StudyMetadata studyMetadata = mm.createStudy(studyName); - int fileId = mm.registerFile(studyMetadata.getId(), "folder/file.vcf", Arrays.asList("s1", "s2")); - mm.addIndexedFiles(studyMetadata.getId(), Collections.singletonList(fileId)); + + VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject("test@1000G").getMetadataManager(); + StudyMetadata studyMetadata = metadataManager.createStudy(studyName); + int fileId = metadataManager.registerFile(studyMetadata.getId(), "folder/file.vcf", Arrays.asList("s1", "s2")); + metadataManager.addIndexedFiles(studyMetadata.getId(), Collections.singletonList(fileId)); for (Study study : opencga.getCatalogManager().getStudyManager().searchInOrganization("test", new Query(), new QueryOptions(), opencga.getAdminToken()).getResults()) { Assert.assertNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); @@ -65,11 +65,15 @@ protected void testMigration(Class migration, String da } private void setup(String dataset) throws Exception { + setup(dataset, false); + } + + private void setup(String dataset, boolean storageHadoop) throws Exception { if (opencga != null) { opencga.after(); opencga = null; } - opencga = new OpenCGATestExternalResource(false); + opencga = new OpenCGATestExternalResource(storageHadoop); opencga.before(); URL resource = getClass().getResource("/datasets/opencga/" + dataset + "/"); opencga.restore(resource); From f35865a33c8ea8c167be52afade2769974352a53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 23 May 2024 10:25:07 +0200 Subject: [PATCH 013/128] analysis: update Exomiser analysis in order to use the Docker version 14.0.0 by taking it from the configuration file, #TASK-6297, #TASK-6255 --- .../ExomiserInterpretationAnalysis.java | 32 ++++- .../opencga/analysis/tools/OpenCgaTool.java | 2 +- .../analysis/variant/gwas/GwasAnalysis.java | 2 +- .../DockerWrapperAnalysisExecutor.java | 52 ++++++-- .../exomiser/ExomiserWrapperAnalysis.java | 9 ++ .../ExomiserWrapperAnalysisExecutor.java | 112 +++++++++++++----- .../ExomiserInterpretationAnalysisTest.java | 11 +- .../resources/exomiser/application.properties | 4 +- .../analysis/exomiser/application.properties | 4 +- .../VariantInternalCommandExecutor.java | 1 + .../options/VariantCommandOptions.java | 3 + .../src/test/resources/configuration-test.yml | 16 +++ .../opencb/opencga/core/config/Analysis.java | 34 ++++++ .../org/opencb/opencga/core/config/Tool.java | 76 ++++++++++++ .../ExomiserInterpretationAnalysisParams.java | 15 ++- .../clinical/ExomiserWrapperParams.java | 14 ++- .../core/tools/OpenCgaToolExecutor.java | 11 +- .../tools/variant/GwasAnalysisExecutor.java | 14 +-- .../src/main/resources/configuration.yml | 22 +++- .../rest/analysis/VariantWebService.java | 2 +- .../GwasHBaseMapReduceAnalysisExecutor.java | 2 +- 21 files changed, 367 insertions(+), 71 deletions(-) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 39de4806eb0..92ffe157b7b 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -32,6 +32,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.clinical.InterpretationAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -68,6 +69,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis { private String studyId; private String clinicalAnalysisId; private String sampleId; + private String exomiserVersion; private ClinicalAnalysis clinicalAnalysis; @@ -116,6 +118,13 @@ protected void check() throws Exception { } sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId(); + // Check exomiser version + if (StringUtils.isEmpty(exomiserVersion)) { + // Missing exomiser version use the default one + logger.warn("Missing exomiser version, using the default {}", ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION); + exomiserVersion = ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION; + } + // Update executor params with OpenCGA home and session ID setUpStorageEngineExecutor(studyId); } @@ -125,24 +134,26 @@ protected void run() throws ToolException { step(() -> { executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID); - getToolExecutor(ExomiserWrapperAnalysisExecutor.class) + ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(studyId) .setSampleId(sampleId) - .execute(); + .setExomiserVersion(exomiserVersion); + exomiserExecutor.execute(); - saveInterpretation(studyId, clinicalAnalysis); + saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion()); }); } - protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException, + protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion) + throws ToolException, StorageEngineException, CatalogException, IOException { // Interpretation method InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(), GitRepositoryState.getInstance().getCommitId(), Collections.singletonList( new Software() .setName("Exomiser") - .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME) - .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION))); + .setRepository("Docker: " + dockerImage) + .setVersion(dockerImageVersion))); // Analyst ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token); @@ -452,4 +463,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly this.clinicalAnalysisId = clinicalAnalysisId; return this; } + + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java index 7769315137b..c4a3b2aac8c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java @@ -537,7 +537,7 @@ protected final T getToolExecutor(Class clazz toolExecutor.getSource(), toolExecutor.getFramework())); - toolExecutor.setUp(erm, executorParams, outDir); + toolExecutor.setUp(erm, executorParams, outDir, configuration); return toolExecutor; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java index a5d2484b2d7..270e9e0b213 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java @@ -295,7 +295,7 @@ protected void run() throws ToolException { step("gwas", () -> { GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class); - gwasExecutor.setConfiguration(gwasConfiguration) + gwasExecutor.setGwasConfiguration(gwasConfiguration) .setStudy(study) .setSampleList1(caseCohortSamples) .setSampleList2(controlCohortSamples) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index ae2b5072473..d2bd9cf63dd 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -7,8 +7,8 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; -import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.Tool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.slf4j.Logger; @@ -18,12 +18,11 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; -public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { +import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.EXOMISER_PREFIX; + +public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { public final static String DOCKER_INPUT_PATH = "/data/input"; public final static String DOCKER_OUTPUT_PATH = "/data/output"; @@ -32,11 +31,50 @@ public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor public static final String STDERR_FILENAME = "stderr.txt"; public String getDockerImageName() { - return "opencb/opencga-ext-tools"; + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[0]; } public String getDockerImageVersion() { - return GitRepositoryState.getInstance().getBuildVersion(); + if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) { + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1]; + } else { + return GitRepositoryState.getInstance().getBuildVersion(); + } + } + + public String getDockerImageName(String toolKey) { + for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { + if (entry.getKey().equalsIgnoreCase(toolKey)) { + return entry.getValue().getDockerId().split(":")[0]; + } + } + return null; + } + + public String getDockerImageVersion(String toolKey) { + for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { + if (entry.getKey().equalsIgnoreCase(toolKey)) { + if (entry.getValue().getDockerId().contains(":")) { + return entry.getValue().getDockerId().split(":")[1]; + } else { + return null; + } + } + } + return null; + } + + protected String getToolResource(String toolKey, String resourceKey) throws ToolException { + // Get resources from the configuration file + if (!getConfiguration().getAnalysis().getTools().containsKey(toolKey)) { + throw new ToolException("Error getting tool " + toolKey + ": it does not exist in the configuration file"); + } + Tool tool = getConfiguration().getAnalysis().getTools().get(toolKey); + if (!tool.getResources().containsKey(resourceKey)) { + throw new ToolException("Error getting resource " + resourceKey + " of tool " + toolKey + ": it does not exist in the" + + " configuration file"); + } + return tool.getResources().get(resourceKey); } private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index 3d8253861a4..b1d7ad8a92c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -33,6 +33,15 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy { public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants" + " from whole-exome or whole-genome sequencing data."; + public final static String DEFAULT_EXOMISER_VERSION = "14.0.0"; + + // It must match the tool prefix in the tool keys for exomiser in the configuration file + public final static String EXOMISER_PREFIX = "exomiser-"; + + // It must match the resources key in the exomiser/tool section in the configuration file + public final static String HG38_RESOURCE_KEY = "HG38"; + public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE"; + @ToolParams protected final ExomiserWrapperParams analysisParams = new ExomiserWrapperParams(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 86bb3760b3c..111c3a88d49 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -16,6 +16,7 @@ import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.FamilyManager; +import org.opencb.opencga.core.config.Tool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.family.Family; @@ -29,10 +30,13 @@ import java.io.*; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.*; + @ToolExecutor(id = ExomiserWrapperAnalysisExecutor.ID, tool = ExomiserWrapperAnalysis.ID, source = ToolExecutor.Source.STORAGE, @@ -45,11 +49,13 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private final static String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; private static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; - public final static String DOCKER_IMAGE_NAME = "exomiser/exomiser-cli"; - public final static String DOCKER_IMAGE_VERSION = "13.1.0"; + // These constants must match in the file application.properties to be replaced + private static final String HG38_DATA_VERSION_MARK = "put_here_hg38_data_version"; + private static final String PHENOTYPE_DATA_VERSION_MARK = "put_here_phenotype_data_version"; private String studyId; private String sampleId; + private String exomiserVersion; private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -155,10 +161,14 @@ public void run() throws ToolException { throw new ToolException("Error copying Exomiser analysis file", e); } - // Copy the application.properties + // Copy the application.properties and update data according to Exomiser version try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile()); + Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); + FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), target.toFile()); + Command cmd = new Command("sed -i \"s/" + HG38_DATA_VERSION_MARK + "/" + getHg38DataVersion() + "/g\" " + target); + cmd.run(); + cmd = new Command("sed -i \"s/" + PHENOTYPE_DATA_VERSION_MARK + "/" + getPhenotypeDataVersion() + "/g\" " + target); + cmd.run(); } catch (IOException e) { throw new ToolException("Error copying Exomiser properties file", e); } @@ -359,7 +369,7 @@ private Path getAnalysisDataPath(String analysisId) throws ToolException { } private Path getExomiserDataPath(Path openCgaHome) throws ToolException { - Path exomiserDataPath = openCgaHome.resolve("analysis/resources/exomiser"); + Path exomiserDataPath = openCgaHome.resolve("analysis/resources/" + ExomiserWrapperAnalysis.ID); if (!exomiserDataPath.toFile().exists()) { if (!exomiserDataPath.toFile().mkdirs()) { throw new ToolException("Error creating the Exomiser data directory"); @@ -368,33 +378,34 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { // Mutex management to avoid multiple downloadings at the same time // the first to come, download data, others have to wait for - File readyFile = exomiserDataPath.resolve("READY").toFile(); - File preparingFile = exomiserDataPath.resolve("PREPARING").toFile(); + File readyFile = exomiserDataPath.resolve("READY-" + exomiserVersion).toFile(); + File preparingFile = exomiserDataPath.resolve("PREPARING-" + exomiserVersion).toFile(); // If all is ready, then return if (readyFile.exists()) { - logger.info("{}: Exomiser data is already downloaded, so Exomiser analysis is ready to be executed.", ID); + logger.info("Exomiser {} data is already downloaded, so Exomiser analysis is ready to be executed.", exomiserVersion); return exomiserDataPath; } // If it is preparing, then wait for ready and then return if (preparingFile.exists()) { long startTime = System.currentTimeMillis(); - logger.info("{}: Exomiser data is downloading, waiting for it...", ID); + logger.info("Exomiser {} data is downloading, waiting for it...", exomiserVersion); while (!readyFile.exists()) { try { Thread.sleep(10000); } catch (InterruptedException e) { // Nothing to do here + preparingFile.delete(); throw new ToolException(e); } long elapsedTime = System.currentTimeMillis() - startTime; if (elapsedTime > 18000000) { - throw new ToolException("Unable to run the Exomiser analysis because of Exomiser data is not ready yet: maximum" - + " waiting time exceeded"); + throw new ToolException("Unable to run the Exomiser analysis because of Exomiser " + exomiserVersion + " data is not" + + " ready yet: maximum waiting time exceeded"); } } - logger.info("{}: Exomiser data is now downloaded: Exomiser analysis is ready to be executed", ID); + logger.info("Exomiser {} data is now downloaded: Exomiser analysis is ready to be executed", exomiserVersion); return exomiserDataPath; } @@ -402,38 +413,49 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { try { preparingFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error creating the Exomiser data directory"); + preparingFile.delete(); + throw new ToolException("Error creating the Exomiser " + exomiserVersion + " data directory"); } - // Download and unzip files + // Download resources and unzip files try { - downloadAndUnzip(exomiserDataPath, "2109_hg38.zip"); - downloadAndUnzip(exomiserDataPath, "2109_phenotype.zip"); + downloadAndUnzip(exomiserDataPath, HG38_RESOURCE_KEY); + downloadAndUnzip(exomiserDataPath, PHENOTYPE_RESOURCE_KEY); } catch (ToolException e) { // If something wrong happened, the preparing file has to be deleted preparingFile.delete(); - throw new ToolException("Something wrong happened when preparing Exomiser data", e); + throw new ToolException("Something wrong happened when preparing Exomiser " + exomiserVersion + " data", e); } // Mutex management, signal exomiser data is ready try { readyFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error preparing Exomiser data", e); + throw new ToolException("Error preparing Exomiser " + exomiserVersion + " data", e); } preparingFile.delete(); return exomiserDataPath; } + private String getHg38DataVersion() throws ToolException { + String resource = getToolResource(EXOMISER_PREFIX + exomiserVersion, HG38_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + + private String getPhenotypeDataVersion() throws ToolException { + String resource = getToolResource(EXOMISER_PREFIX + exomiserVersion, PHENOTYPE_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + @Override public String getDockerImageName() { - return DOCKER_IMAGE_NAME; + return getDockerImageName(EXOMISER_PREFIX + exomiserVersion); } @Override public String getDockerImageVersion() { - return DOCKER_IMAGE_VERSION; + return getDockerImageVersion(EXOMISER_PREFIX + exomiserVersion); } public String getStudyId() { @@ -445,21 +467,38 @@ public ExomiserWrapperAnalysisExecutor setStudyId(String studyId) { return this; } - private void downloadAndUnzip(Path exomiserDataPath, String filename) throws ToolException { - URL url = null; - - // Download data - try { - url = new URL("http://resources.opencb.org/opencb/opencga/analysis/exomiser/" + filename); + private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws ToolException { + String filename; + String resource = getToolResource(EXOMISER_PREFIX + exomiserVersion, resourceKey); + if (resource.startsWith("file://")) { + // Copy resouce + try { + Path sourcePath = Paths.get(resource); + filename = sourcePath.getFileName().toString(); + Files.copy(sourcePath, exomiserDataPath.resolve(filename)); + } catch (IOException e) { + throw new ToolException("Error copying Exomiser data from " + resource, e); + } + } else { + // Download resource + String url; + if (resource.startsWith("http://") || resource.startsWith("https://") || resource.startsWith("ftp://")) { + url = resource; + } else { + url = getConfiguration().getAnalysis().getResourceUrl() + resource; + } logger.info("{}: Downloading Exomiser data: {} in {}", ID, url, exomiserDataPath); - ResourceUtils.downloadThirdParty(url, exomiserDataPath); - } catch (IOException e) { - throw new ToolException("Error downloading Exomiser data from " + url, e); + try { + ResourceUtils.downloadThirdParty(new URL(url), exomiserDataPath); + filename = Paths.get(url).getFileName().toString(); + } catch (IOException e) { + throw new ToolException("Error downloading Exomiser data from " + url, e); + } } // Unzip try { - logger.info("{}: Decompressing Exomiser data: {}", ID, filename); + logger.info("Decompressing Exomiser {} data: {}", exomiserDataPath, filename); new Command("unzip -o -d " + exomiserDataPath + " " + exomiserDataPath + "/" + filename) .setOutputOutputStream(new DataOutputStream(new FileOutputStream(getOutDir().resolve("stdout_unzip_" + filename + ".txt").toFile()))) @@ -467,7 +506,7 @@ private void downloadAndUnzip(Path exomiserDataPath, String filename) throws Too + filename + ".txt").toFile()))) .run(); } catch (FileNotFoundException e) { - throw new ToolException("Error unzipping Exomiser data: " + filename, e); + throw new ToolException("Error unzipping Exomiser " + exomiserVersion + " data: " + filename, e); } // Free disk space @@ -483,4 +522,13 @@ public ExomiserWrapperAnalysisExecutor setSampleId(String sampleId) { this.sampleId = sampleId; return this; } + + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperAnalysisExecutor setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index 387b439571b..7ac1012241f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -12,6 +12,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.clinical.ClinicalAnalysisUtilsTest; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractClinicalManagerTest; import org.opencb.opencga.core.exceptions.ToolException; @@ -79,11 +80,14 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE ClinicalAnalysis clinicalAnalysis = caResult.getResults().get(0); assertEquals(0, clinicalAnalysis.getSecondaryInterpretations().size()); - ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); + System.out.println("opencga.getOpencgaHome() = " + opencga.getOpencgaHome().toAbsolutePath()); + System.out.println("outDir = " + outDir); + ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID2); + .setClinicalAnalysisId(clinicalTest.CA_ID2) + .setExomiserVersion("13.1.0"); ExecutionResult result = exomiser.start(); @@ -109,7 +113,8 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID3); + .setClinicalAnalysisId(clinicalTest.CA_ID3) + .setExomiserVersion("13.1.0"); ExecutionResult result = exomiser.start(); diff --git a/opencga-analysis/src/test/resources/exomiser/application.properties b/opencga-analysis/src/test/resources/exomiser/application.properties index e722c2fd89b..ee367632b7f 100644 --- a/opencga-analysis/src/test/resources/exomiser/application.properties +++ b/opencga-analysis/src/test/resources/exomiser/application.properties @@ -37,9 +37,9 @@ exomiser.data-directory=/data #remm.version=0.3.1.post1 #cadd.version=1.4 #exomiser.hg19.data-version=1811 -exomiser.hg38.data-version=2109 +exomiser.hg38.data-version=put_here_hg38_data_version #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz -exomiser.phenotype.data-version=2109 +exomiser.phenotype.data-version=put_here_phenotype_data_version logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/application.properties b/opencga-app/app/analysis/exomiser/application.properties index e722c2fd89b..ee367632b7f 100644 --- a/opencga-app/app/analysis/exomiser/application.properties +++ b/opencga-app/app/analysis/exomiser/application.properties @@ -37,9 +37,9 @@ exomiser.data-directory=/data #remm.version=0.3.1.post1 #cadd.version=1.4 #exomiser.hg19.data-version=1811 -exomiser.hg38.data-version=2109 +exomiser.hg38.data-version=put_here_hg38_data_version #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz -exomiser.phenotype.data-version=2109 +exomiser.phenotype.data-version=put_here_phenotype_data_version logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index e502850d9ee..e052bd94638 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -1017,6 +1017,7 @@ private void exomiser() throws Exception { ObjectMap params = new ExomiserWrapperParams( cliOptions.sample, + cliOptions.exomiserVersion, cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index bad1beff087..dbb8378535c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -1844,6 +1844,9 @@ public class ExomiserAnalysisCommandOptions { @Parameter(names = {"--sample"}, description = "Sample ID.", required = true) public String sample; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.") + public String exomiserVersion = ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION; + @Parameter(names = {"-o", "--outdir"}, description = "Output directory.") public String outdir; } diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 7b84c12702c..27db4eaeee9 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -19,6 +19,22 @@ analysis: packages: # List of packages where to find analysis tools - "org.opencb.opencga" scratchDir: "/tmp/" # Scratch folder for the analysis. + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + exomiser-13.1.0: + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + exomiser-14.0.0: + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 6eb2beecb49..53864a0037d 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -17,13 +17,19 @@ package org.opencb.opencga.core.config; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class Analysis { private List packages; private String scratchDir; + private String resourceUrl; + + private String opencgaExtTools; + private Map tools; private Execution execution; @@ -31,6 +37,7 @@ public class Analysis { public Analysis() { packages = new ArrayList<>(); + tools = new HashMap<>(); execution = new Execution(); frameworks = new ArrayList<>(); } @@ -53,6 +60,33 @@ public Analysis setScratchDir(String scratchDir) { return this; } + public String getResourceUrl() { + return resourceUrl; + } + + public Analysis setResourceUrl(String resourceUrl) { + this.resourceUrl = resourceUrl; + return this; + } + + public String getOpencgaExtTools() { + return opencgaExtTools; + } + + public Analysis setOpencgaExtTools(String opencgaExtTools) { + this.opencgaExtTools = opencgaExtTools; + return this; + } + + public Map getTools() { + return tools; + } + + public Analysis setTools(Map tools) { + this.tools = tools; + return this; + } + public Execution getExecution() { return execution; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java new file mode 100644 index 00000000000..11800116985 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java @@ -0,0 +1,76 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.config; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Tool { + + private String dockerId; + private String params; + private Map resources; + + public Tool() { + resources = new HashMap<>(); + } + + public Tool(String dockerId, String params, Map resources) { + this.dockerId = dockerId; + this.params = params; + this.resources = resources; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Tool{"); + sb.append("dockerId='").append(dockerId).append('\''); + sb.append(", params='").append(params).append('\''); + sb.append(", resources=").append(resources); + sb.append('}'); + return sb.toString(); + } + + public String getDockerId() { + return dockerId; + } + + public Tool setDockerId(String dockerId) { + this.dockerId = dockerId; + return this; + } + + public String getParams() { + return params; + } + + public Tool setParams(String params) { + this.params = params; + return this; + } + + public Map getResources() { + return resources; + } + + public Tool setResources(Map resources) { + this.resources = resources; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java index 0ebc7f92d22..ac3bf057d08 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java @@ -8,19 +8,21 @@ public class ExomiserInterpretationAnalysisParams extends ToolParams { public static final String DESCRIPTION = "Exomizer interpretation analysis params"; private String clinicalAnalysis; + private String exomiserVersion; public ExomiserInterpretationAnalysisParams() { } - public ExomiserInterpretationAnalysisParams(String clinicalAnalysis) { + public ExomiserInterpretationAnalysisParams(String clinicalAnalysis, String exomiserVersion) { this.clinicalAnalysis = clinicalAnalysis; - + this.exomiserVersion = exomiserVersion; } @Override public String toString() { - final StringBuilder sb = new StringBuilder("TieringInterpretationAnalysisParams{"); + final StringBuilder sb = new StringBuilder("ExomiserInterpretationAnalysisParams{"); sb.append("clinicalAnalysis='").append(clinicalAnalysis).append('\''); + sb.append(", exomiserVersion='").append(exomiserVersion).append('\''); sb.append('}'); return sb.toString(); } @@ -34,5 +36,12 @@ public ExomiserInterpretationAnalysisParams setClinicalAnalysis(String clinicalA return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + public ExomiserInterpretationAnalysisParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java index 1fa40dfbaf7..3a2ff5722b2 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java @@ -8,13 +8,15 @@ public class ExomiserWrapperParams extends ToolParams { public static final String DESCRIPTION = "Exomiser parameters"; private String sample; + private String exomiserVersion; private String outdir; public ExomiserWrapperParams() { } - public ExomiserWrapperParams(String sample, String outdir) { + public ExomiserWrapperParams(String sample, String exomiserVersion, String outdir) { this.sample = sample; + this.exomiserVersion = exomiserVersion; this.outdir = outdir; } @@ -22,6 +24,7 @@ public ExomiserWrapperParams(String sample, String outdir) { public String toString() { final StringBuilder sb = new StringBuilder("ExomiserWrapperParams{"); sb.append("sample='").append(sample).append('\''); + sb.append("exomiserVersion='").append(exomiserVersion).append('\''); sb.append(", outdir='").append(outdir).append('\''); sb.append('}'); return sb.toString(); @@ -36,6 +39,15 @@ public ExomiserWrapperParams setSample(String sample) { return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } + public String getOutdir() { return outdir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java index 15cf7a51215..0b16875bebb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java @@ -17,9 +17,10 @@ package org.opencb.opencga.core.tools; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.core.tools.annotations.ToolExecutor; +import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import java.nio.file.Path; @@ -31,6 +32,7 @@ public abstract class OpenCgaToolExecutor { private ObjectMap executorParams; private Path outDir; private ExecutionResultManager arm; + private Configuration configuration; protected OpenCgaToolExecutor() { } @@ -51,10 +53,11 @@ public final ToolExecutor.Source getSource() { return this.getClass().getAnnotation(ToolExecutor.class).source(); } - public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir) { + public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir, Configuration configuration) { this.arm = arm; this.executorParams = executorParams; this.outDir = outDir; + this.configuration = configuration; } public final void execute() throws ToolException { @@ -77,6 +80,10 @@ public final Path getOutDir() { return outDir; } + public final Configuration getConfiguration() { + return configuration; + } + protected final String getToken() { return getExecutorParams().getString("token"); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java index 2af4d769513..980a0a8d494 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java @@ -33,7 +33,7 @@ public abstract class GwasAnalysisExecutor extends OpenCgaToolExecutor { private String cohort1; private String cohort2; private Path outputFile; - private GwasConfiguration configuration; + private GwasConfiguration gwasConfiguration; public GwasAnalysisExecutor() { } @@ -49,7 +49,7 @@ protected List getHeaderColumns() { columns.add("gene"); columns.add("biotype"); columns.add("consequence-types"); - if (configuration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { + if (gwasConfiguration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { columns.add("chi-square"); } columns.add("p-value"); @@ -70,7 +70,7 @@ public String toString() { sb.append(", phenotype2='").append(phenotype2).append('\''); sb.append(", cohort1='").append(cohort1).append('\''); sb.append(", cohort2='").append(cohort2).append('\''); - sb.append(", configuration=").append(configuration); + sb.append(", configuration=").append(gwasConfiguration); sb.append(", executorParams=").append(getExecutorParams()); sb.append(", outDir=").append(getOutDir()); sb.append('}'); @@ -149,12 +149,12 @@ public GwasAnalysisExecutor setOutputFile(Path outputFile) { return this; } - public GwasConfiguration getConfiguration() { - return configuration; + public GwasConfiguration getGwasConfiguration() { + return gwasConfiguration; } - public GwasAnalysisExecutor setConfiguration(GwasConfiguration configuration) { - this.configuration = configuration; + public GwasAnalysisExecutor setGwasConfiguration(GwasConfiguration gwasConfiguration) { + this.gwasConfiguration = gwasConfiguration; return this; } } diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index ac4acf283d8..82a80023f9f 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -107,9 +107,27 @@ healthCheck: analysis: - packages: # List of packages where to find analysis tools + # List of packages where to find analysis tools + packages: - "org.opencb.opencga" - scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" # Scratch folder for the analysis. + # Scratch folder for the analysis. + scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + exomiser-13.1.0: + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + exomiser-14.0.0: + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG38: "exomiser-14.0.0/2402_hg38.zip" + PHENOTYPE: "exomiser-14.0.0/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java index a62b08e0413..2a98f6412dc 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java @@ -1270,7 +1270,7 @@ public Response circos( ObjectMap executorParams = new ObjectMap(); executorParams.put("opencgaHome", opencgaHome); executorParams.put("token", token); - executor.setUp(null, executorParams, outDir.toPath()); + executor.setUp(null, executorParams, outDir.toPath(), null); // Run Circos executor StopWatch watch = StopWatch.createStarted(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java index ce29e3f18fe..102638aab8c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java @@ -25,7 +25,7 @@ public void run() throws ToolException { List samples1 = getSampleList1(); List samples2 = getSampleList2(); - if (getConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { + if (getGwasConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { addWarning("Unable to calculate chi-square test."); } From 4867904689dab5886b5a762dbea4d2e7e591942e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 23 May 2024 10:36:10 +0200 Subject: [PATCH 014/128] app: add Exomiser version as paremeter for the Exomiser interpretation analysis, #TASK-6297, #TASK-6255 --- .../app/cli/internal/executors/ClinicalCommandExecutor.java | 3 ++- .../app/cli/internal/options/ClinicalCommandOptions.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java index 0a32ddece0a..d58fca25609 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java @@ -339,7 +339,8 @@ private void exomiserInterpretation() throws Exception { ExomiserInterpretationAnalysis exomiserInterpretationAnalysis = new ExomiserInterpretationAnalysis(); exomiserInterpretationAnalysis.setUp(opencgaHome.toString(), new ObjectMap(), outDir, token); exomiserInterpretationAnalysis.setStudyId(cliOptions.study) - .setClinicalAnalysisId(cliOptions.clinicalAnalysis); + .setClinicalAnalysisId(cliOptions.clinicalAnalysis) + .setExomiserVersion(cliOptions.exomiserVersion); // exomiserInterpretationAnalysis.setPrimary(cliOptions.primary); exomiserInterpretationAnalysis.start(); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java index 853d280736d..8758f9532e7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java @@ -13,6 +13,7 @@ import org.opencb.opencga.analysis.clinical.tiering.CancerTieringInterpretationAnalysis; import org.opencb.opencga.analysis.clinical.tiering.TieringInterpretationAnalysis; import org.opencb.opencga.analysis.clinical.zetta.ZettaInterpretationAnalysis; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.app.cli.GeneralCliOptions; import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser; import org.opencb.opencga.core.api.ParamConstants; @@ -338,6 +339,9 @@ public class ExomiserInterpretationCommandOptions extends GeneralCliOptions.Stud @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = "Clinical analysis", required = true, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.") + public String exomiserVersion = ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION; + @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", arity = 1) public String outdir; } From ad7bd6accf39a95f026803466ddafda831e84d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 23 May 2024 11:12:43 +0200 Subject: [PATCH 015/128] app: add parameter descriptions, #TASK-6297, #TASK-6255 --- .../exomiser/ExomiserInterpretationAnalysis.java | 8 ++++---- .../wrappers/exomiser/ExomiserWrapperAnalysis.java | 3 +-- .../app/cli/internal/options/ClinicalCommandOptions.java | 8 +++++--- .../app/cli/internal/options/VariantCommandOptions.java | 7 ++++--- .../clinical/ExomiserInterpretationAnalysisParams.java | 8 +++++++- .../core/models/clinical/ExomiserWrapperParams.java | 9 +++++++++ 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 92ffe157b7b..b6dcd30fb26 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -36,6 +36,7 @@ import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; @@ -98,8 +99,7 @@ protected void check() throws Exception { try { clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(), token); - } catch ( - CatalogException e) { + } catch (CatalogException e) { throw new ToolException(e); } if (clinicalAnalysisQueryResult.getNumResults() != 1) { @@ -121,8 +121,8 @@ protected void check() throws Exception { // Check exomiser version if (StringUtils.isEmpty(exomiserVersion)) { // Missing exomiser version use the default one - logger.warn("Missing exomiser version, using the default {}", ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION); - exomiserVersion = ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION; + logger.warn("Missing exomiser version, using the default {}", FieldConstants.EXOMISER_DEFAULT_VERSION); + exomiserVersion = FieldConstants.EXOMISER_DEFAULT_VERSION; } // Update executor params with OpenCGA home and session ID diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index b1d7ad8a92c..03fb92099e2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -33,8 +33,6 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy { public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants" + " from whole-exome or whole-genome sequencing data."; - public final static String DEFAULT_EXOMISER_VERSION = "14.0.0"; - // It must match the tool prefix in the tool keys for exomiser in the configuration file public final static String EXOMISER_PREFIX = "exomiser-"; @@ -51,6 +49,7 @@ protected void check() throws Exception { if (StringUtils.isEmpty(getStudy())) { throw new ToolException("Missing study"); } + } @Override diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java index 8758f9532e7..c1074f62ee9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java @@ -24,6 +24,7 @@ import static org.opencb.opencga.analysis.clinical.InterpretationAnalysis.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.*; +import static org.opencb.opencga.core.api.FieldConstants.*; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @Parameters(commandNames = {"clinical"}, commandDescription = "Clinical analysis commands") @@ -336,11 +337,12 @@ public class ExomiserInterpretationCommandOptions extends GeneralCliOptions.Stud @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = "Clinical analysis", required = true, arity = 1) + @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, + required = true, arity = 1) public String clinicalAnalysis; - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.") - public String exomiserVersion = ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION; + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion = EXOMISER_DEFAULT_VERSION; @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", arity = 1) public String outdir; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index dbb8378535c..4005d4a9df5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -69,6 +69,7 @@ import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSampleQueryCommandOptions.SAMPLE_QUERY_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexCommandOptions.SECONDARY_INDEX_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexDeleteCommandOptions.SECONDARY_INDEX_DELETE_COMMAND; +import static org.opencb.opencga.core.api.FieldConstants.*; import static org.opencb.opencga.core.api.ParamConstants.*; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND_DESCRIPTION; @@ -1841,11 +1842,11 @@ public class ExomiserAnalysisCommandOptions { @Parameter(names = {"--study"}, description = "Study where all the samples belong to.") public String study; - @Parameter(names = {"--sample"}, description = "Sample ID.", required = true) + @Parameter(names = {"--sample"}, description = EXOMISER_SAMPLE_DESCRIPTION, required = true) public String sample; - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.") - public String exomiserVersion = ExomiserWrapperAnalysis.DEFAULT_EXOMISER_VERSION; + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion = EXOMISER_DEFAULT_VERSION; @Parameter(names = {"-o", "--outdir"}, description = "Output directory.") public String outdir; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java index ac3bf057d08..61d33a06ebd 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java @@ -1,13 +1,19 @@ package org.opencb.opencga.core.models.clinical; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_DEFAULT_VERSION; public class ExomiserInterpretationAnalysisParams extends ToolParams { - public static final String DESCRIPTION = "Exomizer interpretation analysis params"; + public static final String DESCRIPTION = "Exomiser interpretation analysis params"; + @DataField(id = "clinicalAnalysis", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, required = true) private String clinicalAnalysis; + + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION, defaultValue = EXOMISER_DEFAULT_VERSION) private String exomiserVersion; public ExomiserInterpretationAnalysisParams() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java index 3a2ff5722b2..15a3954935c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java @@ -1,14 +1,23 @@ package org.opencb.opencga.core.models.clinical; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; import java.util.Map; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_DEFAULT_VERSION; + public class ExomiserWrapperParams extends ToolParams { public static final String DESCRIPTION = "Exomiser parameters"; + @DataField(id = "sample", description = FieldConstants.EXOMISER_SAMPLE_DESCRIPTION, required = true) private String sample; + + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION, defaultValue = EXOMISER_DEFAULT_VERSION) private String exomiserVersion; + + @DataField(id = "outdir", description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) private String outdir; public ExomiserWrapperParams() { From 28e32c53769618ad36063833a09ae5ca5e1c0234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 23 May 2024 11:13:14 +0200 Subject: [PATCH 016/128] client: generate clients, #TASK-6297, #TASK-6255 --- .../app/cli/main/OpenCgaCompleter.java | 2 +- .../app/cli/main/OpencgaCliOptionsParser.java | 2 +- .../AnalysisClinicalCommandExecutor.java | 1 + .../AnalysisVariantCommandExecutor.java | 1 + .../OrganizationsCommandExecutor.java | 1 + .../main/executors/UsersCommandExecutor.java | 1 - .../AnalysisClinicalCommandOptions.java | 5 +++- .../AnalysisVariantCommandOptions.java | 7 ++++-- .../cli/main/options/UsersCommandOptions.java | 3 --- opencga-client/src/main/R/R/Admin-methods.R | 2 +- .../src/main/R/R/Alignment-methods.R | 2 +- opencga-client/src/main/R/R/AllGenerics.R | 24 +++++++++---------- .../src/main/R/R/Clinical-methods.R | 6 ++--- opencga-client/src/main/R/R/Cohort-methods.R | 4 ++-- opencga-client/src/main/R/R/Family-methods.R | 4 ++-- opencga-client/src/main/R/R/File-methods.R | 4 ++-- opencga-client/src/main/R/R/GA4GH-methods.R | 4 ++-- .../src/main/R/R/Individual-methods.R | 4 ++-- opencga-client/src/main/R/R/Job-methods.R | 4 ++-- opencga-client/src/main/R/R/Meta-methods.R | 2 +- .../src/main/R/R/Operation-methods.R | 2 +- .../src/main/R/R/Organization-methods.R | 2 +- opencga-client/src/main/R/R/Panel-methods.R | 4 ++-- opencga-client/src/main/R/R/Project-methods.R | 4 ++-- opencga-client/src/main/R/R/Sample-methods.R | 4 ++-- opencga-client/src/main/R/R/Study-methods.R | 4 ++-- opencga-client/src/main/R/R/User-methods.R | 4 ++-- opencga-client/src/main/R/R/Variant-methods.R | 2 +- .../client/rest/clients/AdminClient.java | 4 ++-- .../client/rest/clients/AlignmentClient.java | 4 ++-- .../rest/clients/ClinicalAnalysisClient.java | 6 ++--- .../client/rest/clients/CohortClient.java | 4 ++-- .../rest/clients/DiseasePanelClient.java | 4 ++-- .../client/rest/clients/FamilyClient.java | 4 ++-- .../client/rest/clients/FileClient.java | 4 ++-- .../client/rest/clients/GA4GHClient.java | 4 ++-- .../client/rest/clients/IndividualClient.java | 4 ++-- .../client/rest/clients/JobClient.java | 4 ++-- .../client/rest/clients/MetaClient.java | 4 ++-- .../rest/clients/OrganizationClient.java | 4 ++-- .../client/rest/clients/ProjectClient.java | 4 ++-- .../client/rest/clients/SampleClient.java | 4 ++-- .../client/rest/clients/StudyClient.java | 4 ++-- .../client/rest/clients/UserClient.java | 4 ++-- .../client/rest/clients/VariantClient.java | 4 ++-- .../rest/clients/VariantOperationClient.java | 4 ++-- opencga-client/src/main/javascript/Admin.js | 2 +- .../src/main/javascript/Alignment.js | 2 +- .../src/main/javascript/ClinicalAnalysis.js | 4 ++-- opencga-client/src/main/javascript/Cohort.js | 2 +- .../src/main/javascript/DiseasePanel.js | 2 +- opencga-client/src/main/javascript/Family.js | 2 +- opencga-client/src/main/javascript/File.js | 2 +- opencga-client/src/main/javascript/GA4GH.js | 2 +- .../src/main/javascript/Individual.js | 2 +- opencga-client/src/main/javascript/Job.js | 2 +- opencga-client/src/main/javascript/Meta.js | 2 +- .../src/main/javascript/Organization.js | 2 +- opencga-client/src/main/javascript/Project.js | 2 +- opencga-client/src/main/javascript/Sample.js | 2 +- opencga-client/src/main/javascript/Study.js | 2 +- opencga-client/src/main/javascript/User.js | 2 +- opencga-client/src/main/javascript/Variant.js | 2 +- .../src/main/javascript/VariantOperation.js | 2 +- .../pyopencga/rest_clients/admin_client.py | 4 ++-- .../rest_clients/alignment_client.py | 4 ++-- .../rest_clients/clinical_analysis_client.py | 6 ++--- .../pyopencga/rest_clients/cohort_client.py | 4 ++-- .../rest_clients/disease_panel_client.py | 4 ++-- .../pyopencga/rest_clients/family_client.py | 4 ++-- .../pyopencga/rest_clients/file_client.py | 4 ++-- .../pyopencga/rest_clients/ga4gh_client.py | 4 ++-- .../rest_clients/individual_client.py | 4 ++-- .../pyopencga/rest_clients/job_client.py | 4 ++-- .../pyopencga/rest_clients/meta_client.py | 4 ++-- .../rest_clients/organization_client.py | 4 ++-- .../pyopencga/rest_clients/project_client.py | 4 ++-- .../pyopencga/rest_clients/sample_client.py | 4 ++-- .../pyopencga/rest_clients/study_client.py | 4 ++-- .../pyopencga/rest_clients/user_client.py | 4 ++-- .../pyopencga/rest_clients/variant_client.py | 4 ++-- .../rest_clients/variant_operation_client.py | 4 ++-- .../opencga/core/api/FieldConstants.java | 5 ++++ 83 files changed, 155 insertions(+), 145 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index 623e23db9fb..e83115c8d1c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2024-04-10 OpenCB +* Copyright 2015-2024-05-23 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index 5b96a113110..a00af1b0ca1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2024-04-10 OpenCB +* Copyright 2015-2024-05-23 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index e96ddce01c2..22b4fcf7fa0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -573,6 +573,7 @@ private RestResponse runInterpreterExomiser() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion",commandOptions.exomiserVersion, true); exomiserInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 33dfbc54a41..62b81b2df89 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -427,6 +427,7 @@ private RestResponse runExomiser() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion",commandOptions.exomiserVersion, true); putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); exomiserWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index e18895e85b2..991f44fd9d8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -22,6 +22,7 @@ import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.organizations.TokenConfiguration; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java index f8bd7213886..794d32fd1f6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java @@ -284,7 +284,6 @@ private RestResponse update() throws Exception { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); userUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index fcfe1679c19..11a5501bd5f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -649,9 +649,12 @@ public class RunInterpreterExomiserCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = false, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", required = false, arity = 1) + public String exomiserVersion = "14.0.0"; + } @Parameters(commandNames = {"interpreter-team-run"}, commandDescription ="Run TEAM interpretation analysis") diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 82b9edc17a4..f3d490e4fbf 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -410,10 +410,13 @@ public class RunExomiserCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) + @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; - @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", required = false, arity = 1) + public String exomiserVersion = "14.0.0"; + + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java index e17ed6588e4..6cdb25f0cda 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java @@ -251,9 +251,6 @@ public class UpdateCommandOptions { @Parameter(names = {"--email"}, description = "The body web service email parameter", required = false, arity = 1) public String email; - @Parameter(names = {"--organization"}, description = "The body web service organization parameter", required = false, arity = 1) - public String organization; - @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index 8f5d4877e7f..1aef5e23bb1 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index e9b2195c871..036b855a053 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index fe127eafb19..2a76b8da965 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -5,52 +5,52 @@ setGeneric("organizationClient", function(OpencgaR, organization, id, endpointNa # ############################################################################## ## UserClient -setGeneric("userClient", function(OpencgaR, users, user, filterId, endpointName, params=NULL, ...) +setGeneric("userClient", function(OpencgaR, users, filterId, user, endpointName, params=NULL, ...) standardGeneric("userClient")) # ############################################################################## ## ProjectClient -setGeneric("projectClient", function(OpencgaR, project, projects, endpointName, params=NULL, ...) +setGeneric("projectClient", function(OpencgaR, projects, project, endpointName, params=NULL, ...) standardGeneric("projectClient")) # ############################################################################## ## StudyClient -setGeneric("studyClient", function(OpencgaR, study, studies, variableSet, group, templateId, members, id, endpointName, params=NULL, ...) +setGeneric("studyClient", function(OpencgaR, members, templateId, group, studies, study, variableSet, id, endpointName, params=NULL, ...) standardGeneric("studyClient")) # ############################################################################## ## FileClient -setGeneric("fileClient", function(OpencgaR, file, folder, annotationSet, members, files, endpointName, params=NULL, ...) +setGeneric("fileClient", function(OpencgaR, members, files, file, annotationSet, folder, endpointName, params=NULL, ...) standardGeneric("fileClient")) # ############################################################################## ## JobClient -setGeneric("jobClient", function(OpencgaR, jobs, members, job, endpointName, params=NULL, ...) +setGeneric("jobClient", function(OpencgaR, members, jobs, job, endpointName, params=NULL, ...) standardGeneric("jobClient")) # ############################################################################## ## SampleClient -setGeneric("sampleClient", function(OpencgaR, samples, annotationSet, members, sample, endpointName, params=NULL, ...) +setGeneric("sampleClient", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) standardGeneric("sampleClient")) # ############################################################################## ## IndividualClient -setGeneric("individualClient", function(OpencgaR, individual, annotationSet, members, individuals, endpointName, params=NULL, ...) +setGeneric("individualClient", function(OpencgaR, members, annotationSet, individual, individuals, endpointName, params=NULL, ...) standardGeneric("individualClient")) # ############################################################################## ## FamilyClient -setGeneric("familyClient", function(OpencgaR, families, members, family, annotationSet, endpointName, params=NULL, ...) +setGeneric("familyClient", function(OpencgaR, members, family, annotationSet, families, endpointName, params=NULL, ...) standardGeneric("familyClient")) # ############################################################################## ## CohortClient -setGeneric("cohortClient", function(OpencgaR, cohort, members, cohorts, annotationSet, endpointName, params=NULL, ...) +setGeneric("cohortClient", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) standardGeneric("cohortClient")) # ############################################################################## ## PanelClient -setGeneric("panelClient", function(OpencgaR, panels, members, endpointName, params=NULL, ...) +setGeneric("panelClient", function(OpencgaR, members, panels, endpointName, params=NULL, ...) standardGeneric("panelClient")) # ############################################################################## @@ -65,7 +65,7 @@ setGeneric("variantClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## ClinicalClient -setGeneric("clinicalClient", function(OpencgaR, clinicalAnalysis, clinicalAnalyses, interpretations, interpretation, annotationSet, members, endpointName, params=NULL, ...) +setGeneric("clinicalClient", function(OpencgaR, members, interpretations, annotationSet, interpretation, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) standardGeneric("clinicalClient")) # ############################################################################## @@ -80,7 +80,7 @@ setGeneric("metaClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## GA4GHClient -setGeneric("ga4ghClient", function(OpencgaR, study, file, endpointName, params=NULL, ...) +setGeneric("ga4ghClient", function(OpencgaR, file, study, endpointName, params=NULL, ...) standardGeneric("ga4ghClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index 32983a2b6bc..357d379bd5a 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -62,7 +62,7 @@ #' [*]: Required parameter #' @export -setMethod("clinicalClient", "OpencgaR", function(OpencgaR, clinicalAnalysis, clinicalAnalyses, interpretations, interpretation, annotationSet, members, endpointName, params=NULL, ...) { +setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, interpretations, annotationSet, interpretation, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/analysis/clinical/acl/{members}/update: @@ -217,7 +217,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, clinicalAnalysis, cli #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param data Exomizer interpretation analysis params. + #' @param data Exomiser interpretation analysis params. runInterpreterExomiser=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/exomiser", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/Cohort-methods.R b/opencga-client/src/main/R/R/Cohort-methods.R index 6f37317a6fb..1f3a876aaa9 100644 --- a/opencga-client/src/main/R/R/Cohort-methods.R +++ b/opencga-client/src/main/R/R/Cohort-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("cohortClient", "OpencgaR", function(OpencgaR, cohort, members, cohorts, annotationSet, endpointName, params=NULL, ...) { +setMethod("cohortClient", "OpencgaR", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/cohorts/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Family-methods.R b/opencga-client/src/main/R/R/Family-methods.R index 18950845d09..f38a6e85189 100644 --- a/opencga-client/src/main/R/R/Family-methods.R +++ b/opencga-client/src/main/R/R/Family-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +37,7 @@ #' [*]: Required parameter #' @export -setMethod("familyClient", "OpencgaR", function(OpencgaR, families, members, family, annotationSet, endpointName, params=NULL, ...) { +setMethod("familyClient", "OpencgaR", function(OpencgaR, members, family, annotationSet, families, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/families/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index 2a108bf3c3c..f1bd5542eab 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +54,7 @@ #' [*]: Required parameter #' @export -setMethod("fileClient", "OpencgaR", function(OpencgaR, file, folder, annotationSet, members, files, endpointName, params=NULL, ...) { +setMethod("fileClient", "OpencgaR", function(OpencgaR, members, files, file, annotationSet, folder, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/files/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/GA4GH-methods.R b/opencga-client/src/main/R/R/GA4GH-methods.R index 6962ac02c5d..35dc15c1113 100644 --- a/opencga-client/src/main/R/R/GA4GH-methods.R +++ b/opencga-client/src/main/R/R/GA4GH-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -31,7 +31,7 @@ #' [*]: Required parameter #' @export -setMethod("ga4ghClient", "OpencgaR", function(OpencgaR, study, file, endpointName, params=NULL, ...) { +setMethod("ga4ghClient", "OpencgaR", function(OpencgaR, file, study, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/ga4gh/reads/search: diff --git a/opencga-client/src/main/R/R/Individual-methods.R b/opencga-client/src/main/R/R/Individual-methods.R index 6db8609a2e0..065c23c60a1 100644 --- a/opencga-client/src/main/R/R/Individual-methods.R +++ b/opencga-client/src/main/R/R/Individual-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("individualClient", "OpencgaR", function(OpencgaR, individual, annotationSet, members, individuals, endpointName, params=NULL, ...) { +setMethod("individualClient", "OpencgaR", function(OpencgaR, members, annotationSet, individual, individuals, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/individuals/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index a08f7f1cff9..8ea4cb9e9d4 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpointName, params=NULL, ...) { +setMethod("jobClient", "OpencgaR", function(OpencgaR, members, jobs, job, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/jobs/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Meta-methods.R b/opencga-client/src/main/R/R/Meta-methods.R index e07d2af2b26..7ce863ddb61 100644 --- a/opencga-client/src/main/R/R/Meta-methods.R +++ b/opencga-client/src/main/R/R/Meta-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 466172bd59c..2793051d9de 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Organization-methods.R b/opencga-client/src/main/R/R/Organization-methods.R index 178042aea75..6c42ae9b06b 100644 --- a/opencga-client/src/main/R/R/Organization-methods.R +++ b/opencga-client/src/main/R/R/Organization-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index a5302a2260d..9e4720691d7 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -36,7 +36,7 @@ #' [*]: Required parameter #' @export -setMethod("panelClient", "OpencgaR", function(OpencgaR, panels, members, endpointName, params=NULL, ...) { +setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/panels/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Project-methods.R b/opencga-client/src/main/R/R/Project-methods.R index 2b1e2a13c3e..aca59165834 100644 --- a/opencga-client/src/main/R/R/Project-methods.R +++ b/opencga-client/src/main/R/R/Project-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -33,7 +33,7 @@ #' [*]: Required parameter #' @export -setMethod("projectClient", "OpencgaR", function(OpencgaR, project, projects, endpointName, params=NULL, ...) { +setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/projects/create: diff --git a/opencga-client/src/main/R/R/Sample-methods.R b/opencga-client/src/main/R/R/Sample-methods.R index 14252d367ec..65871dcb8bb 100644 --- a/opencga-client/src/main/R/R/Sample-methods.R +++ b/opencga-client/src/main/R/R/Sample-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("sampleClient", "OpencgaR", function(OpencgaR, samples, annotationSet, members, sample, endpointName, params=NULL, ...) { +setMethod("sampleClient", "OpencgaR", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/samples/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index 55a5cc88b4c..22276f49f57 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -49,7 +49,7 @@ #' [*]: Required parameter #' @export -setMethod("studyClient", "OpencgaR", function(OpencgaR, study, studies, variableSet, group, templateId, members, id, endpointName, params=NULL, ...) { +setMethod("studyClient", "OpencgaR", function(OpencgaR, members, templateId, group, studies, study, variableSet, id, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/studies/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/User-methods.R b/opencga-client/src/main/R/R/User-methods.R index f074c546a88..76772b8a2d4 100644 --- a/opencga-client/src/main/R/R/User-methods.R +++ b/opencga-client/src/main/R/R/User-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("userClient", "OpencgaR", function(OpencgaR, users, user, filterId, endpointName, params=NULL, ...) { +setMethod("userClient", "OpencgaR", function(OpencgaR, users, filterId, user, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/users/anonymous: diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index 2b18e5f5063..70c435fcc6c 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-10 +# Autogenerated on: 2024-05-23 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index 77e825b9edb..3b43605603f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ /** * This class contains methods for the Admin webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: admin */ public class AdminClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index 916156065b5..4d2c0d9fab4 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -40,7 +40,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -49,7 +49,7 @@ /** * This class contains methods for the Alignment webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: analysis/alignment */ public class AlignmentClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index 4ad23e20b19..4bfe0704ebb 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -55,7 +55,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -64,7 +64,7 @@ /** * This class contains methods for the ClinicalAnalysis webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: analysis/clinical */ public class ClinicalAnalysisClient extends AbstractParentClient { @@ -291,7 +291,7 @@ public RestResponse runInterpreterCancerTiering(CancerTieringInterpretation /** * Run exomiser interpretation analysis. - * @param data Exomizer interpretation analysis params. + * @param data Exomiser interpretation analysis params. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java index db3160eeb6c..0163f130151 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Cohort webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: cohorts */ public class CohortClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index 83d0ffa1175..0ca3b876bcc 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +44,7 @@ /** * This class contains methods for the DiseasePanel webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: panels */ public class DiseasePanelClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java index 8f8bd680ffc..4f53af041e6 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +44,7 @@ /** * This class contains methods for the Family webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: families */ public class FamilyClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index 02e3d914f6d..687e43e6ea5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -43,7 +43,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -52,7 +52,7 @@ /** * This class contains methods for the File webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: files */ public class FileClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java index b687c3d0e47..765d86384d2 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java @@ -27,7 +27,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -36,7 +36,7 @@ /** * This class contains methods for the GA4GH webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: ga4gh */ public class GA4GHClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java index 09e91a32ff7..939a53e5a86 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +44,7 @@ /** * This class contains methods for the Individual webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: individuals */ public class IndividualClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index f6d27510706..8840fd8c7c8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Job webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: jobs */ public class JobClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java index 3e77702534c..2fb1c5df290 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java @@ -28,7 +28,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +37,7 @@ /** * This class contains methods for the Meta webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: meta */ public class MetaClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java index 1415aeb5605..bbc2d910bbc 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java @@ -33,7 +33,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -42,7 +42,7 @@ /** * This class contains methods for the Organization webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: organizations */ public class OrganizationClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java index 4f241541143..570af443ec9 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java @@ -31,7 +31,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -40,7 +40,7 @@ /** * This class contains methods for the Project webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: projects */ public class ProjectClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java index 5feb046554a..117401694cc 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +44,7 @@ /** * This class contains methods for the Sample webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: samples */ public class SampleClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index a5187a4c7d8..424a5cbc509 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -47,7 +47,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -56,7 +56,7 @@ /** * This class contains methods for the Study webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: studies */ public class StudyClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java index 71b158bca58..468dffded80 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the User webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: users */ public class UserClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index bf91b0a72df..6dc2fdb9bd8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -62,7 +62,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -71,7 +71,7 @@ /** * This class contains methods for the Variant webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: analysis/variant */ public class VariantClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index b208b0332f5..52b0333e713 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -50,7 +50,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-10 +* Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -59,7 +59,7 @@ /** * This class contains methods for the VariantOperation webservices. - * Client version: 3.1.0-SNAPSHOT + * Client version: 3.2.0-SNAPSHOT * PATH: operation */ public class VariantOperationClient extends AbstractParentClient { diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index a17bf1209e7..49ab10c86c2 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index 3b0cf840b5a..005457da0ab 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index 3a2d4044b61..302b3084adc 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -214,7 +214,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { } /** Run exomiser interpretation analysis - * @param {Object} data - Exomizer interpretation analysis params. + * @param {Object} data - Exomiser interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/javascript/Cohort.js b/opencga-client/src/main/javascript/Cohort.js index e8c016fe834..e07f6c58dab 100644 --- a/opencga-client/src/main/javascript/Cohort.js +++ b/opencga-client/src/main/javascript/Cohort.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index f9401a6090f..1a04bd3c77f 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Family.js b/opencga-client/src/main/javascript/Family.js index 1568ca19ad2..e12be40b17d 100644 --- a/opencga-client/src/main/javascript/Family.js +++ b/opencga-client/src/main/javascript/Family.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index 3aaf33d45a4..ab7b241d600 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index 50c5f456909..eaff0099476 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Individual.js b/opencga-client/src/main/javascript/Individual.js index 81bc55cc81d..e1304c41a58 100644 --- a/opencga-client/src/main/javascript/Individual.js +++ b/opencga-client/src/main/javascript/Individual.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index a7faefab056..ad50f4b336b 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Meta.js b/opencga-client/src/main/javascript/Meta.js index 5f46d66682f..87fb7ba03ce 100644 --- a/opencga-client/src/main/javascript/Meta.js +++ b/opencga-client/src/main/javascript/Meta.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Organization.js b/opencga-client/src/main/javascript/Organization.js index 6e96cfe293a..2b94af1924f 100644 --- a/opencga-client/src/main/javascript/Organization.js +++ b/opencga-client/src/main/javascript/Organization.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index 8728cf80482..839907f1003 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Sample.js b/opencga-client/src/main/javascript/Sample.js index 5687de66b71..1af15198986 100644 --- a/opencga-client/src/main/javascript/Sample.js +++ b/opencga-client/src/main/javascript/Sample.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 1d8a7d97876..0145d803d6b 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index 1b72aa833c4..de654d8761e 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index 60baa0f5a0d..9bd0510f536 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index bfd6f99aa40..02f03335138 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-10 + * Autogenerated on: 2024-05-23 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index 1f344f41988..1b5c9fdd673 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Admin(_ParentRestClient): """ This class contains methods for the 'Admin' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/admin """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index 08ba05a5b76..abdd7e41f3c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Alignment(_ParentRestClient): """ This class contains methods for the 'Analysis - Alignment' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/analysis/alignment """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index c8cf1a2e23b..edd4c870c13 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class ClinicalAnalysis(_ParentRestClient): """ This class contains methods for the 'Analysis - Clinical' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/analysis/clinical """ @@ -275,7 +275,7 @@ def run_interpreter_exomiser(self, data=None, **options): Run exomiser interpretation analysis. PATH: /{apiVersion}/analysis/clinical/interpreter/exomiser/run - :param dict data: Exomizer interpretation analysis params. (REQUIRED) + :param dict data: Exomiser interpretation analysis params. (REQUIRED) :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py index 1d37bffadd2..a93c8378fdb 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Cohort(_ParentRestClient): """ This class contains methods for the 'Cohorts' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/cohorts """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index 61d984ad048..4da214c272c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class DiseasePanel(_ParentRestClient): """ This class contains methods for the 'Disease Panels' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/panels """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py index 9df67f56a94..935e447a4c7 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Family(_ParentRestClient): """ This class contains methods for the 'Families' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/families """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index 881832c2eef..a370bb911cc 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class File(_ParentRestClient): """ This class contains methods for the 'Files' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/files """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py index 4b66f11cf5f..c6d5a0bf458 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class GA4GH(_ParentRestClient): """ This class contains methods for the 'GA4GH' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/ga4gh """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py index 5bef563d519..32f7516aed3 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Individual(_ParentRestClient): """ This class contains methods for the 'Individuals' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/individuals """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 94b4b8d27e5..6def8e09fa7 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Job(_ParentRestClient): """ This class contains methods for the 'Jobs' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/jobs """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py index a89eb56e683..5090a04eb23 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Meta(_ParentRestClient): """ This class contains methods for the 'Meta' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/meta """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py index 2ba35452d48..7c71bbadfed 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Organization(_ParentRestClient): """ This class contains methods for the 'Organizations' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/organizations """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py index 46bc074062d..da71c68e814 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Project(_ParentRestClient): """ This class contains methods for the 'Projects' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/projects """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py index 74636b383fb..df13fd02245 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Sample(_ParentRestClient): """ This class contains methods for the 'Samples' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/samples """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 42433932e42..6b804bc9fd0 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Study(_ParentRestClient): """ This class contains methods for the 'Studies' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/studies """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py index d1ebc104bd8..f7c61eb29be 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class User(_ParentRestClient): """ This class contains methods for the 'Users' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/users """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index ecc28f50daa..548eae4ff0a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Variant(_ParentRestClient): """ This class contains methods for the 'Analysis - Variant' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/analysis/variant """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index 688badda837..a4d4d61a095 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-10 + Autogenerated on: 2024-05-23 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class VariantOperation(_ParentRestClient): """ This class contains methods for the 'Operations - Variant Storage' webservices - Client version: 3.1.0-SNAPSHOT + Client version: 3.2.0-SNAPSHOT PATH: /{apiVersion}/operation """ diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index c2d7cfe4961..a506527e38f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -523,4 +523,9 @@ public class FieldConstants { + AlignmentQcParams.FASTQC_METRICS_SKIP_VALUE; public static final String ALIGNMENT_QC_OVERWRITE_DESCRIPTION = "To overwrite the QC metrics already computed."; + // Exomiser + public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID."; + public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID."; + public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version."; + public static final String EXOMISER_DEFAULT_VERSION = "14.0.0"; } From bdd4f80671d69cdbb42afdb8d60f3af21a707648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 23 May 2024 10:38:04 +0100 Subject: [PATCH 017/128] storage: Fix tests. #TASK-5861, #TASK-5448 --- .../analysis/clinical/ClinicalAnalysisUtilsTest.java | 4 +++- .../opencga/catalog/db/mongodb/MongoBackupUtils.java | 8 ++++---- .../hadoop/variant/HadoopVariantStorageEngine.java | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java index 26bf31c9dfc..a669a9288a4 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java @@ -24,6 +24,7 @@ import org.opencb.biodata.models.variant.avro.SequenceOntologyTerm; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractClinicalManagerTest; @@ -65,11 +66,12 @@ public static AbstractClinicalManagerTest getClinicalTest(OpenCGATestExternalRes .append(VariantStorageOptions.ANNOTATE.key(), true) .append(VariantStorageOptions.STATS_CALCULATE.key(), false); - VariantStorageManager variantStorageManager = new VariantStorageManager(opencga.getCatalogManager(), opencga.getStorageEngineFactory()); + VariantStorageManager variantStorageManager = opencga.getVariantStorageManager(); Path outDir = Paths.get("target/test-data").resolve("junit_clinical_analysis_" + RandomStringUtils.randomAlphabetic(10)); Files.createDirectories(outDir); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, clinicalTest.studyFqn, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "family.vcf", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "exomiser.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "HG004.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java index 84806f9a959..252085c353b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java @@ -37,6 +37,7 @@ */ public class MongoBackupUtils { + private static final String TEMPORAL_FOLDER_HERE = "TEMPORAL_FOLDER_HERE"; private static Logger logger = LoggerFactory.getLogger(MongoBackupUtils.class); public static void dump(CatalogManager catalogManager, Path opencgaHome) throws CatalogDBException { @@ -135,14 +136,13 @@ public static void restore(CatalogManager catalogManager, Path opencgaHome, Path try (Stream stream = Files.list(restoreFolder)) { stream.forEach(file -> organizationIds.add(file.getFileName().toString())); } - System.out.println("organizationIds = " + organizationIds); if (organizationIds.isEmpty()) { throw new CatalogDBException("No organization found in the restore folder '" + restoreFolder + "'"); } restore(catalogManager, opencgaHome, restoreFolder, organizationIds); } - public static void restore(CatalogManager catalogManager, Path opencgaHome, Object source, Collection organizationIds) + private static void restore(CatalogManager catalogManager, Path opencgaHome, Object source, Collection organizationIds) throws Exception { logger.info("Restore opencga from source " + source + " for organizations " + organizationIds); StopWatch stopWatch = StopWatch.createStarted(); @@ -211,8 +211,8 @@ private static void restoreDatabase(CatalogManager catalogManager, Path opencgaH // Write actual temporal folder in database String uri = document.getString("uri"); - String temporalFolder = opencgaHome.getFileName().toString(); - String replacedUri = uri.replace("TEMPORAL_FOLDER_HERE", temporalFolder); + String uriPath = uri.substring(uri.indexOf(TEMPORAL_FOLDER_HERE) + TEMPORAL_FOLDER_HERE.length() + 1); + String replacedUri = opencgaHome.resolve(uriPath).toUri().toString(); document.put("uri", replacedUri); if (OrganizationMongoDBAdaptorFactory.FILE_COLLECTION.equals(collection)) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index 5c80958e7bf..6b32efdaedd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -1100,7 +1100,7 @@ public ObjectMap inferConfigurationParams(VariantSetupParams params) { int defaultVariantPreSplit = configuredOptions .getInt(VARIANT_TABLE_PRESPLIT_SIZE.key(), VARIANT_TABLE_PRESPLIT_SIZE.defaultValue()); float variantsFileToHBaseMultiplier = 1.3f; - Long averageFileSize = IOUtils.fromHumanReadableToByte(params.getAverageFileSize()); + Long averageFileSize = IOUtils.fromHumanReadableToByte(params.getAverageFileSize(), true); float variantsTableSize = params.getExpectedFiles() * averageFileSize * variantsFileToHBaseMultiplier; int variantPreSplit = (int) (variantsTableSize / expectedHBaseRegionSize); options.put(VARIANT_TABLE_PRESPLIT_SIZE.key(), Math.max(defaultVariantPreSplit, variantPreSplit)); From f5ea5a475fa773e5106814473267e10a87d11ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 23 May 2024 10:38:25 +0100 Subject: [PATCH 018/128] core: Add data field descriptions to VariantSetupPatams. #TASK-5861, #TASK-5448 --- .../models/variant/VariantSetupParams.java | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java index ed6aa1801a0..ecac975e112 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -1,21 +1,41 @@ package org.opencb.opencga.core.models.variant; +import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.tools.ToolParams; public class VariantSetupParams extends ToolParams { + @DataField(description = "Expected number of samples that will be loaded. Used to infer some parameters. " + + "This number is only used as a hint. " + + "If the real number of samples is different, if it grows beyond expectation, or if , the loader should be able to handle it.", + required = true) private Integer expectedSamples; + @DataField(description = "Expected number of files that will be loaded. Used to infer some parameters. " + + "This number is only used as a hint. " + + "If the real number of files is different, the loader should be able to handle it.", required = true) private Integer expectedFiles; + @DataField(description = "Main type of the files that will be loaded. If the dataset contains multiple types of files," + + " provide the one that matches most of the files.") private FileType fileType; + @DataField(description = "Average size of the files that will be loaded. This number is only used as a hint. " + + "If the real size of the files is different, the loader should be able to handle it. Accepts units. e.g. 435MB, 2GB, 86KB. " + + "If not provided, the value will be inferred from the file type.") private String averageFileSize; + @DataField(description = "Number of variants per sample. This number is only used as a hint. " + + "If the real number of variants per sample is different, the loader should be able to handle it. " + + "If not provided, the value will be inferred from the file type.") private Integer variantsPerSample; + @DataField(description = "Average number of samples per file. This number is only used as a hint. " + + "If the real number of samples per file is different, the loader should be able to handle it. " + + "If not provided, the value will be inferred from the expectedSamples and expectedFiles and dataDistribution.") private Float averageSamplesPerFile; + @DataField(description = "Data distribution of the files. This parameter is used to infer the number of samples per file.") private DataDistribution dataDistribution; public VariantSetupParams(VariantSetupParams params) { @@ -37,27 +57,39 @@ public enum DataDistribution { // - Platinum gVCF // - Cancer germline // - RD germline without family calling + @DataField(description = "Single sample VCF files. One file per sample. e.g. Platinum gVCF, Cancer germline, RD germline without family calling") SINGLE_SAMPLE_FILES, + // Multi samples VCF files. One file with multiple samples. // e.g. // - Corpasome // - RD germline with family calling + @DataField(description = "Multi samples VCF files. One file with multiple samples. e.g. Corpasome, RD germline with family calling") MULTI_SAMPLE_FILES, + // Multiple files per sample. Each file might have multiple samples. // e.g. // - Somatic study with multiple callers + @DataField(description = "Multiple files per sample. Each file might have multiple samples. e.g. Somatic study with multiple callers") MULTIPLE_FILE_PER_SAMPLE, + // Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. // e.g. // - 1000 genomes + @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. e.g. 1000 genomes") MULTI_SAMPLE_FILES_SPLIT_BY_CHROMOSOME, - // Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific region. + + // Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region. + @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region.") MULTI_SAMPLE_FILES_SPLIT_BY_REGION, } public enum FileType { + @DataField(description = "Whole genome VCF file.") GENOME_VCF, + @DataField(description = "Whole genome gVCF file.") GENOME_gVCF, + @DataField(description = "Exome VCF file.") EXOME } From d1efc049dbfa137be84e0121b9b7437a72673c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 23 May 2024 13:41:38 +0200 Subject: [PATCH 019/128] core: fix configuration file, #TASK-6297, #TASK-6255 --- opencga-core/src/main/resources/configuration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 82a80023f9f..75d980fc12d 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -126,8 +126,8 @@ analysis: exomiser-14.0.0: dockerId: "exomiser/exomiser-cli:14.0.0" resources: - HG38: "exomiser-14.0.0/2402_hg38.zip" - PHENOTYPE: "exomiser-14.0.0/2402_phenotype.zip" + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory From e3967729f6e674bd822b8a601c57104c4624e957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 23 May 2024 15:30:25 +0100 Subject: [PATCH 020/128] storage: Add normalization extensions to setup params. #TASK-5861, #TASK-5448 --- .../VariantSetupOperationManager.java | 14 +-- .../operations/VariantIndexOperationTool.java | 8 +- .../manager/VariantOperationsTest.java | 62 +++++++++++-- .../models/variant/VariantSetupParams.java | 25 ++++- .../core/variant/VariantStorageEngine.java | 41 ++++++++- .../core/variant/VariantStoragePipeline.java | 92 +++++++++++++++++-- ...HadoopLocalLoadVariantStoragePipeline.java | 78 ---------------- 7 files changed, 211 insertions(+), 109 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java index c19ab26d0f1..31e8191ec2c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -92,18 +92,18 @@ private void inferParams(VariantSetupParams params) { params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); } else { switch (params.getDataDistribution()) { - case SINGLE_SAMPLE_FILES: + case SINGLE_SAMPLE_PER_FILE: params.setAverageSamplesPerFile(1f); break; - case MULTI_SAMPLE_FILES: + case MULTIPLE_SAMPLES_PER_FILE: params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); break; - case MULTIPLE_FILE_PER_SAMPLE: + case MULTIPLE_FILES_PER_SAMPLE: // Hard to tell. Let's assume 2 samples per file params.setAverageSamplesPerFile(2f); break; - case MULTI_SAMPLE_FILES_SPLIT_BY_CHROMOSOME: - case MULTI_SAMPLE_FILES_SPLIT_BY_REGION: + case FILES_SPLIT_BY_CHROMOSOME: + case FILES_SPLIT_BY_REGION: params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue()); break; default: @@ -132,10 +132,10 @@ private void check(String studyStr, VariantSetupParams params, String token) thr } if (params.getExpectedFiles() == null || params.getExpectedFiles() <= 0) { - throw new IllegalArgumentException("Missing expectedFilesNumber"); + throw new IllegalArgumentException("Missing expectedFiles"); } if (params.getExpectedSamples() == null || params.getExpectedSamples() <= 0) { - throw new IllegalArgumentException("Missing expectedSamplesNumber"); + throw new IllegalArgumentException("Missing expectedSamples"); } if (params.getAverageFileSize() == null && params.getFileType() == null) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java index cce9fc91e8a..3a909c0fdf8 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java @@ -83,7 +83,9 @@ protected void check() throws Exception { params.putIfNotEmpty(VariantStorageOptions.INCLUDE_GENOTYPE.key(), indexParams.getIncludeGenotypes()); params.put(VariantStorageOptions.STATS_AGGREGATION.key(), indexParams.getAggregated()); params.putIfNotEmpty(VariantStorageOptions.STATS_AGGREGATION_MAPPING_FILE.key(), indexParams.getAggregationMappingFile()); - params.put(VariantStorageOptions.GVCF.key(), indexParams.isGvcf()); + if (indexParams.isGvcf()) { + params.put(VariantStorageOptions.GVCF.key(), indexParams.isGvcf()); + } // queryOptions.putIfNotNull(VariantFileIndexerStorageOperation.TRANSFORMED_FILES, indexParams.transformedPaths); @@ -98,7 +100,9 @@ protected void check() throws Exception { params.put(VariantStorageOptions.FAMILY.key(), indexParams.isFamily()); params.put(VariantStorageOptions.SOMATIC.key(), indexParams.isSomatic()); params.putIfNotEmpty(VariantStorageOptions.LOAD_SPLIT_DATA.key(), indexParams.getLoadSplitData()); - params.put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), indexParams.isLoadMultiFileData()); + if (indexParams.isLoadMultiFileData()) { + params.put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), indexParams.isLoadMultiFileData()); + } params.putIfNotEmpty(VariantStorageOptions.LOAD_SAMPLE_INDEX.key(), indexParams.getLoadSampleIndex()); params.putIfNotEmpty(VariantStorageOptions.LOAD_ARCHIVE.key(), indexParams.getLoadArchive()); params.putIfNotEmpty(VariantStorageOptions.LOAD_HOM_REF.key(), indexParams.getLoadHomRef()); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index 22e23ff997f..d50f75a3ad8 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -65,6 +65,8 @@ import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -94,8 +96,9 @@ public class VariantOperationsTest { public static final String USER = "user"; public static final String PASSWORD = TestParamConstants.PASSWORD; public static final String PROJECT = "project"; + public static final String PROJECT_FQN = ORGANIZATION + '@' + PROJECT; public static final String STUDY = "study"; - public static final String STUDY_FQN = ORGANIZATION + '@' + PROJECT + ':' + STUDY; + public static final String STUDY_FQN = PROJECT_FQN + ':' + STUDY; public static final String PHENOTYPE_NAME = "myPhenotype"; public static final Phenotype PHENOTYPE = new Phenotype(PHENOTYPE_NAME, PHENOTYPE_NAME, "mySource") .setStatus(Phenotype.Status.OBSERVED); @@ -331,14 +334,11 @@ public void setUpCatalogManager() throws Exception { @Test public void testSetup() throws Exception { - VariantSetupParams setupParams = new VariantSetupParams(); - setupParams - .setFileType(VariantSetupParams.FileType.GENOME_VCF) - .setDataDistribution(VariantSetupParams.DataDistribution.MULTI_SAMPLE_FILES) - .setExpectedFiles(20) - .setExpectedSamples(100); String study2 = "study2"; - catalogManager.getStudyManager().create(PROJECT, study2, null, "Phase 1", "Done", null, null, null, null, null, token); + String study2fqn = catalogManager.getStudyManager() + .create(PROJECT, study2, null, "Phase 1", "Done", null, null, null, null, null, token) + .first().getFqn(); + File file = opencga.createFile(study2, "variant-test-file.vcf.gz", token); try { toolRunner.execute(VariantIndexOperationTool.class, study2, @@ -352,9 +352,55 @@ public void testSetup() throws Exception { MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); } + try { + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_SAMPLES_PER_FILE) + .setExpectedFiles(20) + .setExpectedSamples(100) + .setNormalizeExtensions(Arrays.asList("VS", "SV")); + variantStorageManager.variantSetup(study2, setupParams, token); + fail("should have failed"); + } catch (Exception e) { + System.err.println(e.getMessage()); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Unsupported normalize extensions")); + } + + try { + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_SAMPLES_PER_FILE) + .setExpectedSamples(100) + .setNormalizeExtensions(Arrays.asList("VS", "SV")); + variantStorageManager.variantSetup(study2, setupParams, token); + fail("should have failed"); + } catch (Exception e) { + System.err.println(e.getMessage()); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Missing expectedFiles")); + } + + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_FILES_PER_SAMPLE) + .setExpectedFiles(20) + .setAverageSamplesPerFile(2.5f) + .setExpectedSamples(10) + .setNormalizeExtensions(Arrays.asList("SV", "VAF")); VariantSetupResult result = variantStorageManager.variantSetup(study2, setupParams, token); assertEquals(VariantSetupResult.Status.READY, result.getStatus()); + toolRunner.execute(VariantIndexOperationTool.class, study2, + new VariantIndexParams() + .setFile(file.getId()) + .setLoadHomRef(YesNoAuto.YES.name()), + Paths.get(opencga.createTmpOutdir("_index")), "index", token); + + VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject(PROJECT_FQN).getMetadataManager(); + int studyId = metadataManager.getStudyId(study2fqn); + int sampleId = metadataManager.getSampleId(studyId, "NA19600"); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); + assertEquals(VariantStorageEngine.SplitData.MULTI, sampleMetadata.getSplitData()); + try { variantStorageManager.variantSetup(STUDY, setupParams, token); fail("Should fail"); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java index ecac975e112..2c323846d50 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -3,6 +3,8 @@ import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.tools.ToolParams; +import java.util.List; + public class VariantSetupParams extends ToolParams { @DataField(description = "Expected number of samples that will be loaded. Used to infer some parameters. " @@ -38,6 +40,9 @@ public class VariantSetupParams extends ToolParams { @DataField(description = "Data distribution of the files. This parameter is used to infer the number of samples per file.") private DataDistribution dataDistribution; + @DataField(description = "List of normalization extensions") + private List normalizeExtensions; + public VariantSetupParams(VariantSetupParams params) { this.expectedSamples = params.expectedSamples; this.expectedFiles = params.expectedFiles; @@ -46,6 +51,7 @@ public VariantSetupParams(VariantSetupParams params) { this.variantsPerSample = params.variantsPerSample; this.averageSamplesPerFile = params.averageSamplesPerFile; this.dataDistribution = params.dataDistribution; + this.normalizeExtensions = params.normalizeExtensions; } public VariantSetupParams() { @@ -58,30 +64,30 @@ public enum DataDistribution { // - Cancer germline // - RD germline without family calling @DataField(description = "Single sample VCF files. One file per sample. e.g. Platinum gVCF, Cancer germline, RD germline without family calling") - SINGLE_SAMPLE_FILES, + SINGLE_SAMPLE_PER_FILE, // Multi samples VCF files. One file with multiple samples. // e.g. // - Corpasome // - RD germline with family calling @DataField(description = "Multi samples VCF files. One file with multiple samples. e.g. Corpasome, RD germline with family calling") - MULTI_SAMPLE_FILES, + MULTIPLE_SAMPLES_PER_FILE, // Multiple files per sample. Each file might have multiple samples. // e.g. // - Somatic study with multiple callers @DataField(description = "Multiple files per sample. Each file might have multiple samples. e.g. Somatic study with multiple callers") - MULTIPLE_FILE_PER_SAMPLE, + MULTIPLE_FILES_PER_SAMPLE, // Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. // e.g. // - 1000 genomes @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. e.g. 1000 genomes") - MULTI_SAMPLE_FILES_SPLIT_BY_CHROMOSOME, + FILES_SPLIT_BY_CHROMOSOME, // Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region. @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region.") - MULTI_SAMPLE_FILES_SPLIT_BY_REGION, + FILES_SPLIT_BY_REGION, } public enum FileType { @@ -155,4 +161,13 @@ public VariantSetupParams setDataDistribution(DataDistribution dataDistribution) this.dataDistribution = dataDistribution; return this; } + + public List getNormalizeExtensions() { + return normalizeExtensions; + } + + public VariantSetupParams setNormalizeExtensions(List normalizeExtensions) { + this.normalizeExtensions = normalizeExtensions; + return this; + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index 74d075f2fc3..07e7f2dc5ba 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -24,9 +24,11 @@ import org.opencb.biodata.models.variant.avro.VariantAnnotation; import org.opencb.biodata.models.variant.metadata.SampleVariantStats; import org.opencb.biodata.models.variant.metadata.VariantMetadata; +import org.opencb.biodata.tools.variant.normalizer.extensions.VariantNormalizerExtensionFactory; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.commons.datastore.core.*; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; @@ -154,7 +156,12 @@ public static SplitData from(ObjectMap options) { String loadSplitDataStr = options.getString(LOAD_SPLIT_DATA.key()); boolean multiFile = options.getBoolean(LOAD_MULTI_FILE_DATA.key()); if (StringUtils.isNotEmpty(loadSplitDataStr) && multiFile) { - throw new IllegalArgumentException("Unable to mix loadSplitFile and loadMultiFile"); + if (loadSplitDataStr.equalsIgnoreCase("multi")) { + return MULTI; + } else { + throw new IllegalArgumentException("Unable to mix " + LOAD_MULTI_FILE_DATA.key() + "=true and " + + LOAD_SPLIT_DATA.key() + "='" + loadSplitDataStr + "'"); + } } if (StringUtils.isEmpty(loadSplitDataStr) && !multiFile) { return null; @@ -1408,6 +1415,38 @@ public VariantAggregationExecutor getVariantAggregationExecutor(Query query, Que public ObjectMap inferConfigurationParams(VariantSetupParams params) { ObjectMap options = new ObjectMap(); + List normalizeExtensions = params.getNormalizeExtensions(); + if (normalizeExtensions != null && !normalizeExtensions.isEmpty()) { + if (!normalizeExtensions.equals(Collections.singletonList(ParamConstants.ALL))) { + List unsupportedExtensions = new ArrayList<>(); + for (String normalizeExtension : normalizeExtensions) { + if (!VariantNormalizerExtensionFactory.ALL_EXTENSIONS.contains(normalizeExtension)) { + unsupportedExtensions.add(normalizeExtension); + } + } + if (!unsupportedExtensions.isEmpty()) { + throw new IllegalArgumentException("Unsupported normalize extensions: " + unsupportedExtensions + ". Supported " + + "extensions are: " + VariantNormalizerExtensionFactory.ALL_EXTENSIONS); + } + } + options.put(NORMALIZATION_EXTENSIONS.key(), normalizeExtensions); + } + if (params.getDataDistribution() != null) { + switch (params.getDataDistribution()) { + case FILES_SPLIT_BY_CHROMOSOME: + options.put(LOAD_SPLIT_DATA.key(), SplitData.CHROMOSOME); + break; + case FILES_SPLIT_BY_REGION: + options.put(LOAD_SPLIT_DATA.key(), SplitData.REGION); + break; + case MULTIPLE_FILES_PER_SAMPLE: + options.put(LOAD_MULTI_FILE_DATA.key(), true); + options.put(LOAD_SPLIT_DATA.key(), SplitData.MULTI); + break; + default: + break; + } + } return options; } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 19b0631659f..7c0197ac034 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -60,9 +60,7 @@ import org.opencb.opencga.storage.core.io.plain.StringDataReader; import org.opencb.opencga.storage.core.io.plain.StringDataWriter; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils; @@ -77,6 +75,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; @@ -555,17 +554,19 @@ public URI preLoad(URI input, URI output) throws StorageEngineException { return input; } - protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata fileMetadata) throws StorageEngineException { + protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata variantFileMetadata) throws StorageEngineException { final int fileId; String virtualFile = options.getString(LOAD_VIRTUAL_FILE.key()); + boolean loadSampleIndex = YesNoAuto.parse(options, LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); + VariantStorageEngine.SplitData splitData = VariantStorageEngine.SplitData.from(options); - if (VariantStorageEngine.SplitData.isPartialSplit(options)) { + if (VariantStorageEngine.SplitData.isPartialSplit(splitData)) { if (StringUtils.isEmpty(virtualFile)) { - fileId = getMetadataManager().registerFile(studyId, fileMetadata); + fileId = getMetadataManager().registerFile(studyId, variantFileMetadata); // throw new StorageEngineException("Unable to load file with 'split-data'. Missing virtual file belonging! " // + "Please, define " + LOAD_VIRTUAL_FILE.key()); } else { - fileId = getMetadataManager().registerPartialFile(studyId, virtualFile, fileMetadata); + fileId = getMetadataManager().registerPartialFile(studyId, virtualFile, variantFileMetadata); } } else { if (StringUtils.isNotEmpty(virtualFile)) { @@ -574,10 +575,85 @@ protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata f + " to " + VariantStorageEngine.SplitData.REGION + " or " + VariantStorageEngine.SplitData.CHROMOSOME); } else { - fileId = getMetadataManager().registerFile(studyId, fileMetadata); + fileId = getMetadataManager().registerFile(studyId, variantFileMetadata); } } setFileId(fileId); + FileMetadata fileMetadata = getMetadataManager().getFileMetadata(studyId, getFileId()); + + int version = getMetadataManager().getStudyMetadata(studyId).getSampleIndexConfigurationLatest().getVersion(); + Set alreadyIndexedSamples = new LinkedHashSet<>(); + Set processedSamples = new LinkedHashSet<>(); + Set samplesWithoutSplitData = new LinkedHashSet<>(); + for (String sample : variantFileMetadata.getSampleIds()) { + Integer sampleId = getMetadataManager().getSampleId(studyId, sample); + SampleMetadata sampleMetadata = getMetadataManager().getSampleMetadata(studyId, sampleId); + if (splitData != null && sampleMetadata.getSplitData() != null) { + if (splitData != sampleMetadata.getSplitData()) { + throw new StorageEngineException("Incompatible split data methods. " + + "Unable to mix requested " + splitData + + " with existing " + sampleMetadata.getSplitData()); + } + } + if (sampleMetadata.isIndexed()) { + if (sampleMetadata.getFiles().size() == 1 && sampleMetadata.getFiles().contains(fileMetadata.getId())) { + // It might happen that the sample is marked as INDEXED, but not the file. + // If the sample only belongs to this file (i.e. it's only file is this file), then ignore + // the overwrite the current sample metadata index status + sampleMetadata = getMetadataManager().updateSampleMetadata(studyId, sampleId, + sm -> sm.setIndexStatus(fileMetadata.getIndexStatus())); + } + } + if (sampleMetadata.isIndexed()) { + alreadyIndexedSamples.add(sample); + if (sampleMetadata.isAnnotated() + || !loadSampleIndex && sampleMetadata.getSampleIndexStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.getSampleIndexAnnotationStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.getFamilyIndexStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.isFamilyIndexDefined()) { + processedSamples.add(sampleMetadata.getId()); + } + } + + if (splitData != null && splitData != sampleMetadata.getSplitData()) { + samplesWithoutSplitData.add(sampleId); + } + } + + if (!alreadyIndexedSamples.isEmpty()) { + if (splitData != null) { + logger.info("Loading split data"); + } else { + String fileName = Paths.get(variantFileMetadata.getPath()).getFileName().toString(); + throw StorageEngineException.alreadyLoadedSamples(fileName, new ArrayList<>(alreadyIndexedSamples)); + } + for (Integer sampleId : processedSamples) { + getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + if (!loadSampleIndex) { + for (Integer v : sampleMetadata.getSampleIndexVersions()) { + sampleMetadata.setSampleIndexStatus(TaskMetadata.Status.NONE, v); + } + } + for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { + sampleMetadata.setSampleIndexAnnotationStatus(TaskMetadata.Status.NONE, v); + } + for (Integer v : sampleMetadata.getFamilyIndexVersions()) { + sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.NONE, v); + } + sampleMetadata.setAnnotationStatus(TaskMetadata.Status.NONE); + sampleMetadata.setMendelianErrorStatus(TaskMetadata.Status.NONE); + }); + } + } + + if (splitData != null) { + // Register loadSplitData + for (Integer sampleId : samplesWithoutSplitData) { + getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.setSplitData(splitData); + }); + } + } } /** diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java index 462aa6858b0..0f85e6204a4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java @@ -37,7 +37,6 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -95,83 +94,6 @@ public HadoopLocalLoadVariantStoragePipeline(StorageConfiguration configuration, @Override protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata variantFileMetadata) throws StorageEngineException { super.preLoadRegisterAndValidateFile(studyId, variantFileMetadata); - boolean loadSampleIndex = YesNoAuto.parse(getOptions(), LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); - FileMetadata fileMetadata = getMetadataManager().getFileMetadata(studyId, getFileId()); - - int version = getMetadataManager().getStudyMetadata(studyId).getSampleIndexConfigurationLatest().getVersion(); - Set alreadyIndexedSamples = new LinkedHashSet<>(); - Set processedSamples = new LinkedHashSet<>(); - Set samplesWithoutSplitData = new LinkedHashSet<>(); - VariantStorageEngine.SplitData splitData = VariantStorageEngine.SplitData.from(options); - for (String sample : variantFileMetadata.getSampleIds()) { - Integer sampleId = getMetadataManager().getSampleId(studyId, sample); - SampleMetadata sampleMetadata = getMetadataManager().getSampleMetadata(studyId, sampleId); - if (splitData != null && sampleMetadata.getSplitData() != null) { - if (!splitData.equals(sampleMetadata.getSplitData())) { - throw new StorageEngineException("Incompatible split data methods. " - + "Unable to mix requested " + splitData - + " with existing " + sampleMetadata.getSplitData()); - } - } - if (sampleMetadata.isIndexed()) { - if (sampleMetadata.getFiles().size() == 1 && sampleMetadata.getFiles().contains(fileMetadata.getId())) { - // It might happen that the sample is marked as INDEXED, but not the file. - // If the sample only belongs to this file (i.e. it's only file is this file), then ignore - // the overwrite the current sample metadata index status - sampleMetadata = getMetadataManager().updateSampleMetadata(studyId, sampleId, - sm -> sm.setIndexStatus(fileMetadata.getIndexStatus())); - } - } - if (sampleMetadata.isIndexed()) { - alreadyIndexedSamples.add(sample); - if (sampleMetadata.isAnnotated() - || !loadSampleIndex && sampleMetadata.getSampleIndexStatus(version) == Status.READY - || sampleMetadata.getSampleIndexAnnotationStatus(version) == Status.READY - || sampleMetadata.getFamilyIndexStatus(version) == Status.READY - || sampleMetadata.isFamilyIndexDefined()) { - processedSamples.add(sampleMetadata.getId()); - } - } - - if (splitData != null && splitData != sampleMetadata.getSplitData()) { - samplesWithoutSplitData.add(sampleId); - } - } - - if (!alreadyIndexedSamples.isEmpty()) { - if (splitData != null) { - logger.info("Loading split data"); - } else { - String fileName = Paths.get(variantFileMetadata.getPath()).getFileName().toString(); - throw StorageEngineException.alreadyLoadedSamples(fileName, new ArrayList<>(alreadyIndexedSamples)); - } - for (Integer sampleId : processedSamples) { - getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - if (!loadSampleIndex) { - for (Integer v : sampleMetadata.getSampleIndexVersions()) { - sampleMetadata.setSampleIndexStatus(Status.NONE, v); - } - } - for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { - sampleMetadata.setSampleIndexAnnotationStatus(Status.NONE, v); - } - for (Integer v : sampleMetadata.getFamilyIndexVersions()) { - sampleMetadata.setFamilyIndexStatus(Status.NONE, v); - } - sampleMetadata.setAnnotationStatus(Status.NONE); - sampleMetadata.setMendelianErrorStatus(Status.NONE); - }); - } - } - - if (splitData != null) { - // Register loadSplitData - for (Integer sampleId : samplesWithoutSplitData) { - getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.setSplitData(splitData); - }); - } - } } @Override From 5f1029a6419951ec51f6806734439f239a1f2dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Mon, 27 May 2024 11:29:23 +0100 Subject: [PATCH 021/128] storage: Fix VariantSearchTest. #TASK-6136 --- .../core/variant/search/VariantSearchTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java index 56588d25863..17d7d073c51 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java @@ -187,15 +187,16 @@ public void testSpecialCharacter() throws Exception { List variants = getVariants(limit); List annotatedVariants = annotatedVariants(variants); - String studyId = "abyu12"; - String fileId = "a.vcf"; + String study = "abyu12"; + String file = "a.vcf"; - variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(fileId); + variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(file); System.out.println(variants.get(0).getStudies().get(0).getFiles().get(0).getFileId()); //System.exit(-1); - scm.createStudy(studyId); - + int studyId = scm.createStudy(study).getId(); + int fileId = scm.registerFile(studyId, file, Arrays.asList("A-A", "B", "C", "D")); + scm.addIndexedFiles(studyId, Collections.singletonList(fileId)); String collection = solr.coreName; variantSearchManager.create(collection); @@ -204,13 +205,13 @@ public void testSpecialCharacter() throws Exception { samplePosition.put("B", 1); samplePosition.put("C", 2); samplePosition.put("D", 3); - annotatedVariants.get(0).getStudies().get(0).setStudyId(studyId).setSortedSamplesPosition(samplePosition); + annotatedVariants.get(0).getStudies().get(0).setStudyId(study).setSortedSamplesPosition(samplePosition); variantSearchManager.insert(collection, annotatedVariants); Query query = new Query(); - query.put(VariantQueryParam.STUDY.key(), studyId); + query.put(VariantQueryParam.STUDY.key(), study); // query.put(VariantQueryParam.SAMPLE.key(), samplePosition.keySet().toArray()[0]); - query.put(VariantQueryParam.FILE.key(), fileId); + query.put(VariantQueryParam.FILE.key(), file); query.put(VariantQueryParam.FILTER.key(), "PASS"); query.put(VariantQueryParam.ANNOT_CLINICAL_SIGNIFICANCE.key(), "benign"); VariantQueryResult results = variantSearchManager.query(collection, variantStorageEngine.parseQuery(query, From fc84089b4714692278cee3d6bc2b8e34fe6c6e18 Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 27 May 2024 15:10:16 +0200 Subject: [PATCH 022/128] catalog: add 'version' to ClinicalAnalysis, #TASK-5964 #TASK-6289 --- ...AddVersionToClinicalAnalysisMigration.java | 47 ++++ .../db/api/ClinicalAnalysisDBAdaptor.java | 2 + .../ClinicalAnalysisMongoDBAdaptor.java | 209 +++++++++--------- .../catalog/db/mongodb/MongoDBAdaptor.java | 1 + .../OrganizationMongoDBAdaptorFactory.java | 6 +- .../SnapshotVersionedMongoDBAdaptor.java | 15 +- .../catalog/migration/MigrationTool.java | 21 ++ .../src/main/resources/catalog-indexes.txt | 50 ++--- .../opencga/core/api/ParamConstants.java | 3 + .../models/clinical/ClinicalAnalysis.java | 25 ++- .../ClinicalAnalysisCreateParams.java | 2 +- .../ClinicalAnalysisUpdateParams.java | 2 +- .../opencga/core/models/sample/Sample.java | 3 +- .../rest/analysis/ClinicalWebService.java | 3 + 14 files changed, 245 insertions(+), 144 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java new file mode 100644 index 00000000000..01dabcda776 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java @@ -0,0 +1,47 @@ +package org.opencb.opencga.app.migrations.v3_2_0; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Collections; + +@Migration(id = "add_version_to_clinicalAnalysis", description = "Add version to Clinical Analysis #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240527) +public class AddVersionToClinicalAnalysisMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // Add missing mandatory "versioning" fields to Clinical Analysis + logger.info("Adding missing 'versioning' fields to Clinical Analysis..."); + Bson versionDoesNotExistQuery = Filters.exists("version", false); + Bson projection = Projections.include("release"); + migrateCollection(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, versionDoesNotExistQuery, projection, + (document, bulk) -> { + int release = document.get("release", Number.class).intValue(); + Document updateDocument = new Document() + .append("version", 1) + .append("_releaseFromVersion", Collections.singletonList(release)) + .append("_lastOfVersion", true) + .append("_lastOfRelease", true); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); + }); + + // Recalculate indexes + logger.info("Installing new indexes..."); + catalogManager.installIndexes(organizationId, token); + + // Copy all Clinical Analyses to the archive collection + logger.info("Copying all the data from the main collection to the archive one..."); + copyData(new Document(), OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); + + + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 04f1aeafbe4..0a4c46bc1b6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -78,7 +78,9 @@ enum QueryParams implements QueryParam { RESPONSIBLE("responsible", OBJECT, ""), FLAGS("flags", OBJECT, ""), FLAGS_ID("flags.id", TEXT, ""), + VERSION("version", INTEGER, ""), RELEASE("release", INTEGER, ""), + SNAPSHOT("snapshot", INTEGER, ""), PANEL_LOCK("panelLock", BOOLEAN, ""), LOCKED("locked", BOOLEAN, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index a4073a8c192..c5b67364079 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -82,16 +82,23 @@ public class ClinicalAnalysisMongoDBAdaptor extends AnnotationMongoDBAdaptor result = updateAnnotationSets(clientSession, clinical.getStudyUid(), clinicalAnalysisUid, parameters, variableSetList, - queryOptions, false); + Query tmpQuery = new Query() + .append(STUDY_UID.key(), clinical.getStudyUid()) + .append(QueryParams.UID.key(), clinicalAnalysisUid); + Bson bsonQuery = parseQuery(tmpQuery); + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, (entryList) -> { + DataResult result = updateAnnotationSets(clientSession, clinical.getStudyUid(), clinicalAnalysisUid, parameters, + variableSetList, queryOptions, false); - // Perform the update - Query query = new Query(QueryParams.UID.key(), clinicalAnalysisUid); - UpdateDocument updateDocument = parseAndValidateUpdateParams(parameters, clinicalAuditList, query, queryOptions); + // Perform the update + UpdateDocument updateDocument = parseAndValidateUpdateParams(parameters, clinicalAuditList, tmpQuery, queryOptions); - Document updateOperation = updateDocument.toFinalUpdateDocument(); - List events = new ArrayList<>(); - if (!updateOperation.isEmpty() || !updateDocument.getNestedUpdateList().isEmpty()) { - DataResult update; + Document updateOperation = updateDocument.toFinalUpdateDocument(); + List events = new ArrayList<>(); + if (!updateOperation.isEmpty() || !updateDocument.getNestedUpdateList().isEmpty()) { + DataResult update; - if (!updateOperation.isEmpty()) { - Bson bsonQuery = Filters.eq(PRIVATE_UID, clinicalAnalysisUid); + if (!updateOperation.isEmpty()) { + logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); + update = clinicalCollection.update(clientSession, bsonQuery, updateOperation, null); - logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); - update = clinicalCollection.update(clientSession, bsonQuery, updateOperation, null); + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + } + if (update.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, clinicalAnalysisId, "Clinical Analysis was already updated")); + } - if (update.getNumMatches() == 0) { - throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); - } - if (update.getNumUpdated() == 0) { - events.add(new Event(Event.Type.WARNING, clinicalAnalysisId, "Clinical Analysis was already updated")); - } + if (parameters.getBoolean(LOCKED.key())) { + // Propagate locked value to Interpretations + logger.debug("Propagating case lock to all the Interpretations"); + dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, + parameters.getBoolean(LOCKED.key())); + } - if (parameters.getBoolean(LOCKED.key())) { - // Propagate locked value to Interpretations - logger.debug("Propagating case lock to all the Interpretations"); - dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, - parameters.getBoolean(LOCKED.key())); + logger.debug("Clinical Analysis {} successfully updated", clinicalAnalysisId); } - logger.debug("Clinical Analysis {} successfully updated", clinicalAnalysisId); - } - - if (!updateDocument.getNestedUpdateList().isEmpty()) { - for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + if (!updateDocument.getNestedUpdateList().isEmpty()) { + // Nested documents are used to update reports + for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + Bson tmpBsonQuery = parseQuery(nestedDocument.getQuery().append(QueryParams.UID.key(), clinicalAnalysisUid)); + logger.debug("Update nested element from Clinical Analysis. Query: {}, Update: {}", + tmpBsonQuery.toBsonDocument(), nestedDocument.getSet()); + update = clinicalCollection.update(clientSession, tmpBsonQuery, nestedDocument.getSet(), null); - Bson bsonQuery = parseQuery(nestedDocument.getQuery().append(QueryParams.UID.key(), clinicalAnalysisUid)); - logger.debug("Update nested element from Clinical Analysis. Query: {}, Update: {}", - bsonQuery.toBsonDocument(), nestedDocument.getSet()); - - update = clinicalCollection.update(clientSession, bsonQuery, nestedDocument.getSet(), null); - - if (update.getNumMatches() == 0) { - throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + } } } - } - } else if (result.getNumUpdated() == 0) { - throw new CatalogDBException("Nothing to update"); - } + } else if (result.getNumUpdated() == 0) { + throw new CatalogDBException("Nothing to update"); + } - return endWrite(tmpStartTime, 1, 1, events); + return endWrite(tmpStartTime, 1, 1, events); + }, null, null); } @Override @@ -354,22 +363,24 @@ OpenCGAResult transactionalUpdate(ClientSession clientSession, Document updateOperation = updateDocument.toFinalUpdateDocument(); if (!updateOperation.isEmpty()) { - logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument); - DataResult update = clinicalCollection.update(clientSession, query, updateOperation, null); - - if (updateDocument.getSet().getBoolean(LOCKED.key(), false)) { - // Propagate locked value to Interpretations - logger.debug("Propagating case lock to all the Interpretations"); - MongoDBIterator iterator = clinicalCollection.iterator(clientSession, query, null, clinicalConverter, - ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); - while (iterator.hasNext()) { - ClinicalAnalysis clinical = iterator.next(); - dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, true); + return versionedMongoDBAdaptor.update(clientSession, query, entryList -> { + logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument); + DataResult update = clinicalCollection.update(clientSession, query, updateOperation, null); + + if (updateDocument.getSet().getBoolean(LOCKED.key(), false)) { + // Propagate locked value to Interpretations + logger.debug("Propagating case lock to all the Interpretations"); + MongoDBIterator iterator = clinicalCollection.iterator(clientSession, query, null, clinicalConverter, + ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); + while (iterator.hasNext()) { + ClinicalAnalysis clinical = iterator.next(); + dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, true); + } } - } - logger.debug("{} clinical analyses successfully updated", update.getNumUpdated()); - return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList()); + logger.debug("{} clinical analyses successfully updated", update.getNumUpdated()); + return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList()); + }, null, null); } else { throw new CatalogDBException("Nothing to update"); } @@ -765,52 +776,18 @@ OpenCGAResult privateDelete(ClientSession clientSession, ClinicalAnalysis cli } } + // Add Audit to ClinicalAnalysis + transactionalUpdate(clientSession, clinicalAnalysis, new ObjectMap(), Collections.emptyList(), clinicalAuditList, + QueryOptions.empty()); + + // And delete ClinicalAnalysis Query query = new Query() - .append(STUDY_UID.key(), clinicalAnalysis.getStudyUid()) + .append(QueryParams.STUDY_UID.key(), clinicalAnalysis.getStudyUid()) .append(UID.key(), clinicalAnalysis.getUid()); - OpenCGAResult result = nativeGet(clientSession, query, QueryOptions.empty()); - if (result.getNumResults() == 0) { - throw new CatalogDBException("Internal error: Clinical Analysis '" + clinicalAnalysis.getId() + "' not found."); - } - - String clinicalId = result.first().getString(QueryParams.ID.key()); - long clinicalUid = result.first().getLong(PRIVATE_UID); - long studyUid = result.first().getLong(PRIVATE_STUDY_UID); - - logger.debug("Deleting Clinical Analysis {} ({})", clinicalId, clinicalUid); - - // Delete any documents that might have been already deleted with that id - Bson bsonQuery = new Document() - .append(QueryParams.ID.key(), clinicalId) - .append(PRIVATE_STUDY_UID, studyUid); - deletedClinicalCollection.remove(clientSession, bsonQuery, new QueryOptions(MongoDBCollection.MULTI, true)); - - // Set status to DELETED - nestedPut(QueryParams.INTERNAL_STATUS.key(), getMongoDBDocument(new InternalStatus(InternalStatus.DELETED), "status"), - result.first()); - - // Add audit - List auditList = result.first().getList(AUDIT.key(), Document.class); - for (ClinicalAudit clinicalAudit : clinicalAuditList) { - auditList.add(getMongoDBDocument(clinicalAudit, "ClinicalAudit")); - } - result.first().put(AUDIT.key(), auditList); - - // Insert the document in the DELETE collection - deletedClinicalCollection.insert(clientSession, replaceDotsInKeys(result.first()), null); - logger.debug("Inserted Clinical Analysis uid '{}' in DELETE collection", clinicalUid); - - // Remove the document from the main Clinical collection - bsonQuery = parseQuery(new Query(QueryParams.UID.key(), clinicalUid)); - DataResult remove = clinicalCollection.remove(clientSession, bsonQuery, null); - if (remove.getNumMatches() == 0) { - throw new CatalogDBException("Clinical Analysis " + clinicalId + " not found"); - } - if (remove.getNumDeleted() == 0) { - throw new CatalogDBException("Clinical Analysis " + clinicalId + " could not be deleted"); - } + Bson bsonQuery = parseQuery(query); + versionedMongoDBAdaptor.delete(clientSession, bsonQuery); - logger.debug("Clinical Analysis {}({}) deleted", clinicalId, clinicalUid); + logger.debug("Clinical Analysis {}({}) deleted", clinicalAnalysis.getId(), clinicalAnalysis.getUid()); return endWrite(tmpStartTime, 1, 0, 0, 1, Collections.emptyList()); } @@ -944,12 +921,8 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = removeInnerProjections(qOptions, QueryParams.SECONDARY_INTERPRETATIONS.key()); logger.debug("Clinical analysis query : {}", bson.toBsonDocument()); - - if (!query.getBoolean(QueryParams.DELETED.key())) { - return clinicalCollection.iterator(clientSession, bson, null, null, qOptions); - } else { - return deletedClinicalCollection.iterator(clientSession, bson, null, null, qOptions); - } + MongoDBCollection collection = getQueryCollection(query, clinicalCollection, archiveClinicalCollection, deletedClinicalCollection); + return collection.iterator(clientSession, bson, null, null, qOptions); } @Override @@ -1082,7 +1055,7 @@ ClinicalAnalysis insert(ClientSession clientSession, long studyId, ClinicalAnaly clinicalDocument.put(PRIVATE_DUE_DATE, TimeUtils.toDate(clinicalAnalysis.getDueDate())); logger.debug("Inserting ClinicalAnalysis '{}' ({})...", clinicalAnalysis.getId(), clinicalAnalysis.getUid()); - clinicalCollection.insert(clientSession, clinicalDocument, null); + versionedMongoDBAdaptor.insert(clientSession, clinicalDocument); logger.debug("ClinicalAnalysis '{}' successfully inserted", clinicalAnalysis.getId()); return clinicalAnalysis; @@ -1309,12 +1282,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) queryCopy.remove(ParamConstants.ACL_PARAM); } + if ("all".equalsIgnoreCase(queryCopy.getString(QueryParams.VERSION.key()))) { + queryCopy.put(Constants.ALL_VERSIONS, true); + queryCopy.remove(QueryParams.VERSION.key()); + } + boolean uidVersionQueryFlag = versionedMongoDBAdaptor.generateUidVersionQuery(queryCopy, andBsonList); + for (Map.Entry entry : queryCopy.entrySet()) { String key = entry.getKey().split("\\.")[0]; QueryParams queryParam = QueryParams.getParam(entry.getKey()) != null ? QueryParams.getParam(entry.getKey()) : QueryParams.getParam(key); if (queryParam == null) { - if (Constants.PRIVATE_ANNOTATION_PARAM_TYPES.equals(entry.getKey())) { + if (Constants.ALL_VERSIONS.equals(entry.getKey()) || Constants.PRIVATE_ANNOTATION_PARAM_TYPES.equals(entry.getKey())) { continue; } throw new CatalogDBException("Unexpected parameter " + entry.getKey() + ". The parameter does not exist or cannot be " @@ -1387,6 +1366,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case PRIORITY_ID: case FLAGS_ID: case QUALITY_CONTROL_SUMMARY: + case VERSION: case RELEASE: case COMMENTS_DATE: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); @@ -1400,6 +1380,17 @@ private Bson parseQuery(Query query, Document extraQuery, String user) } } + // If the user doesn't look for a concrete version... + if (!uidVersionQueryFlag && !queryCopy.getBoolean(Constants.ALL_VERSIONS) && !queryCopy.containsKey(QueryParams.VERSION.key())) { + if (queryCopy.containsKey(QueryParams.SNAPSHOT.key())) { + // If the user looks for anything from some release, we will try to find the latest from the release (snapshot) + andBsonList.add(Filters.eq(LAST_OF_RELEASE, true)); + } else { + // Otherwise, we will always look for the latest version + andBsonList.add(Filters.eq(LAST_OF_VERSION, true)); + } + } + if (annotationDocument != null && !annotationDocument.isEmpty()) { andBsonList.add(annotationDocument); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java index 0b6f07d4269..a4721dffc8d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java @@ -58,6 +58,7 @@ public abstract class MongoDBAdaptor extends AbstractDBAdaptor { static final String PRIVATE_PROJECT_UUID = PRIVATE_PROJECT + '.' + PRIVATE_UUID; public static final String PRIVATE_STUDY_UID = "studyUid"; public static final String VERSION = "version"; + public static final String RELEASE = "release"; static final String FILTER_ROUTE_STUDIES = "projects.studies."; static final String FILTER_ROUTE_COHORTS = "projects.studies.cohorts."; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index b1c1c6704ec..be76b219705 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -48,6 +48,7 @@ public class OrganizationMongoDBAdaptorFactory { public static final String INDIVIDUAL_ARCHIVE_COLLECTION = "individual_archive"; public static final String FAMILY_ARCHIVE_COLLECTION = "family_archive"; public static final String PANEL_ARCHIVE_COLLECTION = "panel_archive"; + public static final String CLINICAL_ANALYSIS_ARCHIVE_COLLECTION = "clinical_archive"; public static final String INTERPRETATION_ARCHIVE_COLLECTION = "interpretation_archive"; public static final String DELETED_NOTE_COLLECTION = "note_deleted"; @@ -89,6 +90,7 @@ public class OrganizationMongoDBAdaptorFactory { INDIVIDUAL_ARCHIVE_COLLECTION, FAMILY_ARCHIVE_COLLECTION, PANEL_ARCHIVE_COLLECTION, + CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, INTERPRETATION_ARCHIVE_COLLECTION, DELETED_NOTE_COLLECTION, @@ -176,6 +178,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa MongoDBCollection individualArchivedCollection = mongoDataStore.getCollection(INDIVIDUAL_ARCHIVE_COLLECTION); MongoDBCollection familyArchivedCollection = mongoDataStore.getCollection(FAMILY_ARCHIVE_COLLECTION); MongoDBCollection panelArchivedCollection = mongoDataStore.getCollection(PANEL_ARCHIVE_COLLECTION); + MongoDBCollection clinicalArchivedCollection = mongoDataStore.getCollection(CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); MongoDBCollection interpretationArchivedCollection = mongoDataStore.getCollection(INTERPRETATION_ARCHIVE_COLLECTION); MongoDBCollection deletedNotesCollection = mongoDataStore.getCollection(DELETED_NOTE_COLLECTION); @@ -211,7 +214,8 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa userDBAdaptor = new UserMongoDBAdaptor(userCollection, deletedUserCollection, configuration, this); cohortDBAdaptor = new CohortMongoDBAdaptor(cohortCollection, deletedCohortCollection, configuration, this); panelDBAdaptor = new PanelMongoDBAdaptor(panelCollection, panelArchivedCollection, deletedPanelCollection, configuration, this); - clinicalDBAdaptor = new ClinicalAnalysisMongoDBAdaptor(clinicalCollection, deletedClinicalCollection, configuration, this); + clinicalDBAdaptor = new ClinicalAnalysisMongoDBAdaptor(clinicalCollection, clinicalArchivedCollection, deletedClinicalCollection, + configuration, this); interpretationDBAdaptor = new InterpretationMongoDBAdaptor(interpretationCollection, interpretationArchivedCollection, deletedInterpretationCollection, configuration, this); // metaDBAdaptor = new MetaMongoDBAdaptor(metaCollection, configuration, this); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java index a86ebb9cc36..c54c80df169 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java @@ -137,6 +137,12 @@ DBIterator iterator(ClientSession session, Query query, QueryOptions options) } protected void insert(ClientSession session, Document document) { + // Versioning private parameters + document.put(VERSION, 1); + document.put(RELEASE_FROM_VERSION, Arrays.asList(document.getInteger(RELEASE))); + document.put(LAST_OF_VERSION, true); + document.put(LAST_OF_RELEASE, true); + String uuid = getClientSessionUuid(session); document.put(PRIVATE_TRANSACTION_ID, uuid); collection.insert(session, document, QueryOptions.empty()); @@ -348,12 +354,17 @@ protected Document revertToVersion(ClientSession clientSession, long uid, int ve return document; } - protected void delete(ClientSession session, Bson query) { + protected void delete(ClientSession session, Bson query) throws CatalogDBException { // Remove any old documents from the "delete" collection matching the criteria deletedCollection.remove(session, query, QueryOptions.empty()); // Remove document from main collection - collection.remove(session, query, QueryOptions.empty()); + DataResult remove = collection.remove(session, query, QueryOptions.empty()); + if (remove.getNumDeleted() == 0) { + logger.error("Delete operation for '{}' could not be performed. Num matches: {}", query.toBsonDocument(), + remove.getNumMatches()); + throw new CatalogDBException("Delete operation could not be performed"); + } // Add versioned documents to "delete" collection InternalStatus internalStatus = new InternalStatus(InternalStatus.DELETED); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java index 47a701f0ed2..26060798aa0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java @@ -3,6 +3,8 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.IndexOptions; +import com.mongodb.client.model.InsertOneModel; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.WriteModel; import org.bson.Document; import org.bson.conversions.Bson; @@ -169,6 +171,25 @@ protected final void migrateCollection(MongoCollection inputCollection } } + protected void copyData(Bson query, String sourceCol, String targetCol) throws CatalogDBException { + MongoCollection sourceMongoCollection = getMongoCollection(sourceCol); + MongoCollection targetMongoCollection = getMongoCollection(targetCol); + copyData(query, sourceMongoCollection, targetMongoCollection); + } + + protected void copyData(Bson query, MongoCollection sourceCol, MongoCollection targetCol) { + // Move data to the new collection + logger.info("Copying data from {} to {}", sourceCol.getNamespace(), targetCol.getNamespace()); + migrateCollection(sourceCol, targetCol, query, Projections.exclude("_id"), + (document, bulk) -> bulk.add(new InsertOneModel<>(document))); + } + + protected void moveData(Bson query, MongoCollection sourceCol, MongoCollection targetCol) { + copyData(query, sourceCol, targetCol); + // Remove data from the source collection + sourceCol.deleteMany(query); + } + protected final void createIndex(String collection, Document index) throws CatalogDBException { createIndex(getMongoCollection(collection), index, new IndexOptions().background(true)); } diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 6cabeed184d..dab0064fbb5 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -207,31 +207,31 @@ {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"uuid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"uid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"type": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"panelLock": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"type": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"panelLock": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 41c17e5a787..b669c3e4dbc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -435,6 +435,9 @@ public class ParamConstants { public static final String CLINICAL_RELEASE_PARAM = RELEASE_PARAM; public static final String CLINICAL_STATUS_PARAM = STATUS_PARAM; public static final String CLINICAL_INTERNAL_STATUS_PARAM = INTERNAL_STATUS_PARAM; + public static final String CLINICAL_VERSION_PARAM = "version"; + public static final String CLINICAL_VERSION_DESCRIPTION = "Comma separated list of clinical versions. 'all' to get all the clinical" + + " versions. Not supported if multiple clinical ids are provided"; public static final String CLINICAL_TYPE_DESCRIPTION = "Clinical Analysis type"; public static final String CLINICAL_DISORDER_DESCRIPTION = "Clinical Analysis disorder" + REGEX_SUPPORT; public static final String CLINICAL_FILES_DESCRIPTION = "Clinical Analysis files"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java index 61bc01c644f..060e27696c6 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java @@ -179,6 +179,14 @@ public class ClinicalAnalysis extends Annotable { description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; + /** + * Generic: Autoincremental version assigned to the registered entry. + * + * @apiNote Immutable + */ + @DataField(id = "version", managed = true, indexed = true, description = FieldConstants.GENERIC_VERSION_DESCRIPTION) + private int version; + @DataField(id = "qualityControl", indexed = true, description = FieldConstants.GENERIC_QUALITY_CONTROL) private ClinicalAnalysisQualityControl qualityControl; @@ -227,9 +235,9 @@ public ClinicalAnalysis(String id, String description, Type type, Disorder disor List secondaryInterpretations, ClinicalConsentAnnotation consent, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, ClinicalPriorityAnnotation priority, List flags, String creationDate, String modificationDate, - String dueDate, int release, List comments, ClinicalAnalysisQualityControl qualityControl, - List audit, ClinicalAnalysisInternal internal, List annotationSets, - Map attributes, Status status) { + String dueDate, int release, int version, List comments, + ClinicalAnalysisQualityControl qualityControl, List audit, ClinicalAnalysisInternal internal, + List annotationSets, Map attributes, Status status) { this.id = id; this.description = description; this.type = type; @@ -254,6 +262,7 @@ public ClinicalAnalysis(String id, String description, Type type, Disorder disor this.dueDate = dueDate; this.qualityControl = qualityControl; this.release = release; + this.version = version; this.comments = comments; this.audit = audit; this.internal = internal; @@ -289,6 +298,7 @@ public String toString() { sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", dueDate='").append(dueDate).append('\''); sb.append(", release=").append(release); + sb.append(", version=").append(version); sb.append(", qualityControl=").append(qualityControl); sb.append(", comments=").append(comments); sb.append(", audit=").append(audit); @@ -532,6 +542,15 @@ public ClinicalAnalysis setRelease(int release) { return this; } + public int getVersion() { + return version; + } + + public ClinicalAnalysis setVersion(int version) { + this.version = version; + return this; + } + public List getComments() { return comments; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java index 7ffbf4b4a9a..1809e375628 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java @@ -235,7 +235,7 @@ public ClinicalAnalysis toClinicalAnalysis() { consent != null ? consent.toClinicalConsentAnnotation() : null, clinicalAnalystList, report, request, responsible, priority != null ? priority.toClinicalPriorityAnnotation() : null, flags != null ? flags.stream().map(FlagValueParam::toFlagAnnotation).collect(Collectors.toList()) : null, creationDate, modificationDate, dueDate, - 1, + 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, new LinkedList<>(), null, annotationSets, attributes, status != null ? status.toStatus() : null); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java index 020a44ead6a..8f6b5e1cef3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java @@ -133,7 +133,7 @@ public ClinicalAnalysis toClinicalAnalysis() { report, request, responsible, priority != null ? priority.toClinicalPriorityAnnotation() : null, flags != null ? flags.stream().map(FlagValueParam::toFlagAnnotation).collect(Collectors.toList()) : null, creationDate, modificationDate, dueDate, - 1, + 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, null, null, annotationSets, attributes, status != null ? status.toStatus() : null); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java index f45e545a292..10b42230abe 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java @@ -107,8 +107,7 @@ public class Sample extends Annotable { private int release; /** - * Generic: Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable - * versioning, users must set the `incVersion` flag from the /update web service when updating the document. + * Generic: Autoincremental version assigned to the registered entry. * * @apiNote Immutable */ diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java index d69c237e421..11bb051b37f 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java @@ -398,6 +398,7 @@ public Response delete( public Response info( @ApiParam(value = ParamConstants.CLINICAL_ANALYSES_DESCRIPTION) @PathParam(value = "clinicalAnalysis") String clinicalAnalysisStr, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.CLINICAL_VERSION_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_VERSION_PARAM) String version, @ApiParam(value = ParamConstants.DELETED_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.DELETED_PARAM) boolean deleted) { try { query.remove(ParamConstants.STUDY_PARAM); @@ -447,6 +448,7 @@ public Response search( @ApiParam(value = ParamConstants.CLINICAL_DUE_DATE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_DUE_DATE_PARAM) String dueDate, @ApiParam(value = ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_PARAM) String qualityControl, @ApiParam(value = ParamConstants.CLINICAL_RELEASE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, @ApiParam(value = ParamConstants.CLINICAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_STATUS_PARAM) String status, @ApiParam(value = ParamConstants.CLINICAL_INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam(Constants.ANNOTATION) String annotation, @@ -487,6 +489,7 @@ public Response distinct( @ApiParam(value = ParamConstants.CLINICAL_DUE_DATE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_DUE_DATE_PARAM) String dueDate, @ApiParam(value = ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_PARAM) String qualityControl, @ApiParam(value = ParamConstants.CLINICAL_RELEASE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, @ApiParam(value = ParamConstants.CLINICAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_STATUS_PARAM) String status, @ApiParam(value = ParamConstants.CLINICAL_INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam(Constants.ANNOTATION) String annotation, From b5d46c6d894d53e145af94fe5b52306c800d86b2 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 28 May 2024 14:37:27 +0200 Subject: [PATCH 023/128] catalog: add new test for versioning, #TASK-5964 #TASK-6289 --- ...AddVersionToClinicalAnalysisMigration.java | 19 ++++++- .../managers/ClinicalAnalysisManagerTest.java | 51 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java index 01dabcda776..34cb362e25c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java @@ -3,6 +3,7 @@ import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.collections4.CollectionUtils; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; @@ -10,6 +11,7 @@ import org.opencb.opencga.catalog.migration.MigrationTool; import java.util.Collections; +import java.util.List; @Migration(id = "add_version_to_clinicalAnalysis", description = "Add version to Clinical Analysis #TASK-5964", version = "3.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240527) @@ -20,12 +22,25 @@ protected void run() throws Exception { // Add missing mandatory "versioning" fields to Clinical Analysis logger.info("Adding missing 'versioning' fields to Clinical Analysis..."); Bson versionDoesNotExistQuery = Filters.exists("version", false); - Bson projection = Projections.include("release"); + Bson projection = Projections.include("release", "interpretation", "secondaryInterpretations"); migrateCollection(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, versionDoesNotExistQuery, projection, (document, bulk) -> { + int version = 1; + Document interpretation = document.get("interpretation", Document.class); + if (interpretation != null) { + int interpretationVersion = interpretation.get("version", Number.class).intValue(); + version = Math.max(version, interpretationVersion + 1); + } + List secondaryInterpretations = document.getList("secondaryInterpretations", Document.class); + if (CollectionUtils.isNotEmpty(secondaryInterpretations)) { + for (Document secondaryInterpretation : secondaryInterpretations) { + int secondaryInterpretationVersion = secondaryInterpretation.get("version", Number.class).intValue(); + version = Math.max(version, secondaryInterpretationVersion + 1); + } + } int release = document.get("release", Number.class).intValue(); Document updateDocument = new Document() - .append("version", 1) + .append("version", version) .append("_releaseFromVersion", Collections.singletonList(release)) .append("_lastOfVersion", true) .append("_lastOfRelease", true); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index cf267f04b71..29653b30db5 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -932,6 +932,57 @@ public void updateClinicalAnalysis() throws CatalogException { assertEquals(attributes.get("b"), clinical.getAttributes().get("b")); } + @Test + public void versioningTest() throws CatalogException { + Individual individual = new Individual() + .setId("proband") + .setSamples(Collections.singletonList(new Sample().setId("sample"))); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); + + List findingList = new ArrayList<>(); + VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); + ClinicalVariantEvidence evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); + ClinicalVariant cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + findingList.add(cv1); + variantAvro = new VariantAvro("id2", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); + ClinicalVariant cv2 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + findingList.add(cv2); + + ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() + .setId("Clinical") + .setType(ClinicalAnalysis.Type.SINGLE) + .setProband(individual) + .setInterpretation(new Interpretation() + .setPrimaryFindings(findingList) + ); + ClinicalAnalysis clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken).first(); + assertEquals(1, clinical.getVersion()); + assertEquals(1, clinical.getInterpretation().getVersion()); + + // Update clinical analysis + clinical = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setDescription("my new description"), INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new description", clinical.getDescription()); + assertEquals(2, clinical.getVersion()); + assertEquals(1, clinical.getInterpretation().getVersion()); + + // Update interpretation + Interpretation interpretation = catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), + clinical.getInterpretation().getId(), new InterpretationUpdateParams().setDescription("my new interpretation description"), + null, INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new interpretation description", interpretation.getDescription()); + assertEquals(2, interpretation.getVersion()); + + // Get clinical Analysis + clinical = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("my new description", clinical.getDescription()); + assertEquals(3, clinical.getVersion()); + assertEquals("my new interpretation description", clinical.getInterpretation().getDescription()); + assertEquals(2, clinical.getInterpretation().getVersion()); + } + @Test public void createRepeatedInterpretationPrimaryFindings() throws CatalogException { Individual individual = new Individual() From b62463fb62c864a6702752717ac878b8710abef4 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 28 May 2024 14:56:44 +0200 Subject: [PATCH 024/128] catalog: fix issue in installIndexes, #TASK-5964 --- .../org/opencb/opencga/catalog/managers/CatalogManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 2308b7ad123..ec8c3dcf2ad 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -301,7 +301,7 @@ public void installIndexes(String token) throws CatalogException { public void installIndexes(String organizationId, String token) throws CatalogException { JwtPayload payload = userManager.validateToken(token); - String userId = payload.getUserId(); + String userId = payload.getUserId(organizationId); if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin(); } From 7ae6a0fe8f7200aedb9789c9ed77c2ba89c171e5 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 28 May 2024 16:04:48 +0200 Subject: [PATCH 025/128] catalog: add new ADMIN permission for ClinicalAnalysis, #TASK-6288 - #TASK-5964 --- .../managers/ClinicalAnalysisManager.java | 26 ++++++++++++++++--- .../clinical/ClinicalAnalysisPermissions.java | 3 ++- .../core/models/study/StudyPermissions.java | 5 +++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 452748329d7..530a9427083 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -1393,15 +1393,35 @@ private OpenCGAResult update(String organizationId, Study stud } ClinicalAnalysisStudyConfiguration clinicalConfiguration = study.getInternal().getConfiguration().getClinical(); + // Get the clinical status that are CLOSED + Set closedStatus = new HashSet<>(); + for (ClinicalStatusValue clinicalStatusValue : clinicalConfiguration.getStatus().get(clinicalAnalysis.getType())) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + closedStatus.add(clinicalStatusValue.getId()); + } + } + + // If the current clinical analysis: + // - is locked + // - the user wants to update the locked status + // - the user wants to update the status to/from a closed status + boolean adminPermissionsChecked = false; + if (clinicalAnalysis.isLocked() || closedStatus.contains(clinicalAnalysis.getInternal().getStatus().getId()) + || updateParamsClone.getLocked() != null + || (updateParams.getStatus() != null && closedStatus.contains(updateParams.getStatus().getId()))) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.ADMIN); + adminPermissionsChecked = true; + } // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets - if (updateParamsClone.getAnnotationSets() != null) { + if (!adminPermissionsChecked && updateParamsClone.getAnnotationSets() != null) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets - if ((parameters.size() == 1 && !parameters.containsKey(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key())) - || parameters.size() > 1) { + if (!adminPermissionsChecked && ((parameters.size() == 1 + && !parameters.containsKey(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1)) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java index e0cdfadd0c0..724f4ae04a5 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java @@ -9,7 +9,8 @@ public enum ClinicalAnalysisPermissions { DELETE(Arrays.asList(VIEW, WRITE)), VIEW_ANNOTATIONS(Collections.singletonList(VIEW)), WRITE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, VIEW)), - DELETE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, VIEW)); + DELETE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, VIEW)), + ADMIN(Arrays.asList(VIEW, WRITE, DELETE, VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, DELETE_ANNOTATIONS)); private final List implicitPermissions; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java index b7dba9c36de..75f627af7f3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java @@ -109,7 +109,10 @@ public enum Permissions { WRITE_CLINICAL_ANNOTATIONS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, VIEW_CLINICAL_ANNOTATIONS), ClinicalAnalysisPermissions.WRITE_ANNOTATIONS.name(), CLINICAL_ANALYSIS), DELETE_CLINICAL_ANNOTATIONS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, VIEW_CLINICAL_ANNOTATIONS, WRITE_CLINICAL_ANNOTATIONS), - ClinicalAnalysisPermissions.DELETE_ANNOTATIONS.name(), CLINICAL_ANALYSIS); + ClinicalAnalysisPermissions.DELETE_ANNOTATIONS.name(), CLINICAL_ANALYSIS), + ADMIN_CLINICAL_ANALYSIS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, WRITE_CLINICAL_ANALYSIS, DELETE_CLINICAL_ANALYSIS, + VIEW_CLINICAL_ANNOTATIONS, WRITE_CLINICAL_ANNOTATIONS, DELETE_CLINICAL_ANNOTATIONS), + ClinicalAnalysisPermissions.ADMIN.name(), CLINICAL_ANALYSIS); private final static Map map; From 9f46a7cf2c3c95bfb6a10bd7d834e4b7c4d58b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 28 May 2024 17:20:02 +0100 Subject: [PATCH 026/128] storage: Fix tests from HadoopVariantStorageEngineSplitDataTest #TASK-6136 --- ...adoopVariantStorageEngineSplitDataTest.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java index e4f1f2e14b4..1401a7b5ad9 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java @@ -224,6 +224,10 @@ public void testMultiChromosomeSplitData() throws Exception { } private void failAtLoadingFile(String x, String file1, URI outputUri) throws Exception { + failAtLoadingFile(x, file1, outputUri, 1); + } + + private void failAtLoadingFile(String x, String file1, URI outputUri, int expectedRunningTasks) throws Exception { try { VariantStorageEngine engine = getMockedStorageEngine(new ObjectMap(VariantStorageOptions.STUDY.key(), STUDY_NAME)); engine.index(Collections.singletonList(getResourceUri(x + file1)), outputUri); @@ -236,11 +240,13 @@ private void failAtLoadingFile(String x, String file1, URI outputUri) throws Exc assertEquals(TaskMetadata.Status.NONE, fileMetadata.getIndexStatus()); List runningTasks = new ArrayList<>(); metadataManager.getRunningTasks(studyId).forEach(runningTasks::add); - assertEquals(1, runningTasks.size()); - assertEquals(TaskMetadata.Type.LOAD, runningTasks.get(0).getType()); - assertEquals(TaskMetadata.Status.RUNNING, runningTasks.get(0).currentStatus()); - assertEquals(Arrays.asList(fileMetadata.getId()), runningTasks.get(0).getFileIds()); + assertEquals(expectedRunningTasks, runningTasks.size()); + TaskMetadata taskMetadata = runningTasks.get(runningTasks.size() - 1); + assertEquals(TaskMetadata.Type.LOAD, taskMetadata.getType()); + assertEquals(TaskMetadata.Status.RUNNING, taskMetadata.currentStatus()); + assertEquals(Arrays.asList(fileMetadata.getId()), taskMetadata.getFileIds()); } catch (AssertionError error) { + error.addSuppressed(e); e.printStackTrace(); throw error; } @@ -594,7 +600,7 @@ public void testLoadMultiFileDataConcurrencyDeleteMany() throws Exception { String file2 = "1K.end.platinum-genomes-vcf-NA12878_S1.vcf.gz"; failAtLoadingFile(resourceDir, file1, outDir); - failAtLoadingFile(resourceDir, file2, outDir); + failAtLoadingFile(resourceDir, file2, outDir, 2); // try { // getMockedStorageEngine().index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir); // fail("Should have thrown an exception"); @@ -656,7 +662,7 @@ public void testLoadMultiFileDataConcurrencyFail() throws Exception { variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), true); variantStorageEngine.getOptions().put(VariantStorageOptions.RESUME.key(), true); - variantStorageEngine.index(Collections.singletonList(getResourceUri(file1)), outDir); + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir); } From a6c17898a357657a307de3c995dc4a55be7ac5f5 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 29 May 2024 10:29:42 +0200 Subject: [PATCH 027/128] catalog: test new ADMIN permission, #TASK-6288 #TASK-5964 --- .../managers/ClinicalAnalysisManagerTest.java | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index 29653b30db5..f2ad85c2740 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -36,6 +36,7 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult; import org.opencb.opencga.catalog.utils.Constants; @@ -61,9 +62,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.sample.SampleUpdateParams; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; +import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.models.study.configuration.ClinicalConsent; import org.opencb.opencga.core.models.study.configuration.*; import org.opencb.opencga.core.models.user.Account; @@ -1644,6 +1643,64 @@ public void updateClinicalAnalysisTest() throws CatalogException { assertEquals("URGENT", ca.getPriority().getId()); } + @Test + public void adminPermissionTest() throws CatalogException { + // Add ADMIN permissions to the user2 + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, + new StudyAclParams(StudyPermissions.Permissions.ADMIN_CLINICAL_ANALYSIS.name(), null), ParamUtils.AclAction.SET, ownerToken); + + DataResult dummyEnvironment = createDummyEnvironment(true, false); + + // Update ClinicalAnalysis with user1 + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description"), INCLUDE_RESULT, normalToken1).first(); + assertEquals("My description", clinicalAnalysis.getDescription()); + + // Update ClinicalAnalysis with user2 + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description 2"), INCLUDE_RESULT, normalToken2).first(); + assertEquals("My description 2", clinicalAnalysis.getDescription()); + + // Set status to CLOSED with user1 - FAIL + assertThrows(CatalogAuthorizationException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, normalToken1) + ); + + // Set status to CLOSED with user2 - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, normalToken2).first(); + assertEquals("CLOSED", clinicalAnalysis.getStatus().getId()); + assertTrue(clinicalAnalysis.isLocked()); + + // Unset status from CLOSED to other with user1 - FAIL + assertThrows(CatalogAuthorizationException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")), INCLUDE_RESULT, normalToken1) + ); + + // Unset status from CLOSED to other with user2 - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")), INCLUDE_RESULT, normalToken2).first(); + assertEquals("READY_FOR_INTERPRETATION", clinicalAnalysis.getStatus().getId()); + + // Set status to CLOSED with study admin - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, studyAdminToken1).first(); + assertEquals("CLOSED", clinicalAnalysis.getStatus().getId()); + + // Unset status from CLOSED to other with study admin - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")).setLocked(false), INCLUDE_RESULT, + studyAdminToken1).first(); + assertEquals("READY_FOR_INTERPRETATION", clinicalAnalysis.getStatus().getId()); + + // Update ClinicalAnalysis with user1 - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description 3"), INCLUDE_RESULT, normalToken1).first(); + assertEquals("My description 3", clinicalAnalysis.getDescription()); + } + @Test public void updateCustomStatusTest() throws CatalogException { Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); From f15978bc1fc7d8ce41779b53c8d5d5956003fb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 31 May 2024 11:30:39 +0100 Subject: [PATCH 028/128] storage: Avoid unnecesary sample metadata updates updating cohorts #TASK-5895 --- .../VariantStorageMetadataManager.java | 87 ++++++++----------- .../VariantStorageMetadataManagerTest.java | 34 ++++++++ 2 files changed, 68 insertions(+), 53 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index 89d86a63a89..a27f908173b 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -56,7 +56,6 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.BiPredicate; @@ -191,14 +190,7 @@ public ObjectMap getConfiguration() { public Lock lockGlobal(long lockDuration, long timeout, String lockName) throws StorageEngineException { - try { - return projectDBAdaptor.lockProject(lockDuration, timeout, lockName); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new StorageEngineException("Unable to lock the Project", e); - } catch (TimeoutException e) { - throw new StorageEngineException("Unable to lock the Project", e); - } + return projectDBAdaptor.lockProject(lockDuration, timeout, lockName); } public Lock lockStudy(int studyId) throws StorageEngineException { @@ -282,8 +274,7 @@ public StudyMetadata updateStudyMetadata(Object study, Upd throws StorageEngineException, E { int studyId = getStudyId(study); - Lock lock = lockStudy(studyId); - try { + try (Lock lock = lockStudy(studyId)) { StudyMetadata sm = getStudyMetadata(studyId); sm = updater.update(sm); @@ -291,8 +282,6 @@ public StudyMetadata updateStudyMetadata(Object study, Upd lock.checkLocked(); unsecureUpdateStudyMetadata(sm); return sm; - } finally { - lock.unlock(); } } @@ -557,16 +546,8 @@ public ProjectMetadata updateProjectMetadata(UpdateConsume public ProjectMetadata updateProjectMetadata(UpdateFunction function) throws StorageEngineException, E { Objects.requireNonNull(function); - Lock lock; - try { - lock = projectDBAdaptor.lockProject(lockDuration, lockTimeout); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new StorageEngineException("Unable to lock the Project", e); - } catch (TimeoutException e) { - throw new StorageEngineException("Unable to lock the Project", e); - } - try { + + try (Lock lock = projectDBAdaptor.lockProject(lockDuration, lockTimeout)) { ProjectMetadata projectMetadata = getProjectMetadata(); int countersHash = (projectMetadata == null ? Collections.emptyMap() : projectMetadata.getCounters()).hashCode(); @@ -579,8 +560,6 @@ public ProjectMetadata updateProjectMetadata(UpdateFunctio lock.checkLocked(); projectDBAdaptor.updateProjectMetadata(projectMetadata, updateCounters); return projectMetadata; - } finally { - lock.unlock(); } } @@ -673,16 +652,14 @@ public void unsecureUpdateFileMetadata(int studyId, FileMetadata file) { public FileMetadata updateFileMetadata(int studyId, int fileId, UpdateConsumer update) throws E, StorageEngineException { getFileName(studyId, fileId); // Check file exists - Lock lock = fileDBAdaptor.lock(studyId, fileId, lockDuration, lockTimeout); - try { + + try (Lock lock = fileDBAdaptor.lock(studyId, fileId, lockDuration, lockTimeout)) { FileMetadata fileMetadata = getFileMetadata(studyId, fileId); update.update(fileMetadata); lock.checkLocked(); unsecureUpdateFileMetadata(studyId, fileMetadata); fileIdIndexedCache.put(studyId, fileId, fileMetadata.isIndexed()); return fileMetadata; - } finally { - lock.unlock(); } } @@ -875,15 +852,13 @@ public void unsecureUpdateSampleMetadata(int studyId, SampleMetadata sample) { public SampleMetadata updateSampleMetadata(int studyId, int sampleId, UpdateConsumer consumer) throws E, StorageEngineException { getSampleName(studyId, sampleId); // Check sample exists - Lock lock = sampleDBAdaptor.lock(studyId, sampleId, lockDuration, lockTimeout); - try { + + try (Lock lock = sampleDBAdaptor.lock(studyId, sampleId, lockDuration, lockTimeout)) { SampleMetadata sample = getSampleMetadata(studyId, sampleId); sample = consumer.toFunction().update(sample); lock.checkLocked(); unsecureUpdateSampleMetadata(studyId, sample); return sample; - } finally { - lock.unlock(); } } @@ -1054,15 +1029,12 @@ public void unsecureUpdateCohortMetadata(int studyId, CohortMetadata cohort) { public CohortMetadata updateCohortMetadata(int studyId, int cohortId, UpdateConsumer update) throws E, StorageEngineException { getCohortName(studyId, cohortId); // Check cohort exists - Lock lock = cohortDBAdaptor.lock(studyId, cohortId, lockDuration, lockTimeout); - try { + try (Lock lock = cohortDBAdaptor.lock(studyId, cohortId, lockDuration, lockTimeout)) { CohortMetadata cohortMetadata = getCohortMetadata(studyId, cohortId); update.update(cohortMetadata); lock.checkLocked(); unsecureUpdateCohortMetadata(studyId, cohortMetadata); return cohortMetadata; - } finally { - lock.unlock(); } } @@ -1190,13 +1162,19 @@ private CohortMetadata updateCohortSamples(int studyId, String cohortName, Colle for (Integer sampleId : sampleIds) { Integer finalCohortId = cohortId; if (secondaryIndexCohort) { - updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.addSecondaryIndexCohort(finalCohortId); - }); + if (!getSampleMetadata(studyId, sampleId).getSecondaryIndexCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.addSecondaryIndexCohort(finalCohortId); + }); + } } else { - updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.addCohort(finalCohortId); - }); + if (!getSampleMetadata(studyId, sampleId).getCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.addCohort(finalCohortId); + }); + } } } @@ -1209,13 +1187,19 @@ private CohortMetadata updateCohortSamples(int studyId, String cohortName, Colle Integer finalCohortId = cohortId; if (!sampleIds.contains(sampleFromCohort)) { if (secondaryIndexCohort) { - updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { - sampleMetadata.getSecondaryIndexCohorts().remove(finalCohortId); - }); + if (getSampleMetadata(studyId, sampleFromCohort).getSecondaryIndexCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { + sampleMetadata.getSecondaryIndexCohorts().remove(finalCohortId); + }); + } } else { - updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { - sampleMetadata.getCohorts().remove(finalCohortId); - }); + if (getSampleMetadata(studyId, sampleFromCohort).getCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { + sampleMetadata.getCohorts().remove(finalCohortId); + }); + } } } } @@ -1326,15 +1310,12 @@ public void unsecureUpdateTask(int studyId, TaskMetadata task) throws StorageEng public TaskMetadata updateTask(int studyId, int taskId, UpdateConsumer consumer) throws E, StorageEngineException { getTask(studyId, taskId); // Check task exists - Lock lock = taskDBAdaptor.lock(studyId, taskId, lockDuration, lockTimeout); - try { + try (Lock lock = taskDBAdaptor.lock(studyId, taskId, lockDuration, lockTimeout)) { TaskMetadata task = getTask(studyId, taskId); consumer.update(task); lock.checkLocked(); unsecureUpdateTask(studyId, task); return task; - } finally { - lock.unlock(); } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java index 1b3311958f4..71ea72de3c0 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java @@ -4,12 +4,14 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageTest; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -103,4 +105,36 @@ public List getTasks(StudyMetadata study, List stat .map(TaskMetadata::getName) .collect(Collectors.toList()); } + + @Test + public void testAddSampleToCohort() throws Exception { + StudyMetadata study = metadataManager.createStudy("study"); + + metadataManager.registerCohort(study.getName(), "cohort1", Collections.emptyList()); + + int numSamples = 100; + List sampleIds = new ArrayList<>(numSamples); + for (int i = 0; i < numSamples; i++) { + sampleIds.add(metadataManager.registerSample(study.getId(), null, "sample_" + i)); + } + + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 10)); + VariantStorageMetadataManager metadataManager = Mockito.spy(this.metadataManager); + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.times(1)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + Mockito.reset(metadataManager); + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.never()).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.never()).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 12)); + Mockito.verify(metadataManager, Mockito.times(1)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + Mockito.reset(metadataManager); + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 6)); + Mockito.verify(metadataManager, Mockito.times(6)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + } + } \ No newline at end of file From e968bd29acea260b91e5e6b74262d4ea92660746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 31 May 2024 12:09:18 +0100 Subject: [PATCH 029/128] storage: Improve HBaseLockManager errors and warn messages. #TASK-5895 --- .../adaptors/ProjectMetadataAdaptor.java | 7 +- .../dummy/DummyProjectMetadataAdaptor.java | 7 +- .../hadoop/utils/HBaseLockManager.java | 274 ++++++++++-------- .../metadata/AbstractHBaseDBAdaptor.java | 9 - .../HBaseProjectMetadataDBAdaptor.java | 16 +- .../hadoop/utils/HBaseLockManagerTest.java | 42 ++- 6 files changed, 195 insertions(+), 160 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java index dc88a85d338..3045ee8f3cd 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java @@ -7,7 +7,6 @@ import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import java.io.IOException; -import java.util.concurrent.TimeoutException; /** * Created on 02/05/18. @@ -17,14 +16,12 @@ public interface ProjectMetadataAdaptor extends AutoCloseable { default Lock lockProject(long lockDuration, long timeout) - throws InterruptedException, TimeoutException, StorageEngineException { + throws StorageEngineException { return lockProject(lockDuration, timeout, null); } Lock lockProject(long lockDuration, long timeout, String lockName) - throws InterruptedException, TimeoutException, StorageEngineException; - - void unLockProject(long lockId) throws StorageEngineException; + throws StorageEngineException; DataResult getProjectMetadata(); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java index bed8d419666..d223180d9d1 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java @@ -16,7 +16,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; /** @@ -29,7 +28,7 @@ public class DummyProjectMetadataAdaptor implements ProjectMetadataAdaptor { private static Map counters = new HashMap<>(); @Override - public Lock lockProject(long lockDuration, long timeout, String lockName) throws InterruptedException, TimeoutException { + public Lock lockProject(long lockDuration, long timeout, String lockName) { return new Lock(0) { @Override public void unlock0() { @@ -43,10 +42,6 @@ public void refresh() { }; } - @Override - public void unLockProject(long lockId) { - } - @Override public synchronized DataResult getProjectMetadata() { final DataResult result = new DataResult<>(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java index 6fc7609b998..b1f827b2baf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.solr.common.StringUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.metadata.models.Lock; import org.slf4j.Logger; @@ -134,18 +133,17 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) // Minimum lock duration of 100ms lockDuration = Math.max(lockDuration, 100); - byte[] lockValue; - String readToken = ""; + LockToken lockToken; StopWatch stopWatch = new StopWatch(); stopWatch.start(); do { - lockValue = readLockValue(row, column); + lockToken = readLockToken(row, column); // If the lock is taken, wait - while (isLockTaken(lockValue)) { + while (lockToken.isTaken()) { Thread.sleep(100); - lockValue = readLockValue(row, column); + lockToken = readLockToken(row, column); //Check if the lock is still valid if (stopWatch.getTime() > timeout) { throw new TimeoutException("Unable to get the lock"); @@ -157,19 +155,19 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) } // Try to lock cell - if (tryToPutToken(token, lockDuration, row, column, lockValue, CURRENT)) { - readToken = parseValidLockToken(readLockValue(row, column)); + if (tryToPutToken(token, lockDuration, row, column, lockToken, CURRENT)) { + lockToken = readLockToken(row, column); } - // You win the lock if the first available lock is yours. - } while (!token.equals(readToken)); + // You win the lock if you manage to write your lock. + } while (!lockToken.equals(token)); - boolean prevTokenExpired = lockValue != null && lockValue.length > 0; + boolean prevTokenExpired = !lockToken.isEmpty() && lockToken.isExpired(); boolean slowQuery = stopWatch.getTime() > 60000; if (prevTokenExpired || slowQuery) { StringBuilder msg = new StringBuilder("Lock column '").append(Bytes.toStringBinary(column)).append("'"); if (prevTokenExpired) { - long expireDate = parseExpireDate(lockValue); + long expireDate = lockToken.getExpireDate(); msg.append(". Previous token expired ") .append(TimeUtils.durationToString(System.currentTimeMillis() - expireDate)) .append(" ago"); @@ -181,105 +179,65 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) logger.warn(msg.toString()); } - long tokenHash = token.hashCode(); - logger.debug("Won the lock with token " + token + " (" + tokenHash + ")"); - - long finalLockDuration = lockDuration; - return new Lock(threadPool, (int) (finalLockDuration / 4), tokenHash) { - @Override - public void unlock0() { - try { - HBaseLockManager.this.unlock(row, column, tokenHash); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + logger.debug("Won the lock with token " + token + " (" + token.hashCode() + ")"); - @Override - public synchronized void refresh() throws IOException { - HBaseLockManager.this.refresh(row, column, tokenHash, finalLockDuration); - } - }; + return new HBaseLock(lockDuration, token, row, column); } - /** - * Refreshes the lock. - * - * @param column Column to find the lock cell - * @param lockToken Lock token - * @param lockDuration Duration un milliseconds of the token. After this time the token is expired. - * @throws IOException if there is an error writing or reading from HBase. - */ - public void refresh(byte[] column, long lockToken, int lockDuration) throws IOException { - refresh(defaultRow, column, lockToken, lockDuration); - } - - /** * Refreshes the lock. * * @param row Row to find the lock cell * @param column Column to find the lock cell - * @param lockToken Lock token + * @param lockTokenHash Lock token * @param lockDuration Duration un milliseconds of the token. After this time the token is expired. * @throws IOException if there is an error writing or reading from HBase. */ - public void refresh(byte[] row, byte[] column, long lockToken, long lockDuration) throws IOException { + private void refresh(byte[] row, byte[] column, long lockTokenHash, long lockDuration) throws IOException { // Check token is valid - byte[] lockValue = readLockValue(row, column); - String currentLockToken = parseValidLockToken(lockValue); - if (currentLockToken == null || currentLockToken.hashCode() != lockToken) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentLockToken, lockValue); + LockToken currentLockToken = readLockToken(row, column); + if (currentLockToken.isEmpty() || currentLockToken.isExpired() || !currentLockToken.equals(lockTokenHash)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentLockToken); + } + if (currentLockToken.getRemainingTime() < lockDuration / 2) { + logger.warn("Refreshing lock with less than half of the duration remaining. Expected duration: {} Remaining time: {}ms", + lockDuration, + currentLockToken.getRemainingTime()); } - if (!tryToPutToken(currentLockToken, lockDuration, row, column, lockValue, REFRESH)) { + if (!tryToPutToken(currentLockToken.token, lockDuration, row, column, currentLockToken, REFRESH)) { // Error refreshing! - lockValue = readLockValue(row, column); - String newLockToken = parseValidLockToken(lockValue); + LockToken newLockToken = readLockToken(row, column); - logger.error("Current lock token:" + currentLockToken); - logger.error("New lock token: " + newLockToken); - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentLockToken, lockValue); + logger.error("Current lock token:" + currentLockToken.token); + logger.error("New lock token: " + newLockToken.token); + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentLockToken); } } - /** - * Releases the lock. - * - * @param column Column to find the lock cell - * @param lockToken Lock token - * @throws IOException if there is an error writing or reading from HBase. - * @throws IllegalLockStatusException if the lockToken does not match with the current lockToken - */ - public void unlock(byte[] column, long lockToken) throws IOException, IllegalLockStatusException { - unlock(defaultRow, column, lockToken); - } - /** * Releases the lock. * * @param row Row to find the lock cell * @param column Column to find the lock cell - * @param lockToken Lock token + * @param lockTokenHash Lock token * @throws IOException if there is an error writing or reading from HBase. * @throws IllegalLockStatusException if the lockToken does not match with the current lockToken */ - public void unlock(byte[] row, byte[] column, long lockToken) throws IOException, IllegalLockStatusException { - byte[] lockValue = readLockValue(row, column); + private void unlock(byte[] row, byte[] column, long lockTokenHash) throws IOException, IllegalLockStatusException { + LockToken currentToken = readLockToken(row, column); - String currentToken = parseValidLockToken(lockValue); - - if (currentToken == null || currentToken.hashCode() != lockToken) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentToken, lockValue); + if (currentToken.isEmpty() || currentToken.isExpired() || !currentToken.equals(lockTokenHash)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentToken); } - logger.debug("Unlock lock with token " + lockToken); - if (!clearLock(row, column, lockValue)) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentToken, lockValue); + logger.debug("Unlock lock with token " + lockTokenHash); + if (!clearLock(row, column, currentToken)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentToken); } } - private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[] qualifier, byte[] lockValue, String type) + private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[] qualifier, LockToken currentLock, String type) throws IOException { return hbaseManager.act(tableName, table -> { Put put = new Put(row) @@ -288,30 +246,31 @@ private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[ + token + LOCK_EXPIRING_DATE_SEPARATOR_STR + (System.currentTimeMillis() + lockDuration))); - return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockValue, put); + return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, currentLock.lockValue, put); }); } - private boolean clearLock(byte[] row, byte[] qualifier, byte[] lockValue) throws IOException { + private boolean clearLock(byte[] row, byte[] qualifier, LockToken lockToken) throws IOException { return hbaseManager.act(tableName, table -> { Put put = new Put(row) .addColumn(columnFamily, qualifier, Bytes.toBytes("")); - return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockValue, put); + return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockToken.lockValue, put); }); } /** - * Parse non-expired lock token. + * Parse lock token. * @param lockValue lock values - * @return Current lock token, if any + * @return Current lock token. */ - protected static String parseValidLockToken(byte[] lockValue) { + protected static LockToken parseLockToken(byte[] lockValue) { if (lockValue == null || lockValue.length == 0) { - return null; + return new LockToken(); } int idx1 = Bytes.indexOf(lockValue, LOCK_PREFIX_SEPARATOR_BYTE); int idx2 = Bytes.indexOf(lockValue, LOCK_EXPIRING_DATE_SEPARATOR_BYTE); + String type = Bytes.toString(lockValue, 0, idx1); String token = Bytes.toString(lockValue, idx1 + 1, idx2 - idx1 - 1); long expireDate; try { @@ -319,45 +278,82 @@ protected static String parseValidLockToken(byte[] lockValue) { } catch (NumberFormatException e) { // Deprecated token. Assume expired token if (Bytes.contains(lockValue, DEPRECATED_LOCK_SEPARATOR_BYTE)) { - return null; + return new LockToken(); } throw e; } + return new LockToken(lockValue, type, token, expireDate); + } + + protected static final class LockToken { + protected final byte[] lockValue; + protected final String type; + protected final String token; + protected final Long expireDate; + + private LockToken() { + this.lockValue = new byte[0]; + this.type = null; + this.token = null; + this.expireDate = null; + } + + private LockToken(byte[] lockValue, String type, String token, long expireDate) { + this.lockValue = lockValue; + this.type = type; + this.token = token; + this.expireDate = expireDate; + } + + /** + * A lock is taken if there is any lockValue, and + * the token has not expired. + * + * @return if the lock is taken + */ + public boolean isTaken() { + return token != null && !isExpired(); + } + + public boolean isExpired() { + return expireDate != null && expireDate < System.currentTimeMillis(); + } - if (isExpired(expireDate)) { - return null; - } else { + public boolean isEmpty() { + return token == null; + } + + public boolean equals(String token) { + return !isEmpty() && this.token.equals(token); + } + + public boolean equals(long tokenHash) { + return !isEmpty() && this.token.hashCode() == tokenHash; + } + + public byte[] getLockValue() { + return lockValue; + } + + public String getType() { + return type; + } + + public String getToken() { return token; } - } - protected static long parseExpireDate(byte[] lockValue) { - int idx2 = Bytes.indexOf(lockValue, LOCK_EXPIRING_DATE_SEPARATOR_BYTE); - try { - return Long.parseLong(Bytes.toString(lockValue, idx2 + 1)); - } catch (NumberFormatException e) { - // Deprecated token. Assume expired token - if (Bytes.contains(lockValue, DEPRECATED_LOCK_SEPARATOR_BYTE)) { - return -1; - } - throw e; + public Long getExpireDate() { + return expireDate; } - } - /** - * A lock is taken if there is any lockValue in the array, and - * the token has not expired. - * - * - * @param lockValue lock values - * @return if the lock is taken - */ - protected static boolean isLockTaken(byte[] lockValue) { - return parseValidLockToken(lockValue) != null; + public long getRemainingTime() { + return expireDate == null ? 0 : expireDate - System.currentTimeMillis(); + } } - private static boolean isExpired(long expireDate) { - return expireDate < System.currentTimeMillis(); + private LockToken readLockToken(byte[] row, byte[] qualifier) throws IOException { + return parseLockToken(readLockValue(row, qualifier)); } private byte[] readLockValue(byte[] row, byte[] qualifier) throws IOException { @@ -380,18 +376,22 @@ public IllegalLockStatusException(String s) { super(s); } - public static IllegalLockStatusException inconsistentLock(byte[] row, byte[] column, long lockToken, String currentLock, - byte[] lockValue) { - if (StringUtils.isEmpty(currentLock)) { - return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! " + private static IllegalLockStatusException inconsistentLock(byte[] row, byte[] column, long lockTokenHash, LockToken currentLock) { + if (currentLock.isEmpty()) { + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Empty lock. " + + "Row: '" + Bytes.toStringBinary(row) + "', " + + "column: '" + Bytes.toStringBinary(column) + "'. " + + "Lock: " + Bytes.toString(currentLock.lockValue) + "."); + } else if (currentLock.isExpired()) { + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Expired lock. " + "Row: '" + Bytes.toStringBinary(row) + "', " + "column: '" + Bytes.toStringBinary(column) + "'. " - + "Lock: " + Bytes.toString(lockValue) + "."); + + "Lock: " + Bytes.toString(currentLock.lockValue) + "."); } else { - return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! " + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Lock is taken. " + "Row: '" + Bytes.toStringBinary(row) + "', " + "column: '" + Bytes.toStringBinary(column) + "'. " - + lockToken + " != " + currentLock.hashCode() + " from " + Bytes.toString(lockValue)); + + lockTokenHash + " != " + currentLock.token.hashCode() + " from " + Bytes.toString(currentLock.lockValue)); } } } @@ -403,4 +403,38 @@ protected static ExecutorService buildThreadPool() { .build()); } + private final class HBaseLock extends Lock { + private final long lockDuration; + private final String token; + private final long tokenHash; + private final byte[] row; + private final byte[] column; + + private HBaseLock(long lockDuration, String token, byte[] row, byte[] column) { + super(HBaseLockManager.threadPool, (int) (lockDuration / 4), token.hashCode()); + this.lockDuration = lockDuration; + this.token = token; + this.tokenHash = token.hashCode(); + this.row = row; + this.column = column; + } + + @Override + public void unlock0() { + try { + synchronized (this) { + HBaseLockManager.this.unlock(row, column, tokenHash); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void refresh() throws IOException { + synchronized (this) { + HBaseLockManager.this.refresh(row, column, tokenHash, lockDuration); + } + } + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java index 3f6a23c5abf..fbdfdf920ec 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java @@ -279,14 +279,5 @@ protected Lock lockToken(byte[] rowKey, byte[] lockName, long lockDuration, long } } - protected void unLock(byte[] rowKey, byte[] lockName, long token) { - try { - this.lock.unlock(rowKey, lockName, token); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java index 85a1d0cec0c..6ddae38be65 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java @@ -49,20 +49,14 @@ public HBaseProjectMetadataDBAdaptor(HBaseManager hBaseManager, String metaTable @Override public Lock lockProject(long lockDuration, long timeout, String lockName) - throws InterruptedException, TimeoutException, StorageEngineException { + throws StorageEngineException { try { ensureTableExists(); return lock.lock(getProjectRowKey(), getLockColumn(lockName), lockDuration, timeout); - } catch (IOException e) { - throw new StorageEngineException("Error locking project in HBase", e); - } - } - - @Override - public void unLockProject(long lockId) throws StorageEngineException { - try { - lock.unlock(getProjectRowKey(), getLockColumn(), lockId); - } catch (IOException e) { + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new StorageEngineException("Unable to lock the Project", e); + } catch (IOException | TimeoutException e) { throw new StorageEngineException("Error locking project in HBase", e); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java index 0f421081ff6..5943a43519f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java @@ -214,23 +214,47 @@ public void testLockRefreshExpiredRefresh() throws Exception { @Test public void testGetCurrent() { long e = System.currentTimeMillis() + 1000; - String s; + HBaseLockManager.LockToken s; + + // null token + s = HBaseLockManager.parseLockToken(null); + assertTrue(s.isEmpty()); + assertFalse(s.isTaken()); + assertEquals(null, s.getType()); + assertArrayEquals(new byte[0], s.getLockValue()); + + // Empty token + s = HBaseLockManager.parseLockToken(Bytes.toBytes("")); + assertTrue(s.isEmpty()); + assertFalse(s.isTaken()); + assertEquals(null, s.getType()); + assertArrayEquals(new byte[0], s.getLockValue()); // Expired current token - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("CURRENT-abc:123")); - assertNull(s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("CURRENT-abc:123")); + assertFalse(s.isEmpty()); + assertEquals("CURRENT", s.getType()); // Valid current token - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("CURRENT-abc:" + e)); - assertEquals("abc", s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("CURRENT-abc:" + e)); + assertEquals("abc", s.token); + assertEquals("CURRENT", s.getType()); + assertFalse(s.isExpired()); + assertTrue(s.isTaken()); // Current expired, first refresh valid - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("REFRESH-abc:" + e)); - assertEquals("abc", s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("REFRESH-abc:" + e)); + assertEquals("abc", s.token); + assertEquals("REFRESH", s.getType()); + assertFalse(s.isExpired()); + assertTrue(s.isTaken()); // Expired refresh - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("REFRESH-abc:200")); - assertNull(s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("REFRESH-abc:200")); + assertEquals("abc", s.token); + assertEquals("REFRESH", s.getType()); + assertTrue(s.isExpired()); + assertFalse(s.isTaken()); } } \ No newline at end of file From dd1dcec6de1746730e5e5a102cc3289194a59cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 31 May 2024 12:11:32 +0100 Subject: [PATCH 030/128] storage: Increase metadata lock duration and timeout. #TASK-5895 --- .../opencga/storage/core/variant/VariantStorageOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java index caefbb5260e..bc1be055466 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java @@ -83,8 +83,8 @@ public enum VariantStorageOptions implements ConfigurationOption { INDEX_SEARCH("indexSearch", false), // Build secondary indexes using search engine. - METADATA_LOCK_DURATION("metadata.lock.duration", 5000), - METADATA_LOCK_TIMEOUT("metadata.lock.timeout", 60000), + METADATA_LOCK_DURATION("metadata.lock.duration", 60000), + METADATA_LOCK_TIMEOUT("metadata.lock.timeout", 600000), METADATA_LOAD_BATCH_SIZE("metadata.load.batchSize", 10), METADATA_LOAD_THREADS("metadata.load.numThreads", 4), From 1c7db628514e40d960c97fdea2b1a782e67b698a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 31 May 2024 14:48:15 +0100 Subject: [PATCH 031/128] storage: Improve testing SampleIndexAggregation of intergenic queries. #TASK-6136 --- .../index/sample/SampleIndexQueryParser.java | 90 ++++++++++++------- .../sample/SampleIndexQueryParserTest.java | 18 +++- .../variant/index/sample/SampleIndexTest.java | 44 ++++++--- 3 files changed, 109 insertions(+), 43 deletions(-) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java index 6137c5bd6c1..803fa75fea5 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java @@ -1216,18 +1216,7 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema CtBtFtCombinationIndexSchema.Filter ctBtTfFilter = schema.getCtBtTfIndex().getField().noOpFilter(); IndexFilter clinicalFilter = schema.getClinicalIndexSchema().noOpFilter(); - Boolean intergenic = null; - - ParsedVariantQuery.VariantQueryXref variantQueryXref = VariantQueryParser.parseXrefs(query); - if (!isValidParam(query, REGION)) { - if (!variantQueryXref.getGenes().isEmpty() - && variantQueryXref.getIds().isEmpty() - && variantQueryXref.getOtherXrefs().isEmpty() - && variantQueryXref.getVariants().isEmpty()) { - // If only filtering by genes, is not intergenic. - intergenic = false; - } - } + final Boolean intergenic = isIntergenicQuery(query); // BiotypeConsquenceTypeFlagCombination combination = BiotypeConsquenceTypeFlagCombination // .fromQuery(query, Arrays.asList(schema.getTranscriptFlagIndexSchema().getField().getConfiguration().getValues())); @@ -1237,18 +1226,10 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema boolean tfCovered = false; if (isValidParam(query, ANNOT_CONSEQUENCE_TYPE)) { - List soNames = query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key()); - soNames = soNames.stream() + List soNames = query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key()) + .stream() .map(ct -> ConsequenceTypeMappings.accessionToTerm.get(VariantQueryUtils.parseConsequenceType(ct))) .collect(Collectors.toList()); - if (!soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT) - && !soNames.contains(VariantAnnotationConstants.REGULATORY_REGION_VARIANT) - && !soNames.contains(VariantAnnotationConstants.TF_BINDING_SITE_VARIANT)) { - // All ct values but "intergenic_variant" and "regulatory_region_variant" are in genes (i.e. non-intergenic) - intergenic = false; - } else if (soNames.size() == 1 && soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT)) { - intergenic = true; - } // else, leave undefined : intergenic = null boolean ctFilterCoveredBySummary = false; boolean ctBtCombinationCoveredBySummary = false; if (SampleIndexSchema.CUSTOM_LOF.containsAll(soNames)) { @@ -1295,14 +1276,17 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema } } - // Do not use ctIndex if the CT filter is covered by the summary - // Use the ctIndex if: + // Do not use ctIndex for intergenic queries (intergenic == true) + // or queries that might return intergenic variants (intergenic == null) + // + // Use the ctIndex if any of: // - The CtFilter is not covered by the summary // - The query has the combination CT+BT , and it is not covered by the summary // - The query has the combination CT+TF - boolean useCtIndexFilter = !ctFilterCoveredBySummary - || (!ctBtCombinationCoveredBySummary && combination.isBiotype()) - || combination.isFlag(); + boolean useCtIndexFilter = + intergenic == Boolean.FALSE && (!ctFilterCoveredBySummary + || (!ctBtCombinationCoveredBySummary && combination.isBiotype()) + || combination.isFlag()); if (useCtIndexFilter) { ctCovered = completeIndex; consequenceTypeFilter = schema.getCtIndex().getField().buildFilter(new OpValue<>("=", soNames)); @@ -1317,8 +1301,6 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema } if (isValidParam(query, ANNOT_BIOTYPE)) { - // All biotype values are in genes (i.e. non-intergenic) - intergenic = false; boolean biotypeFilterCoveredBySummary = false; List biotypes = query.getAsStringList(VariantQueryParam.ANNOT_BIOTYPE.key()); if (BIOTYPE_SET.containsAll(biotypes)) { @@ -1350,8 +1332,6 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema List transcriptFlags = query.getAsStringList(ANNOT_TRANSCRIPT_FLAG.key()); tfFilter = schema.getTranscriptFlagIndexSchema().getField().buildFilter(new OpValue<>("=", transcriptFlags)); tfCovered = completeIndex & tfFilter.isExactFilter(); - // Transcript flags are in transcripts/genes. (i.e. non-intergenic) - intergenic = false; // TranscriptFlag filter is covered by index if (tfCovered) { if (!isValidParam(query, GENE) && simpleCombination(combination)) { @@ -1538,12 +1518,60 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema // If intergenic is undefined, or true, CT and BT filters can not be used. biotypeFilter = schema.getBiotypeIndex().getField().noOpFilter(); consequenceTypeFilter = schema.getCtIndex().getField().noOpFilter(); + if (!biotypeFilter.isNoOp()) { + throw new IllegalStateException("Unexpected BT filter for intergenic=" + intergenic); + } + if (!consequenceTypeFilter.isNoOp()) { + throw new IllegalStateException("Unexpected CT filter for intergenic=" + intergenic); + } } return new SampleAnnotationIndexQuery(new byte[]{annotationIndexMask, annotationIndex}, consequenceTypeFilter, biotypeFilter, tfFilter, ctBtTfFilter, clinicalFilter, populationFrequencyFilter); } + private Boolean isIntergenicQuery(Query query) { + ParsedVariantQuery.VariantQueryXref variantQueryXref = VariantQueryParser.parseXrefs(query); + if (!isValidParam(query, REGION)) { + if (!variantQueryXref.getGenes().isEmpty() + && variantQueryXref.getIds().isEmpty() + && variantQueryXref.getOtherXrefs().isEmpty() + && variantQueryXref.getVariants().isEmpty()) { + // If only filtering by genes, is not intergenic. + return false; + } + } + + if (isValidParam(query, ANNOT_BIOTYPE)) { + // All biotype values are in genes (i.e. non-intergenic) + return false; + } + if (isValidParam(query, ANNOT_BIOTYPE)) { + // All biotype values are in genes (i.e. non-intergenic) + return false; + } + if (isValidParam(query, ANNOT_TRANSCRIPT_FLAG)) { + // Transcript flags are in transcripts/genes. (i.e. non-intergenic) + return false; + } + if (isValidParam(query, ANNOT_CONSEQUENCE_TYPE)) { + List soNames = query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key()); + soNames = soNames.stream() + .map(ct -> ConsequenceTypeMappings.accessionToTerm.get(VariantQueryUtils.parseConsequenceType(ct))) + .collect(Collectors.toList()); + if (!soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT) + && !soNames.contains(VariantAnnotationConstants.REGULATORY_REGION_VARIANT) + && !soNames.contains(VariantAnnotationConstants.TF_BINDING_SITE_VARIANT)) { + // All ct values but "intergenic_variant" and "regulatory_region_variant" are in genes (i.e. non-intergenic) + return false; + } else if (soNames.size() == 1 && soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT)) { + return true; + } // else, leave undefined : intergenic = null + } + // Unable to determine if the query is intergenic or not. Return null for uncertain. + return null; + } + private boolean simpleCombination(BiotypeConsquenceTypeFlagCombination combination) { return combination.numParams() == 1; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java index 7ae4e36f3ec..212b0521b26 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java @@ -1384,7 +1384,10 @@ public void parseIntergenicTest() { checkIntergenic(true, new Query(ANNOT_CONSEQUENCE_TYPE.key(), "intergenic_variant")); checkIntergenic(null, new Query(ANNOT_CONSEQUENCE_TYPE.key(), "missense_variant,intergenic_variant")); checkIntergenic(null, new Query(ANNOT_CONSEQUENCE_TYPE.key(), "intergenic_variant,missense_variant")); - + checkIntergenic(null, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT)); + checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT) + .append(ANNOT_BIOTYPE.key(), "protein_coding")); + // Nonsense combination checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), "intergenic_variant").append(ANNOT_BIOTYPE.key(), "protein_coding")); } @@ -1570,6 +1573,19 @@ public void testCoveredQuery_ct() { parseAnnotationIndexQuery(query, true); assertTrue(query.isEmpty()); + query = new Query().append(ANNOT_CONSEQUENCE_TYPE.key(), String.join(OR, VariantAnnotationConstants.REGULATORY_REGION_VARIANT)); + parseAnnotationIndexQuery(query, true); + indexQuery = parseAnnotationIndexQuery(query, true); + assertTrue(indexQuery.getConsequenceTypeFilter().isNoOp()); + assertFalse(query.isEmpty()); // regulatory_region_variant can't be used for CT filter alone + + query = new Query().append(ANNOT_CONSEQUENCE_TYPE.key(), String.join(OR, VariantAnnotationConstants.REGULATORY_REGION_VARIANT)) + .append(ANNOT_BIOTYPE.key(), "protein_coding"); + indexQuery = parseAnnotationIndexQuery(query, true); + assertFalse(indexQuery.getConsequenceTypeFilter().isNoOp()); + assertFalse(indexQuery.getBiotypeFilter().isNoOp()); + assertTrue(query.isEmpty()); // regulatory_region_variant can be used together with biotype + query = new Query().append(ANNOT_CONSEQUENCE_TYPE.key(), String.join(OR, VariantAnnotationConstants.STOP_LOST)); parseAnnotationIndexQuery(query, false); indexQuery = parseAnnotationIndexQuery(query, false); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index e15cb6ae4bb..9ee362f8724 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -897,29 +897,55 @@ public void testAggregationCorrectnessCt() throws Exception { @Test public void testAggregationCorrectnessTFBS() throws Exception { - testAggregationCorrectness(TF_BINDING_SITE_VARIANT, true); + // Special scenario. This CT might include intergenic values, so can't be used alone + testAggregationCorrectness(new Query(ANNOT_BIOTYPE.key(), "protein_coding"), TF_BINDING_SITE_VARIANT); } @Test public void testAggregationCorrectnessRegulatoryRegionVariant() throws Exception { - testAggregationCorrectness(REGULATORY_REGION_VARIANT); + // Special scenario. This CT might include intergenic values, so can't be used alone + testAggregationCorrectness(new Query(ANNOT_BIOTYPE.key(), "protein_coding"), + REGULATORY_REGION_VARIANT); + } + + @Test + public void testAggregationByIntergenicQuery() throws Exception { + SampleIndexVariantAggregationExecutor executor = new SampleIndexVariantAggregationExecutor(metadataManager, sampleIndexDBAdaptor); + + Query baseQuery = new Query(STUDY.key(), STUDY_NAME_3) + .append(SAMPLE.key(), "NA12877"); + + assertFalse(executor.canUseThisExecutor(new Query(baseQuery) + .append(ANNOT_CONSEQUENCE_TYPE.key(), REGULATORY_REGION_VARIANT), new QueryOptions(QueryOptions.FACET, "consequenceType"))); + assertFalse(executor.canUseThisExecutor(new Query(baseQuery) + .append(ANNOT_CONSEQUENCE_TYPE.key(), TF_BINDING_SITE_VARIANT), new QueryOptions(QueryOptions.FACET, "consequenceType"))); + + assertTrue(executor.canUseThisExecutor(new Query(baseQuery) + .append(ANNOT_CONSEQUENCE_TYPE.key(), REGULATORY_REGION_VARIANT) + .append(ANNOT_BIOTYPE.key(), "protein_coding"), + new QueryOptions(QueryOptions.FACET, "consequenceType"))); + assertTrue(executor.canUseThisExecutor(new Query(baseQuery) + .append(ANNOT_CONSEQUENCE_TYPE.key(), TF_BINDING_SITE_VARIANT) + .append(ANNOT_BIOTYPE.key(), "protein_coding"), + new QueryOptions(QueryOptions.FACET, "consequenceType"))); } private void testAggregationCorrectness(String ct) throws Exception { - testAggregationCorrectness(ct, false); + testAggregationCorrectness(new Query(), ct); } - private void testAggregationCorrectness(String ct, boolean sampleIndexMightBeMoreAccurate) throws Exception { + private void testAggregationCorrectness(Query baseQuery, String ct) throws Exception { SampleIndexVariantAggregationExecutor executor = new SampleIndexVariantAggregationExecutor(metadataManager, sampleIndexDBAdaptor); - Query query = new Query(STUDY.key(), STUDY_NAME_3) + Query query = new Query(baseQuery) + .append(STUDY.key(), STUDY_NAME_3) .append(SAMPLE.key(), "NA12877") .append(ANNOT_CONSEQUENCE_TYPE.key(), ct); assertTrue(executor.canUseThisExecutor(query, new QueryOptions(QueryOptions.FACET, "consequenceType"))); AtomicInteger count = new AtomicInteger(0); sampleIndexDBAdaptor.iterator(new Query(query), new QueryOptions()).forEachRemaining(v -> count.incrementAndGet()); - FacetField facet = executor.aggregation(query, new QueryOptions(QueryOptions.FACET, "consequenceType")).first(); + FacetField facet = executor.aggregation(new Query(query), new QueryOptions(QueryOptions.FACET, "consequenceType")).first(); assertEquals(count.get(), facet.getCount()); FacetField.Bucket bucket = facet.getBuckets().stream().filter(b -> b.getValue().equals(ct)).findFirst().orElse(null); @@ -934,11 +960,7 @@ private void testAggregationCorrectness(String ct, boolean sampleIndexMightBeMor } } else { assertNotNull(msg, bucket); - if (sampleIndexMightBeMoreAccurate) { - assertThat(msg, count.get(), gte(bucket.getCount())); - } else { - assertEquals(msg, count.get(), bucket.getCount()); - } + assertEquals(msg, count.get(), bucket.getCount()); } } From f0e47fafe735c75ec34599a1ab7dcb9e027abafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 3 Jun 2024 10:26:37 +0200 Subject: [PATCH 032/128] analysis: use the FileUtils.copyFile and rename Tool to AnalysisTool, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java modified: opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java renamed: opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java -> opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java --- .../executors/DockerWrapperAnalysisExecutor.java | 10 ++++------ .../ExomiserWrapperAnalysisExecutor.java | 9 ++++----- .../org/opencb/opencga/core/config/Analysis.java | 6 +++--- .../core/config/{Tool.java => AnalysisTool.java} | 16 +++++++--------- 4 files changed, 18 insertions(+), 23 deletions(-) rename opencga-core/src/main/java/org/opencb/opencga/core/config/{Tool.java => AnalysisTool.java} (80%) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index d2bd9cf63dd..380e6d79bfe 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -8,7 +8,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; import org.opencb.opencga.core.common.GitRepositoryState; -import org.opencb.opencga.core.config.Tool; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.slf4j.Logger; @@ -20,8 +20,6 @@ import java.io.FileOutputStream; import java.util.*; -import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.EXOMISER_PREFIX; - public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { public final static String DOCKER_INPUT_PATH = "/data/input"; @@ -43,7 +41,7 @@ public String getDockerImageVersion() { } public String getDockerImageName(String toolKey) { - for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { + for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { if (entry.getKey().equalsIgnoreCase(toolKey)) { return entry.getValue().getDockerId().split(":")[0]; } @@ -52,7 +50,7 @@ public String getDockerImageName(String toolKey) { } public String getDockerImageVersion(String toolKey) { - for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { + for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { if (entry.getKey().equalsIgnoreCase(toolKey)) { if (entry.getValue().getDockerId().contains(":")) { return entry.getValue().getDockerId().split(":")[1]; @@ -69,7 +67,7 @@ protected String getToolResource(String toolKey, String resourceKey) throws Tool if (!getConfiguration().getAnalysis().getTools().containsKey(toolKey)) { throw new ToolException("Error getting tool " + toolKey + ": it does not exist in the configuration file"); } - Tool tool = getConfiguration().getAnalysis().getTools().get(toolKey); + AnalysisTool tool = getConfiguration().getAnalysis().getTools().get(toolKey); if (!tool.getResources().containsKey(resourceKey)) { throw new ToolException("Error getting resource " + resourceKey + " of tool " + toolKey + ": it does not exist in the" + " configuration file"); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 111c3a88d49..6521cb224f8 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; @@ -16,7 +15,6 @@ import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.FamilyManager; -import org.opencb.opencga.core.config.Tool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.family.Family; @@ -35,6 +33,7 @@ import java.nio.file.Paths; import java.util.*; +import static org.opencb.commons.utils.FileUtils.copyFile; import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.*; @ToolExecutor(id = ExomiserWrapperAnalysisExecutor.ID, @@ -155,7 +154,7 @@ public void run() throws ToolException { // Copy the analysis try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(), + copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(), getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser analysis file", e); @@ -164,7 +163,7 @@ public void run() throws ToolException { // Copy the application.properties and update data according to Exomiser version try { Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), target.toFile()); + copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), target.toFile()); Command cmd = new Command("sed -i \"s/" + HG38_DATA_VERSION_MARK + "/" + getHg38DataVersion() + "/g\" " + target); cmd.run(); cmd = new Command("sed -i \"s/" + PHENOTYPE_DATA_VERSION_MARK + "/" + getPhenotypeDataVersion() + "/g\" " + target); @@ -175,7 +174,7 @@ public void run() throws ToolException { // Copy the output options try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), + copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser output options file", e); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 53864a0037d..c0450569451 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -29,7 +29,7 @@ public class Analysis { private String resourceUrl; private String opencgaExtTools; - private Map tools; + private Map tools; private Execution execution; @@ -78,11 +78,11 @@ public Analysis setOpencgaExtTools(String opencgaExtTools) { return this; } - public Map getTools() { + public Map getTools() { return tools; } - public Analysis setTools(Map tools) { + public Analysis setTools(Map tools) { this.tools = tools; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java similarity index 80% rename from opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java rename to opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java index 11800116985..65aa7dc08be 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Tool.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java @@ -16,22 +16,20 @@ package org.opencb.opencga.core.config; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -public class Tool { +public class AnalysisTool { private String dockerId; private String params; private Map resources; - public Tool() { + public AnalysisTool() { resources = new HashMap<>(); } - public Tool(String dockerId, String params, Map resources) { + public AnalysisTool(String dockerId, String params, Map resources) { this.dockerId = dockerId; this.params = params; this.resources = resources; @@ -39,7 +37,7 @@ public Tool(String dockerId, String params, Map resources) { @Override public String toString() { - final StringBuilder sb = new StringBuilder("Tool{"); + final StringBuilder sb = new StringBuilder("AnalysisTool{"); sb.append("dockerId='").append(dockerId).append('\''); sb.append(", params='").append(params).append('\''); sb.append(", resources=").append(resources); @@ -51,7 +49,7 @@ public String getDockerId() { return dockerId; } - public Tool setDockerId(String dockerId) { + public AnalysisTool setDockerId(String dockerId) { this.dockerId = dockerId; return this; } @@ -60,7 +58,7 @@ public String getParams() { return params; } - public Tool setParams(String params) { + public AnalysisTool setParams(String params) { this.params = params; return this; } @@ -69,7 +67,7 @@ public Map getResources() { return resources; } - public Tool setResources(Map resources) { + public AnalysisTool setResources(Map resources) { this.resources = resources; return this; } From 9f2b06d543a52537c69829f6d7a372d694bed70e Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 4 Jun 2024 12:32:48 +0200 Subject: [PATCH 033/128] Prepare next release 2.12.6-SNAPSHOT --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 14 +++++++------- 21 files changed, 27 insertions(+), 27 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 01544f3538a..69a9b33089c 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index afbb2c15342..18071afb30b 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index f4971939bbb..46964bfae1f 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 700c1de69c7..cfdf162a973 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index bdae22a335a..ad02409329b 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index dab473a7362..00abcbf61d7 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 83d7becd861..4e304b5e34c 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index e206eec5c95..798bdd26d41 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 47262e2c647..b5d0d0fad60 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index 6ebbd4567c9..50c8105886a 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index bb9e72406d9..130716cf970 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 44301eb5b63..29e4ac40576 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index 85bd6a7da66..e4d5e3c8439 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index a15d1e80dbd..7831c67ec49 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 5a107fec8fc..f3792747d50 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index 0790a13a764..8b769184fa3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index a4e177c4403..1d3b4e5f795 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 9403f1ba8f9..6a725c2746e 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 3c31682fb6e..1f00aeda658 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index a1df570546a..953a68ee373 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 1bc6c3f8de1..aef3f1a0bc0 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.5 + 2.12.6-SNAPSHOT pom OpenCGA @@ -43,12 +43,12 @@ - 2.12.5 - 2.12.5 - 5.8.3 - 2.12.2 - 4.12.0 - 2.12.5 + 2.12.6_dev + 2.12.6_dev + 5.8.4-SNAPSHOT + 2.12.3-SNAPSHOT + 4.12.1-SNAPSHOT + 2.12.6-SNAPSHOT 0.2.0 2.11.4 From e2b001c45122ef9de0152c6314d4b99e6087794e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Wed, 5 Jun 2024 18:34:45 +0200 Subject: [PATCH 034/128] analysis: use a list of tools instead of a map in the configuration file and update code according to that, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java modified: opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java modified: opencga-catalog/src/test/resources/configuration-test.yml modified: opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java modified: opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java modified: opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java modified: opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java modified: opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java modified: opencga-core/src/main/resources/configuration.yml --- .../opencga/analysis/ConfigurationUtils.java | 38 + .../ExomiserInterpretationAnalysis.java | 5 +- .../DockerWrapperAnalysisExecutor.java | 48 +- .../exomiser/ExomiserWrapperAnalysis.java | 9 + .../ExomiserWrapperAnalysisExecutor.java | 14 +- .../ExomiserInterpretationAnalysisTest.java | 11 +- .../options/ClinicalCommandOptions.java | 6 +- .../options/VariantCommandOptions.java | 5 +- .../AnalysisClinicalCommandOptions.java | 2468 +++++++++-------- .../AnalysisVariantCommandOptions.java | 2426 ++++++++-------- .../src/test/resources/configuration-test.yml | 7 +- .../opencga/core/api/FieldConstants.java | 1 - .../opencb/opencga/core/config/Analysis.java | 8 +- .../opencga/core/config/AnalysisTool.java | 40 +- .../ExomiserInterpretationAnalysisParams.java | 4 +- .../clinical/ExomiserWrapperParams.java | 6 +- .../src/main/resources/configuration.yml | 7 +- 17 files changed, 2594 insertions(+), 2509 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java index bc6e1dc674b..c494f8db7ca 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java @@ -16,9 +16,13 @@ package org.opencb.opencga.analysis; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.commons.utils.FileUtils; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +32,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; public class ConfigurationUtils { private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class); + private ConfigurationUtils() { + throw new IllegalStateException("Utility class"); + } /** * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml. * @@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome) .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml")); } } + + public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException { + List tools = new ArrayList<>(); + for (AnalysisTool tool : configuration.getAnalysis().getTools()) { + if (tool.getId().equals(toolId)) { + tools.add(tool); + } + } + if (CollectionUtils.isEmpty(tools)) { + throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file"); + } + if (tools.size() == 1) { + return tools.get(0).getVersion(); + } + String defaultVersion = null; + for (AnalysisTool tool : tools) { + if (tool.isDefaultVersion()) { + if (!StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("More than one default version found for tool ID '" + toolId + "'"); + } else { + defaultVersion = tool.getVersion(); + } + } + } + if (StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true"); + } + return defaultVersion; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index b6dcd30fb26..0a2b4934c0c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -30,6 +30,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.clinical.InterpretationAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; @@ -121,8 +122,8 @@ protected void check() throws Exception { // Check exomiser version if (StringUtils.isEmpty(exomiserVersion)) { // Missing exomiser version use the default one - logger.warn("Missing exomiser version, using the default {}", FieldConstants.EXOMISER_DEFAULT_VERSION); - exomiserVersion = FieldConstants.EXOMISER_DEFAULT_VERSION; + exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); } // Update executor params with OpenCGA home and session ID diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index 380e6d79bfe..486571e8f36 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -28,11 +28,11 @@ public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor public static final String STDOUT_FILENAME = "stdout.txt"; public static final String STDERR_FILENAME = "stderr.txt"; - public String getDockerImageName() { + public String getDockerImageName() throws ToolException { return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[0]; } - public String getDockerImageVersion() { + public String getDockerImageVersion() throws ToolException { if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) { return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1]; } else { @@ -40,37 +40,35 @@ public String getDockerImageVersion() { } } - public String getDockerImageName(String toolKey) { - for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { - if (entry.getKey().equalsIgnoreCase(toolKey)) { - return entry.getValue().getDockerId().split(":")[0]; + protected AnalysisTool getAnalysisTool(String toolId, String version) throws ToolException { + for (AnalysisTool tool : getConfiguration().getAnalysis().getTools()) { + if (toolId.equals(tool.getId()) && version.equals(tool.getVersion())) { + return tool; } } - return null; + throw new ToolException("Missing analyis tool (ID = " + toolId + ", version = " + version + ") in configuration file"); } - public String getDockerImageVersion(String toolKey) { - for (Map.Entry entry : getConfiguration().getAnalysis().getTools().entrySet()) { - if (entry.getKey().equalsIgnoreCase(toolKey)) { - if (entry.getValue().getDockerId().contains(":")) { - return entry.getValue().getDockerId().split(":")[1]; - } else { - return null; - } - } + public String getDockerImageName(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + return tool.getDockerId().split(":")[0]; + } + + public String getDockerImageVersion(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + if (tool.getDockerId().contains(":")) { + return tool.getDockerId().split(":")[1]; + } else { + return null; } - return null; } - protected String getToolResource(String toolKey, String resourceKey) throws ToolException { + protected String getToolResource(String toolId, String version, String resourceKey) throws ToolException { // Get resources from the configuration file - if (!getConfiguration().getAnalysis().getTools().containsKey(toolKey)) { - throw new ToolException("Error getting tool " + toolKey + ": it does not exist in the configuration file"); - } - AnalysisTool tool = getConfiguration().getAnalysis().getTools().get(toolKey); + AnalysisTool tool = getAnalysisTool(toolId, version); if (!tool.getResources().containsKey(resourceKey)) { - throw new ToolException("Error getting resource " + resourceKey + " of tool " + toolKey + ": it does not exist in the" - + " configuration file"); + throw new ToolException("Error getting resource " + resourceKey + " of analysis tool (ID = " + toolId + ", version = " + + version + "): it does not exist in the configuration file"); } return tool.getResources().get(resourceKey); } @@ -118,7 +116,7 @@ protected Map appendMounts(List> inputFilen return mountMap; } - protected void appendCommand(String command, StringBuilder sb) { + protected void appendCommand(String command, StringBuilder sb) throws ToolException { // Docker image and version sb.append(getDockerImageName()); if (StringUtils.isNotEmpty(getDockerImageVersion())) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index 03fb92099e2..63630a0d64d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; @@ -50,6 +51,13 @@ protected void check() throws Exception { throw new ToolException("Missing study"); } + // Check exomiser version + if (StringUtils.isEmpty(analysisParams.getExomiserVersion())) { + // Missing exomiser version use the default one + String exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); + analysisParams.setExomiserVersion(exomiserVersion); + } } @Override @@ -60,6 +68,7 @@ protected void run() throws Exception { getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(study) .setSampleId(analysisParams.getSample()) + .setExomiserVersion(analysisParams.getExomiserVersion()) .execute(); }); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 6521cb224f8..14753c30e59 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -438,23 +438,23 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { } private String getHg38DataVersion() throws ToolException { - String resource = getToolResource(EXOMISER_PREFIX + exomiserVersion, HG38_RESOURCE_KEY); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, HG38_RESOURCE_KEY); return Paths.get(resource).getFileName().toString().split("_")[0]; } private String getPhenotypeDataVersion() throws ToolException { - String resource = getToolResource(EXOMISER_PREFIX + exomiserVersion, PHENOTYPE_RESOURCE_KEY); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); return Paths.get(resource).getFileName().toString().split("_")[0]; } @Override - public String getDockerImageName() { - return getDockerImageName(EXOMISER_PREFIX + exomiserVersion); + public String getDockerImageName() throws ToolException { + return getDockerImageName(ExomiserWrapperAnalysis.ID, exomiserVersion); } @Override - public String getDockerImageVersion() { - return getDockerImageVersion(EXOMISER_PREFIX + exomiserVersion); + public String getDockerImageVersion() throws ToolException { + return getDockerImageVersion(ExomiserWrapperAnalysis.ID, exomiserVersion); } public String getStudyId() { @@ -468,7 +468,7 @@ public ExomiserWrapperAnalysisExecutor setStudyId(String studyId) { private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws ToolException { String filename; - String resource = getToolResource(EXOMISER_PREFIX + exomiserVersion, resourceKey); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, resourceKey); if (resource.startsWith("file://")) { // Copy resouce try { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index 7ac1012241f..c24cc6ebe41 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -31,6 +31,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeThat; +import static org.junit.Assume.assumeTrue; @Category(MediumTests.class) public class ExomiserInterpretationAnalysisTest { @@ -70,7 +71,8 @@ public void testNormalization() throws NonStandardCompliantSampleField { @Test public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + String exomiserVersion = "13.1"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + exomiserVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_single")); @@ -87,7 +89,7 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) .setClinicalAnalysisId(clinicalTest.CA_ID2) - .setExomiserVersion("13.1.0"); + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); @@ -100,7 +102,8 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE @Test public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + String exomiserVersion = "13.1"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + exomiserVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_family")); @@ -114,7 +117,7 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) .setClinicalAnalysisId(clinicalTest.CA_ID3) - .setExomiserVersion("13.1.0"); + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java index c1074f62ee9..c2bac1cd1aa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java @@ -13,7 +13,6 @@ import org.opencb.opencga.analysis.clinical.tiering.CancerTieringInterpretationAnalysis; import org.opencb.opencga.analysis.clinical.tiering.TieringInterpretationAnalysis; import org.opencb.opencga.analysis.clinical.zetta.ZettaInterpretationAnalysis; -import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.app.cli.GeneralCliOptions; import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser; import org.opencb.opencga.core.api.ParamConstants; @@ -24,7 +23,8 @@ import static org.opencb.opencga.analysis.clinical.InterpretationAnalysis.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.*; -import static org.opencb.opencga.core.api.FieldConstants.*; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @Parameters(commandNames = {"clinical"}, commandDescription = "Clinical analysis commands") @@ -342,7 +342,7 @@ public class ExomiserInterpretationCommandOptions extends GeneralCliOptions.Stud public String clinicalAnalysis; @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) - public String exomiserVersion = EXOMISER_DEFAULT_VERSION; + public String exomiserVersion; @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", arity = 1) public String outdir; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index 4005d4a9df5..06a224bb838 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -69,7 +69,8 @@ import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSampleQueryCommandOptions.SAMPLE_QUERY_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexCommandOptions.SECONDARY_INDEX_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexDeleteCommandOptions.SECONDARY_INDEX_DELETE_COMMAND; -import static org.opencb.opencga.core.api.FieldConstants.*; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_SAMPLE_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.core.api.ParamConstants.*; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND_DESCRIPTION; @@ -1846,7 +1847,7 @@ public class ExomiserAnalysisCommandOptions { public String sample; @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) - public String exomiserVersion = EXOMISER_DEFAULT_VERSION; + public String exomiserVersion; @Parameter(names = {"-o", "--outdir"}, description = "Output directory.") public String outdir; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index 11a5501bd5f..43b5345b02b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -14,14 +14,14 @@ /* -* WARNING: AUTOGENERATED CODE -* -* This code was generated by a tool. -* -* Manual changes to this file may cause unexpected behavior in your application. -* Manual changes to this file will be overwritten if the code is regenerated. -* -*/ + * WARNING: AUTOGENERATED CODE + * + * This code was generated by a tool. + * + * Manual changes to this file may cause unexpected behavior in your application. + * Manual changes to this file will be overwritten if the code is regenerated. + * + */ /** * This class contains methods for the Analysis - Clinical command line. @@ -30,48 +30,48 @@ @Parameters(commandNames = {"clinical"}, commandDescription = "Analysis - Clinical commands") public class AnalysisClinicalCommandOptions { - public JCommander jCommander; - public CommonCommandOptions commonCommandOptions; - - public UpdateAclCommandOptions updateAclCommandOptions; - public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; - public UpdateClinicalConfigurationCommandOptions updateClinicalConfigurationCommandOptions; - public CreateCommandOptions createCommandOptions; - public DistinctCommandOptions distinctCommandOptions; - public DistinctInterpretationCommandOptions distinctInterpretationCommandOptions; - public SearchInterpretationCommandOptions searchInterpretationCommandOptions; - public InfoInterpretationCommandOptions infoInterpretationCommandOptions; - public RunInterpreterCancerTieringCommandOptions runInterpreterCancerTieringCommandOptions; - public RunInterpreterExomiserCommandOptions runInterpreterExomiserCommandOptions; - public RunInterpreterTeamCommandOptions runInterpreterTeamCommandOptions; - public RunInterpreterTieringCommandOptions runInterpreterTieringCommandOptions; - public RunInterpreterZettaCommandOptions runInterpreterZettaCommandOptions; - public LoadCommandOptions loadCommandOptions; - public AggregationStatsRgaCommandOptions aggregationStatsRgaCommandOptions; - public QueryRgaGeneCommandOptions queryRgaGeneCommandOptions; - public SummaryRgaGeneCommandOptions summaryRgaGeneCommandOptions; - public RunRgaIndexCommandOptions runRgaIndexCommandOptions; - public QueryRgaIndividualCommandOptions queryRgaIndividualCommandOptions; - public SummaryRgaIndividualCommandOptions summaryRgaIndividualCommandOptions; - public QueryRgaVariantCommandOptions queryRgaVariantCommandOptions; - public SummaryRgaVariantCommandOptions summaryRgaVariantCommandOptions; - public SearchCommandOptions searchCommandOptions; - public QueryVariantCommandOptions queryVariantCommandOptions; - public AclCommandOptions aclCommandOptions; - public DeleteCommandOptions deleteCommandOptions; - public UpdateCommandOptions updateCommandOptions; - public UpdateAnnotationSetsAnnotationsCommandOptions updateAnnotationSetsAnnotationsCommandOptions; - public InfoCommandOptions infoCommandOptions; - public CreateInterpretationCommandOptions createInterpretationCommandOptions; - public ClearInterpretationCommandOptions clearInterpretationCommandOptions; - public DeleteInterpretationCommandOptions deleteInterpretationCommandOptions; - public RevertInterpretationCommandOptions revertInterpretationCommandOptions; - public UpdateInterpretationCommandOptions updateInterpretationCommandOptions; - public UpdateReportCommandOptions updateReportCommandOptions; + public JCommander jCommander; + public CommonCommandOptions commonCommandOptions; + + public UpdateAclCommandOptions updateAclCommandOptions; + public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; + public UpdateClinicalConfigurationCommandOptions updateClinicalConfigurationCommandOptions; + public CreateCommandOptions createCommandOptions; + public DistinctCommandOptions distinctCommandOptions; + public DistinctInterpretationCommandOptions distinctInterpretationCommandOptions; + public SearchInterpretationCommandOptions searchInterpretationCommandOptions; + public InfoInterpretationCommandOptions infoInterpretationCommandOptions; + public RunInterpreterCancerTieringCommandOptions runInterpreterCancerTieringCommandOptions; + public RunInterpreterExomiserCommandOptions runInterpreterExomiserCommandOptions; + public RunInterpreterTeamCommandOptions runInterpreterTeamCommandOptions; + public RunInterpreterTieringCommandOptions runInterpreterTieringCommandOptions; + public RunInterpreterZettaCommandOptions runInterpreterZettaCommandOptions; + public LoadCommandOptions loadCommandOptions; + public AggregationStatsRgaCommandOptions aggregationStatsRgaCommandOptions; + public QueryRgaGeneCommandOptions queryRgaGeneCommandOptions; + public SummaryRgaGeneCommandOptions summaryRgaGeneCommandOptions; + public RunRgaIndexCommandOptions runRgaIndexCommandOptions; + public QueryRgaIndividualCommandOptions queryRgaIndividualCommandOptions; + public SummaryRgaIndividualCommandOptions summaryRgaIndividualCommandOptions; + public QueryRgaVariantCommandOptions queryRgaVariantCommandOptions; + public SummaryRgaVariantCommandOptions summaryRgaVariantCommandOptions; + public SearchCommandOptions searchCommandOptions; + public QueryVariantCommandOptions queryVariantCommandOptions; + public AclCommandOptions aclCommandOptions; + public DeleteCommandOptions deleteCommandOptions; + public UpdateCommandOptions updateCommandOptions; + public UpdateAnnotationSetsAnnotationsCommandOptions updateAnnotationSetsAnnotationsCommandOptions; + public InfoCommandOptions infoCommandOptions; + public CreateInterpretationCommandOptions createInterpretationCommandOptions; + public ClearInterpretationCommandOptions clearInterpretationCommandOptions; + public DeleteInterpretationCommandOptions deleteInterpretationCommandOptions; + public RevertInterpretationCommandOptions revertInterpretationCommandOptions; + public UpdateInterpretationCommandOptions updateInterpretationCommandOptions; + public UpdateReportCommandOptions updateReportCommandOptions; public AnalysisClinicalCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { - + this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.updateAclCommandOptions = new UpdateAclCommandOptions(); @@ -109,2267 +109,2269 @@ public AnalysisClinicalCommandOptions(CommonCommandOptions commonCommandOptions, this.revertInterpretationCommandOptions = new RevertInterpretationCommandOptions(); this.updateInterpretationCommandOptions = new UpdateInterpretationCommandOptions(); this.updateReportCommandOptions = new UpdateReportCommandOptions(); - + } - + @Parameters(commandNames = {"acl-update"}, commandDescription ="Update the set of permissions granted for the member") public class UpdateAclCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--members"}, description = "Comma separated list of user or group IDs", required = true, arity = 1) - public String members; - + public String members; + @Parameter(names = {"--action"}, description = "Action to be performed [ADD, SET, REMOVE or RESET].", required = true, arity = 1) - public String action = "ADD"; - + public String action = "ADD"; + @Parameter(names = {"--propagate"}, description = "Propagate permissions to related families, individuals, samples and files", required = false, help = true, arity = 0) - public boolean propagate = false; - + public boolean propagate = false; + @Parameter(names = {"--permissions"}, description = "The body web service permissions parameter", required = true, arity = 1) public String permissions; - + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + } @Parameters(commandNames = {"annotation-sets-load"}, commandDescription ="Load annotation sets from a TSV file") public class LoadAnnotationSetsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) - public String variableSetId; - + public String variableSetId; + @Parameter(names = {"--path"}, description = "Path where the TSV file is located in OpenCGA or where it should be located.", required = true, arity = 1) - public String path; - + public String path; + @Parameter(names = {"--parents"}, description = "Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated).", required = false, help = true, arity = 0) - public boolean parents = false; - + public boolean parents = false; + @Parameter(names = {"--annotation-set-id"}, description = "Annotation set id. If not provided, variableSetId will be used.", required = false, arity = 1) - public String annotationSetId; - + public String annotationSetId; + @Parameter(names = {"--content"}, description = "The body web service content parameter", required = false, arity = 1) public String content; - + } @Parameters(commandNames = {"clinical-configuration-update"}, commandDescription ="Update Clinical Analysis configuration.") public class UpdateClinicalConfigurationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @DynamicParameter(names = {"--interpretation-default-filter"}, description = "The body web service defaultFilter parameter. Use: --interpretation-default-filter key=value", required = false) public java.util.Map interpretationDefaultFilter = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"create"}, commandDescription ="Create a new clinical analysis") public class CreateCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--skip-create-default-interpretation"}, description = "Flag to skip creating and initialise an empty default primary interpretation (Id will be '{clinicalAnalysisId}.1'). This flag is only considered if no Interpretation object is passed.", required = false, help = true, arity = 0) - public boolean skipCreateDefaultInterpretation = false; - + public boolean skipCreateDefaultInterpretation = false; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = true, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--type"}, description = "Enum param allowed values: SINGLE, FAMILY, CANCER, COHORT, AUTOCOMPARATIVE", required = false, arity = 1) public String type; - + @Parameter(names = {"--disorder-id"}, description = "The body web service id parameter", required = false, arity = 1) public String disorderId; - + @Parameter(names = {"--proband-id"}, description = "The body web service id parameter", required = false, arity = 1) public String probandId; - + @Parameter(names = {"--family-id"}, description = "The body web service id parameter", required = false, arity = 1) public String familyId; - + @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) public Boolean panelLock; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--report-title"}, description = "Report title.", required = false, arity = 1) public String reportTitle; - + @Parameter(names = {"--report-overview"}, description = "Report overview.", required = false, arity = 1) public String reportOverview; - + @Parameter(names = {"--report-logo"}, description = "Report logo.", required = false, arity = 1) public String reportLogo; - + @Parameter(names = {"--report-signed-by"}, description = "Indicates who has signed the report.", required = false, arity = 1) public String reportSignedBy; - + @Parameter(names = {"--report-signature"}, description = "Report signature.", required = false, arity = 1) public String reportSignature; - + @Parameter(names = {"--report-date"}, description = "Report date.", required = false, arity = 1) public String reportDate; - + @Parameter(names = {"--request-id"}, description = "The body web service id parameter", required = false, arity = 1) public String requestId; - + @Parameter(names = {"--request-justification"}, description = "The body web service justification parameter", required = false, arity = 1) public String requestJustification; - + @Parameter(names = {"--request-date"}, description = "The body web service date parameter", required = false, arity = 1) public String requestDate; - + @DynamicParameter(names = {"--request-attributes"}, description = "The body web service attributes parameter. Use: --request-attributes key=value", required = false) public java.util.Map requestAttributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--responsible-id"}, description = "The body web service id parameter", required = false, arity = 1) public String responsibleId; - + @Parameter(names = {"--responsible-name"}, description = "The body web service name parameter", required = false, arity = 1) public String responsibleName; - + @Parameter(names = {"--responsible-email"}, description = "The body web service email parameter", required = false, arity = 1) public String responsibleEmail; - + @Parameter(names = {"--responsible-organization"}, description = "The body web service organization parameter", required = false, arity = 1) public String responsibleOrganization; - + @Parameter(names = {"--responsible-department"}, description = "The body web service department parameter", required = false, arity = 1) public String responsibleDepartment; - + @Parameter(names = {"--responsible-address"}, description = "The body web service address parameter", required = false, arity = 1) public String responsibleAddress; - + @Parameter(names = {"--responsible-city"}, description = "The body web service city parameter", required = false, arity = 1) public String responsibleCity; - + @Parameter(names = {"--responsible-postcode"}, description = "The body web service postcode parameter", required = false, arity = 1) public String responsiblePostcode; - + @Parameter(names = {"--interpretation-description"}, description = "The body web service description parameter", required = false, arity = 1) public String interpretationDescription; - + @Parameter(names = {"--interpretation-clinical-analysis-id"}, description = "The body web service clinicalAnalysisId parameter", required = false, arity = 1) public String interpretationClinicalAnalysisId; - + @Parameter(names = {"--interpretation-creation-date"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String interpretationCreationDate; - + @Parameter(names = {"--interpretation-modification-date"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String interpretationModificationDate; - + @Parameter(names = {"--interpretation-locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean interpretationLocked; - + @DynamicParameter(names = {"--interpretation-attributes"}, description = "The body web service attributes parameter. Use: --interpretation-attributes key=value", required = false) public java.util.Map interpretationAttributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--quality-control-summary"}, description = "Enum param allowed values: HIGH, MEDIUM, LOW, DISCARD, NEEDS_REVIEW, UNKNOWN", required = false, arity = 1) public String qualityControlSummary; - + @Parameter(names = {"--quality-control-comments"}, description = "The body web service comments parameter", required = false, arity = 1) public String qualityControlComments; - + @Parameter(names = {"--quality-control-files"}, description = "The body web service files parameter", required = false, arity = 1) public String qualityControlFiles; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--due-date"}, description = "The body web service dueDate parameter", required = false, arity = 1) public String dueDate; - + @Parameter(names = {"--priority-id"}, description = "The body web service id parameter", required = false, arity = 1) public String priorityId; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + } @Parameters(commandNames = {"distinct"}, commandDescription ="Clinical Analysis distinct method") public class DistinctCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Clinical Analysis UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--type"}, description = "Clinical Analysis type", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--disorder"}, description = "Clinical Analysis disorder. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String disorder; - + public String disorder; + @Parameter(names = {"--files"}, description = "Clinical Analysis files", required = false, arity = 1) - public String files; - + public String files; + @Parameter(names = {"--sample"}, description = "Sample associated to the proband or any member of a family", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--individual"}, description = "Proband or any member of a family", required = false, arity = 1) - public String individual; - + public String individual; + @Parameter(names = {"--proband"}, description = "Clinical Analysis proband", required = false, arity = 1) - public String proband; - + public String proband; + @Parameter(names = {"--proband-samples"}, description = "Clinical Analysis proband samples", required = false, arity = 1) - public String probandSamples; - + public String probandSamples; + @Parameter(names = {"--family"}, description = "Clinical Analysis family", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-members"}, description = "Clinical Analysis family members", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-member-samples"}, description = "Clinical Analysis family members samples", required = false, arity = 1) - public String familyMemberSamples; - + public String familyMemberSamples; + @Parameter(names = {"--panels"}, description = "Clinical Analysis panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--locked"}, description = "Locked Clinical Analyses", required = false, arity = 1) - public Boolean locked; - + public Boolean locked; + @Parameter(names = {"--analyst-id"}, description = "Clinical Analysis analyst id", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--priority"}, description = "Clinical Analysis priority", required = false, arity = 1) - public String priority; - + public String priority; + @Parameter(names = {"--flags"}, description = "Clinical Analysis flags", required = false, arity = 1) - public String flags; - + public String flags; + @Parameter(names = {"--creation-date", "--cd"}, description = "Clinical Analysis Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Clinical Analysis Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--due-date"}, description = "Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String dueDate; - + public String dueDate; + @Parameter(names = {"--quality-control-summary"}, description = "Clinical Analysis quality control summary", required = false, arity = 1) - public String qualityControlSummary; - + public String qualityControlSummary; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - + public String annotation; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + @Parameter(names = {"--field"}, description = "Comma separated list of fields for which to obtain the distinct values", required = true, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"interpretation-distinct"}, commandDescription ="Interpretation distinct method") public class DistinctInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) - public String clinicalAnalysisId; - + public String clinicalAnalysisId; + @Parameter(names = {"--analyst-id"}, description = "Analyst ID", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--method-name"}, description = "Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String methodName; - + public String methodName; + @Parameter(names = {"--panels"}, description = "Interpretation panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--primary-findings"}, description = "Interpretation primary findings", required = false, arity = 1) - public String primaryFindings; - + public String primaryFindings; + @Parameter(names = {"--secondary-findings"}, description = "Interpretation secondary findings", required = false, arity = 1) - public String secondaryFindings; - + public String secondaryFindings; + @Parameter(names = {"--creation-date", "--cd"}, description = "Interpretation Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Interpretation Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + @Parameter(names = {"--field"}, description = "Comma separated list of fields for which to obtain the distinct values", required = true, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"interpretation-search"}, commandDescription ="Search clinical interpretations") public class SearchInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--sort"}, description = "Sort the results", required = false, help = true, arity = 0) - public boolean sort = false; - + public boolean sort = false; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) - public String clinicalAnalysisId; - + public String clinicalAnalysisId; + @Parameter(names = {"--analyst-id"}, description = "Analyst ID", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--method-name"}, description = "Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String methodName; - + public String methodName; + @Parameter(names = {"--panels"}, description = "Interpretation panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--primary-findings"}, description = "Interpretation primary findings", required = false, arity = 1) - public String primaryFindings; - + public String primaryFindings; + @Parameter(names = {"--secondary-findings"}, description = "Interpretation secondary findings", required = false, arity = 1) - public String secondaryFindings; - + public String secondaryFindings; + @Parameter(names = {"--creation-date", "--cd"}, description = "Interpretation Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Interpretation Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + } @Parameters(commandNames = {"interpretation-info"}, commandDescription ="Clinical interpretation information") public class InfoInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--interpretations"}, description = "Comma separated list of clinical interpretation IDs up to a maximum of 100", required = true, arity = 1) - public String interpretations; - + public String interpretations; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--version"}, description = "Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not supported if multiple interpretation ids are provided.", required = false, arity = 1) - public String version; - + public String version; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + } @Parameters(commandNames = {"interpreter-cancer-tiering-run"}, commandDescription ="Run cancer tiering interpretation analysis") public class RunInterpreterCancerTieringCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--discarded-variants"}, description = "The body web service discardedVariants parameter", required = false, arity = 1) public String discardedVariants; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"interpreter-exomiser-run"}, commandDescription ="Run exomiser interpretation analysis") public class RunInterpreterExomiserCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - - @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - - @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - - @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - - @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated" + + " automatically if not provided.", arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", + arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", arity = 1) + public String jobTags; + + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = true, arity = 1) public String clinicalAnalysis; - - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", required = false, arity = 1) - public String exomiserVersion = "14.0.0"; - + + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", arity = 1) + public String exomiserVersion; + } @Parameters(commandNames = {"interpreter-team-run"}, commandDescription ="Run TEAM interpretation analysis") public class RunInterpreterTeamCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--panels"}, description = "The body web service panels parameter", required = false, arity = 1) public String panels; - + @Parameter(names = {"--family-segregation"}, description = "The body web service familySegregation parameter", required = false, arity = 1) public String familySegregation; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"interpreter-tiering-run"}, commandDescription ="Run tiering interpretation analysis") public class RunInterpreterTieringCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--panels"}, description = "The body web service panels parameter", required = false, arity = 1) public String panels; - + @Parameter(names = {"--penetrance"}, description = "The body web service penetrance parameter", required = false, arity = 1) public String penetrance; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"interpreter-zetta-run"}, commandDescription ="Run Zetta interpretation analysis") public class RunInterpreterZettaCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--type"}, description = "The body web service type parameter", required = false, arity = 1) public String type; - + @Parameter(names = {"--body_study"}, description = "The body web service study parameter", required = false, arity = 1) public String bodyStudy; - + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + @Parameter(names = {"--filter"}, description = "The body web service filter parameter", required = false, arity = 1) public String filter; - + @Parameter(names = {"--qual"}, description = "The body web service qual parameter", required = false, arity = 1) public String qual; - + @Parameter(names = {"--file-data"}, description = "The body web service fileData parameter", required = false, arity = 1) public String fileData; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--sample-data"}, description = "The body web service sampleData parameter", required = false, arity = 1) public String sampleData; - + @Parameter(names = {"--sample-annotation"}, description = "The body web service sampleAnnotation parameter", required = false, arity = 1) public String sampleAnnotation; - + @Parameter(names = {"--sample-metadata"}, description = "The body web service sampleMetadata parameter", required = false, arity = 1) public String sampleMetadata; - + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String cohortStatsRef; - + @Parameter(names = {"--cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String cohortStatsAlt; - + @Parameter(names = {"--cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String cohortStatsMaf; - + @Parameter(names = {"--cohort-stats-mgf"}, description = "The body web service cohortStatsMgf parameter", required = false, arity = 1) public String cohortStatsMgf; - + @Parameter(names = {"--cohort-stats-pass"}, description = "The body web service cohortStatsPass parameter", required = false, arity = 1) public String cohortStatsPass; - + @Parameter(names = {"--score"}, description = "The body web service score parameter", required = false, arity = 1) public String score; - + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--family-disorder"}, description = "The body web service familyDisorder parameter", required = false, arity = 1) public String familyDisorder; - + @Parameter(names = {"--family-segregation"}, description = "The body web service familySegregation parameter", required = false, arity = 1) public String familySegregation; - + @Parameter(names = {"--family-members"}, description = "The body web service familyMembers parameter", required = false, arity = 1) public String familyMembers; - + @Parameter(names = {"--family-proband"}, description = "The body web service familyProband parameter", required = false, arity = 1) public String familyProband; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String ct; - + @Parameter(names = {"--xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String xref; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String proteinSubstitution; - + @Parameter(names = {"--conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String conservation; - + @Parameter(names = {"--population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String populationFrequencyAlt; - + @Parameter(names = {"--population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String populationFrequencyRef; - + @Parameter(names = {"--population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String populationFrequencyMaf; - + @Parameter(names = {"--transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String transcriptFlag; - + @Parameter(names = {"--gene-trait-id"}, description = "The body web service geneTraitId parameter", required = false, arity = 1) public String geneTraitId; - + @Parameter(names = {"--go"}, description = "The body web service go parameter", required = false, arity = 1) public String go; - + @Parameter(names = {"--expression"}, description = "The body web service expression parameter", required = false, arity = 1) public String expression; - + @Parameter(names = {"--protein-keyword"}, description = "The body web service proteinKeyword parameter", required = false, arity = 1) public String proteinKeyword; - + @Parameter(names = {"--drug"}, description = "The body web service drug parameter", required = false, arity = 1) public String drug; - + @Parameter(names = {"--functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String functionalScore; - + @Parameter(names = {"--clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String clinical; - + @Parameter(names = {"--clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String clinicalSignificance; - + @Parameter(names = {"--clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean clinicalConfirmedStatus = false; - + @Parameter(names = {"--custom-annotation"}, description = "The body web service customAnnotation parameter", required = false, arity = 1) public String customAnnotation; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String panelModeOfInheritance; - + @Parameter(names = {"--panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String panelConfidence; - + @Parameter(names = {"--panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String panelRoleInCancer; - + @Parameter(names = {"--trait"}, description = "The body web service trait parameter", required = false, arity = 1) public String trait; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"load"}, commandDescription ="Load clinical analyses from a file") public class LoadCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + } @Parameters(commandNames = {"rga-aggregation-stats"}, commandDescription ="RGA aggregation stats") public class AggregationStatsRgaCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: clinicalSignificances;type. For nested fields use >>, e.g.: type>>clinicalSignificances;knockoutType", required = true, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"rga-gene-query"}, commandDescription ="Query gene RGA") public class QueryRgaGeneCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--include-individual"}, description = "Include only the comma separated list of individuals to the response", required = false, arity = 1) - public String includeIndividual; - + public String includeIndividual; + @Parameter(names = {"--skip-individual"}, description = "Number of individuals to skip", required = false, arity = 1) - public Integer skipIndividual; - + public Integer skipIndividual; + @Parameter(names = {"--limit-individual"}, description = "Limit number of individuals returned (default: 1000)", required = false, arity = 1) - public Integer limitIndividual; - + public Integer limitIndividual; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-gene-summary"}, commandDescription ="RGA gene summary stats") public class SummaryRgaGeneCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-index-run"}, commandDescription ="Generate Recessive Gene Analysis secondary index") public class RunRgaIndexCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--auxiliar-index"}, description = "Index auxiliar collection to improve performance assuming RGA is completely indexed.", required = false, help = true, arity = 0) - public boolean auxiliarIndex = false; - + public boolean auxiliarIndex = false; + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + } @Parameters(commandNames = {"rga-individual-query"}, commandDescription ="Query individual RGA") public class QueryRgaIndividualCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-individual-summary"}, commandDescription ="RGA individual summary stats") public class SummaryRgaIndividualCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-variant-query"}, commandDescription ="Query variant RGA") public class QueryRgaVariantCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--include-individual"}, description = "Include only the comma separated list of individuals to the response", required = false, arity = 1) - public String includeIndividual; - + public String includeIndividual; + @Parameter(names = {"--skip-individual"}, description = "Number of individuals to skip", required = false, arity = 1) - public Integer skipIndividual; - + public Integer skipIndividual; + @Parameter(names = {"--limit-individual"}, description = "Limit number of individuals returned (default: 1000)", required = false, arity = 1) - public Integer limitIndividual; - + public Integer limitIndividual; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-variant-summary"}, commandDescription ="RGA variant summary stats") public class SummaryRgaVariantCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"search"}, commandDescription ="Clinical analysis search.") public class SearchCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) - public boolean flattenAnnotations = false; - + public boolean flattenAnnotations = false; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Clinical Analysis UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--type"}, description = "Clinical Analysis type", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--disorder"}, description = "Clinical Analysis disorder. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String disorder; - + public String disorder; + @Parameter(names = {"--files"}, description = "Clinical Analysis files", required = false, arity = 1) - public String files; - + public String files; + @Parameter(names = {"--sample"}, description = "Sample associated to the proband or any member of a family", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--individual"}, description = "Proband or any member of a family", required = false, arity = 1) - public String individual; - + public String individual; + @Parameter(names = {"--proband"}, description = "Clinical Analysis proband", required = false, arity = 1) - public String proband; - + public String proband; + @Parameter(names = {"--proband-samples"}, description = "Clinical Analysis proband samples", required = false, arity = 1) - public String probandSamples; - + public String probandSamples; + @Parameter(names = {"--family"}, description = "Clinical Analysis family", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-members"}, description = "Clinical Analysis family members", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-member-samples"}, description = "Clinical Analysis family members samples", required = false, arity = 1) - public String familyMemberSamples; - + public String familyMemberSamples; + @Parameter(names = {"--panels"}, description = "Clinical Analysis panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--locked"}, description = "Locked Clinical Analyses", required = false, arity = 1) - public Boolean locked; - + public Boolean locked; + @Parameter(names = {"--analyst-id"}, description = "Clinical Analysis analyst id", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--priority"}, description = "Clinical Analysis priority", required = false, arity = 1) - public String priority; - + public String priority; + @Parameter(names = {"--flags"}, description = "Clinical Analysis flags", required = false, arity = 1) - public String flags; - + public String flags; + @Parameter(names = {"--creation-date", "--cd"}, description = "Clinical Analysis Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Clinical Analysis Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--due-date"}, description = "Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String dueDate; - + public String dueDate; + @Parameter(names = {"--quality-control-summary"}, description = "Clinical Analysis quality control summary", required = false, arity = 1) - public String qualityControlSummary; - + public String qualityControlSummary; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - + public String annotation; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + } @Parameters(commandNames = {"variant-query"}, commandDescription ="Fetch clinical variants") public class QueryVariantCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--approximate-count"}, description = "Get an approximate count, instead of an exact total count. Reduces execution time", required = false, help = true, arity = 0) - public boolean approximateCount = false; - + public boolean approximateCount = false; + @Parameter(names = {"--approximate-count-sampling-size"}, description = "Sampling size to get the approximate count. Larger values increase accuracy but also increase execution time", required = false, arity = 1) - public Integer approximateCountSamplingSize; - + public Integer approximateCountSamplingSize; + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--include-interpretation"}, description = "Interpretation ID to include the fields related to this interpretation", required = false, arity = 1) - public String includeInterpretation; - + public String includeInterpretation; + @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--qual"}, description = "Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4", required = false, arity = 1) - public String qual; - + public String qual; + @Parameter(names = {"--file-data"}, description = "Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file is specified, will use all files from 'file' filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP", required = false, arity = 1) - public String fileData; - + public String fileData; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--sample-data"}, description = "Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will use all samples from 'sample' or 'genotype' filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10", required = false, arity = 1) - public String sampleData; - + public String sampleData; + @Parameter(names = {"--sample-annotation"}, description = "Selects some samples using metadata information from Catalog. e.g. age>20;phenotype=hpo:123,hpo:456;name=smith", required = false, arity = 1) - public String sampleAnnotation; - + public String sampleAnnotation; + @Parameter(names = {"--cohort"}, description = "Select variants with calculated stats for the selected cohorts", required = false, arity = 1) - public String cohort; - + public String cohort; + @Parameter(names = {"--cohort-stats-ref"}, description = "Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsRef; - + public String cohortStatsRef; + @Parameter(names = {"--cohort-stats-alt"}, description = "Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsAlt; - + public String cohortStatsAlt; + @Parameter(names = {"--cohort-stats-maf"}, description = "Minor Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMaf; - + public String cohortStatsMaf; + @Parameter(names = {"--cohort-stats-mgf"}, description = "Minor Genotype Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMgf; - + public String cohortStatsMgf; + @Parameter(names = {"--cohort-stats-pass"}, description = "Filter PASS frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8", required = false, arity = 1) - public String cohortStatsPass; - + public String cohortStatsPass; + @Parameter(names = {"--missing-alleles"}, description = "Number of missing alleles: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingAlleles; - + public String missingAlleles; + @Parameter(names = {"--missing-genotypes"}, description = "Number of missing genotypes: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingGenotypes; - + public String missingGenotypes; + @Parameter(names = {"--score"}, description = "Filter by variant score: [{study:}]{score}[<|>|<=|>=]{number}", required = false, arity = 1) - public String score; - + public String score; + @Parameter(names = {"--family"}, description = "Filter variants where any of the samples from the given family contains the variant (HET or HOM_ALT)", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-disorder"}, description = "Specify the disorder to use for the family segregation", required = false, arity = 1) - public String familyDisorder; - + public String familyDisorder; + @Parameter(names = {"--family-segregation"}, description = "Filter by segregation mode from a given family. Accepted values: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String familySegregation; - + public String familySegregation; + @Parameter(names = {"--family-members"}, description = "Sub set of the members of a given family", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-proband"}, description = "Specify the proband child to use for the family segregation", required = false, arity = 1) - public String familyProband; - + public String familyProband; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) - public String xref; - + public String xref; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--protein-substitution"}, description = "Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant", required = false, arity = 1) - public String proteinSubstitution; - + public String proteinSubstitution; + @Parameter(names = {"--conservation"}, description = "Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1", required = false, arity = 1) - public String conservation; - + public String conservation; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--population-frequency-ref"}, description = "Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyRef; - + public String populationFrequencyRef; + @Parameter(names = {"--population-frequency-maf"}, description = "Population minor allele frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyMaf; - + public String populationFrequencyMaf; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--gene-trait-id"}, description = "List of gene trait association id. e.g. 'umls:C0007222' , 'OMIM:269600'", required = false, arity = 1) - public String geneTraitId; - + public String geneTraitId; + @Parameter(names = {"--go"}, description = "List of GO (Gene Ontology) terms. e.g. 'GO:0002020'", required = false, arity = 1) - public String go; - + public String go; + @Parameter(names = {"--expression"}, description = "List of tissues of interest. e.g. 'lung'", required = false, arity = 1) - public String expression; - + public String expression; + @Parameter(names = {"--protein-keyword"}, description = "List of Uniprot protein variant annotation keywords", required = false, arity = 1) - public String proteinKeyword; - + public String proteinKeyword; + @Parameter(names = {"--drug"}, description = "List of drug names", required = false, arity = 1) - public String drug; - + public String drug; + @Parameter(names = {"--functional-score"}, description = "Functional score: {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , cadd_raw<=0.3", required = false, arity = 1) - public String functionalScore; - + public String functionalScore; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--custom-annotation"}, description = "Custom annotation: {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}", required = false, arity = 1) - public String customAnnotation; - + public String customAnnotation; + @Parameter(names = {"--panel"}, description = "Filter by genes from the given disease panel", required = false, arity = 1) - public String panel; - + public String panel; + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "Filter genes from specific panels that match certain mode of inheritance. Accepted values : [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String panelModeOfInheritance; - + public String panelModeOfInheritance; + @Parameter(names = {"--panel-confidence"}, description = "Filter genes from specific panels that match certain confidence. Accepted values : [ high, medium, low, rejected ]", required = false, arity = 1) - public String panelConfidence; - + public String panelConfidence; + @Parameter(names = {"--panel-role-in-cancer"}, description = "Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, tumorSuppressorGene, fusion ]", required = false, arity = 1) - public String panelRoleInCancer; - + public String panelRoleInCancer; + @Parameter(names = {"--panel-feature-type"}, description = "Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]", required = false, arity = 1) - public String panelFeatureType; - + public String panelFeatureType; + @Parameter(names = {"--panel-intersection"}, description = "Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel.", required = false, help = true, arity = 0) - public boolean panelIntersection = false; - + public boolean panelIntersection = false; + @Parameter(names = {"--trait"}, description = "List of traits, based on ClinVar, HPO, COSMIC, i.e.: IDs, histologies, descriptions,...", required = false, arity = 1) - public String trait; - + public String trait; + } @Parameters(commandNames = {"acl"}, commandDescription ="Returns the acl of the clinical analyses. If member is provided, it will only return the acl for the member.") public class AclCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) - public String clinicalAnalyses; - + public String clinicalAnalyses; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--member"}, description = "User or group ID", required = false, arity = 1) - public String member; - + public String member; + @Parameter(names = {"--silent"}, description = "Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason", required = false, help = true, arity = 0) - public boolean silent = false; - + public boolean silent = false; + } @Parameters(commandNames = {"delete"}, commandDescription ="Delete clinical analyses") public class DeleteCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--force"}, description = "Force deletion if the ClinicalAnalysis contains interpretations or is locked", required = false, help = true, arity = 0) - public boolean force = false; - + public boolean force = false; + @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) - public String clinicalAnalyses; - + public String clinicalAnalyses; + } @Parameters(commandNames = {"update"}, commandDescription ="Update clinical analysis attributes") public class UpdateCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs", required = true, arity = 1) - public String clinicalAnalyses; - + public String clinicalAnalyses; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--analysts-action"}, description = "Action to be performed if the array of analysts is being updated.", required = false, arity = 1) - public String analystsAction = "ADD"; - + public String analystsAction = "ADD"; + @Parameter(names = {"--annotation-sets-action"}, description = "Action to be performed if the array of annotationSets is being updated.", required = false, arity = 1) - public String annotationSetsAction = "ADD"; - + public String annotationSetsAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--type"}, description = "Enum param allowed values: SINGLE, FAMILY, CANCER, COHORT, AUTOCOMPARATIVE", required = false, arity = 1) public String type; - + @Parameter(names = {"--disorder-id"}, description = "The body web service id parameter", required = false, arity = 1) public String disorderId; - + @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) public Boolean panelLock; - + @Parameter(names = {"--proband-id"}, description = "The body web service id parameter", required = false, arity = 1) public String probandId; - + @Parameter(names = {"--family-id"}, description = "The body web service id parameter", required = false, arity = 1) public String familyId; - + @Parameter(names = {"--locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean locked; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--report-title"}, description = "Report title.", required = false, arity = 1) public String reportTitle; - + @Parameter(names = {"--report-overview"}, description = "Report overview.", required = false, arity = 1) public String reportOverview; - + @Parameter(names = {"--report-logo"}, description = "Report logo.", required = false, arity = 1) public String reportLogo; - + @Parameter(names = {"--report-signed-by"}, description = "Indicates who has signed the report.", required = false, arity = 1) public String reportSignedBy; - + @Parameter(names = {"--report-signature"}, description = "Report signature.", required = false, arity = 1) public String reportSignature; - + @Parameter(names = {"--report-date"}, description = "Report date.", required = false, arity = 1) public String reportDate; - + @Parameter(names = {"--request-id"}, description = "The body web service id parameter", required = false, arity = 1) public String requestId; - + @Parameter(names = {"--request-justification"}, description = "The body web service justification parameter", required = false, arity = 1) public String requestJustification; - + @Parameter(names = {"--request-date"}, description = "The body web service date parameter", required = false, arity = 1) public String requestDate; - + @DynamicParameter(names = {"--request-attributes"}, description = "The body web service attributes parameter. Use: --request-attributes key=value", required = false) public java.util.Map requestAttributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--responsible-id"}, description = "The body web service id parameter", required = false, arity = 1) public String responsibleId; - + @Parameter(names = {"--responsible-name"}, description = "The body web service name parameter", required = false, arity = 1) public String responsibleName; - + @Parameter(names = {"--responsible-email"}, description = "The body web service email parameter", required = false, arity = 1) public String responsibleEmail; - + @Parameter(names = {"--responsible-organization"}, description = "The body web service organization parameter", required = false, arity = 1) public String responsibleOrganization; - + @Parameter(names = {"--responsible-department"}, description = "The body web service department parameter", required = false, arity = 1) public String responsibleDepartment; - + @Parameter(names = {"--responsible-address"}, description = "The body web service address parameter", required = false, arity = 1) public String responsibleAddress; - + @Parameter(names = {"--responsible-city"}, description = "The body web service city parameter", required = false, arity = 1) public String responsibleCity; - + @Parameter(names = {"--responsible-postcode"}, description = "The body web service postcode parameter", required = false, arity = 1) public String responsiblePostcode; - + @Parameter(names = {"--quality-control-summary"}, description = "Enum param allowed values: HIGH, MEDIUM, LOW, DISCARD, NEEDS_REVIEW, UNKNOWN", required = false, arity = 1) public String qualityControlSummary; - + @Parameter(names = {"--quality-control-comments"}, description = "The body web service comments parameter", required = false, arity = 1) public String qualityControlComments; - + @Parameter(names = {"--quality-control-files"}, description = "The body web service files parameter", required = false, arity = 1) public String qualityControlFiles; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--due-date"}, description = "The body web service dueDate parameter", required = false, arity = 1) public String dueDate; - + @Parameter(names = {"--priority-id"}, description = "The body web service id parameter", required = false, arity = 1) public String priorityId; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + } @Parameters(commandNames = {"annotation-sets-annotations-update"}, commandDescription ="Update annotations from an annotationSet") public class UpdateAnnotationSetsAnnotationsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) - public String annotationSet; - + public String annotationSet; + @Parameter(names = {"--action"}, description = "Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any.", required = false, arity = 1) - public String action = "ADD"; - + public String action = "ADD"; + } @Parameters(commandNames = {"info"}, commandDescription ="Clinical analysis info") public class InfoCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) - public boolean flattenAnnotations = false; - + public boolean flattenAnnotations = false; + @Parameter(names = {"--clinical-analysis"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + } @Parameters(commandNames = {"interpretation-create"}, commandDescription ="Create a new Interpretation") public class CreateInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study id", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--set-as"}, description = "Set interpretation as", required = false, arity = 1) - public String setAs = "SECONDARY"; - + public String setAs = "SECONDARY"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--clinical-analysis-id"}, description = "The body web service clinicalAnalysisId parameter", required = false, arity = 1) public String clinicalAnalysisId; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--method-name"}, description = "The body web service name parameter", required = false, arity = 1) public String methodName; - + @Parameter(names = {"--method-version"}, description = "The body web service version parameter", required = false, arity = 1) public String methodVersion; - + @Parameter(names = {"--method-commit"}, description = "The body web service commit parameter", required = false, arity = 1) public String methodCommit; - + @Parameter(names = {"--locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean locked; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"interpretation-clear"}, commandDescription ="Clear the fields of the main interpretation of the Clinical Analysis") public class ClearInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--interpretations"}, description = "Interpretation IDs of the Clinical Analysis", required = true, arity = 1) - public String interpretations; - + public String interpretations; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + } @Parameters(commandNames = {"interpretation-delete"}, commandDescription ="Delete interpretation") public class DeleteInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--interpretations"}, description = "Interpretation IDs of the Clinical Analysis", required = true, arity = 1) - public String interpretations; - + public String interpretations; + @Parameter(names = {"--set-as-primary"}, description = "Interpretation id to set as primary from the list of secondaries in case of deleting the actual primary one", required = false, arity = 1) - public String setAsPrimary; - + public String setAsPrimary; + } @Parameters(commandNames = {"interpretation-revert"}, commandDescription ="Revert to a previous interpretation version") public class RevertInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--interpretation"}, description = "Interpretation ID", required = true, arity = 1) - public String interpretation; - + public String interpretation; + @Parameter(names = {"--version"}, description = "Version to revert to", required = true, arity = 1) - public Integer version; - + public Integer version; + } @Parameters(commandNames = {"interpretation-update"}, commandDescription ="Update interpretation fields") public class UpdateInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--set-as"}, description = "Set interpretation as", required = false, arity = 1) - public String setAs; - + public String setAs; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--interpretation"}, description = "Interpretation ID", required = true, arity = 1) - public String interpretation; - + public String interpretation; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--method-name"}, description = "The body web service name parameter", required = false, arity = 1) public String methodName; - + @Parameter(names = {"--method-version"}, description = "The body web service version parameter", required = false, arity = 1) public String methodVersion; - + @Parameter(names = {"--method-commit"}, description = "The body web service commit parameter", required = false, arity = 1) public String methodCommit; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + @Parameter(names = {"--locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean locked; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"report-update"}, commandDescription ="Update clinical analysis report") public class UpdateReportCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--supporting-evidences-action"}, description = "Action to be performed if the array of supporting evidences is being updated.", required = false, arity = 1) - public String supportingEvidencesAction = "ADD"; - + public String supportingEvidencesAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--title"}, description = "Report title.", required = false, arity = 1) public String title; - + @Parameter(names = {"--overview"}, description = "Report overview.", required = false, arity = 1) public String overview; - + @Parameter(names = {"--discussion-author"}, description = "The body web service author parameter", required = false, arity = 1) public String discussionAuthor; - + @Parameter(names = {"--discussion-date"}, description = "The body web service date parameter", required = false, arity = 1) public String discussionDate; - + @Parameter(names = {"--discussion-text"}, description = "The body web service text parameter", required = false, arity = 1) public String discussionText; - + @Parameter(names = {"--logo"}, description = "Report logo.", required = false, arity = 1) public String logo; - + @Parameter(names = {"--signed-by"}, description = "Indicates who has signed the report.", required = false, arity = 1) public String signedBy; - + @Parameter(names = {"--signature"}, description = "Report signature.", required = false, arity = 1) public String signature; - + @Parameter(names = {"--date"}, description = "Report date.", required = false, arity = 1) public String date; - + } } \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index f3d490e4fbf..a1dba3af944 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -1,27 +1,21 @@ package org.opencb.opencga.app.cli.main.options; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.DynamicParameter; -import com.beust.jcommander.ParametersDelegate; +import com.beust.jcommander.*; import java.util.HashMap; -import java.util.Map; -import java.util.List; -import static org.opencb.opencga.app.cli.GeneralCliOptions.*; +import static org.opencb.opencga.app.cli.GeneralCliOptions.CommonCommandOptions; /* -* WARNING: AUTOGENERATED CODE -* -* This code was generated by a tool. -* -* Manual changes to this file may cause unexpected behavior in your application. -* Manual changes to this file will be overwritten if the code is regenerated. -* -*/ + * WARNING: AUTOGENERATED CODE + * + * This code was generated by a tool. + * + * Manual changes to this file may cause unexpected behavior in your application. + * Manual changes to this file will be overwritten if the code is regenerated. + * + */ /** * This class contains methods for the Analysis - Variant command line. @@ -30,52 +24,52 @@ @Parameters(commandNames = {"variant"}, commandDescription = "Analysis - Variant commands") public class AnalysisVariantCommandOptions { - public JCommander jCommander; - public CommonCommandOptions commonCommandOptions; - - public AggregationStatsCommandOptions aggregationStatsCommandOptions; - public MetadataAnnotationCommandOptions metadataAnnotationCommandOptions; - public QueryAnnotationCommandOptions queryAnnotationCommandOptions; - public RunCircosCommandOptions runCircosCommandOptions; - public DeleteCohortStatsCommandOptions deleteCohortStatsCommandOptions; - public InfoCohortStatsCommandOptions infoCohortStatsCommandOptions; - public RunCohortStatsCommandOptions runCohortStatsCommandOptions; - public RunExomiserCommandOptions runExomiserCommandOptions; - public RunExportCommandOptions runExportCommandOptions; - public GenotypesFamilyCommandOptions genotypesFamilyCommandOptions; - public RunFamilyQcCommandOptions runFamilyQcCommandOptions; - public DeleteFileCommandOptions deleteFileCommandOptions; - public RunGatkCommandOptions runGatkCommandOptions; - public RunGenomePlotCommandOptions runGenomePlotCommandOptions; - public RunGwasCommandOptions runGwasCommandOptions; - public RunHrDetectCommandOptions runHrDetectCommandOptions; - public RunIndexCommandOptions runIndexCommandOptions; - public RunIndividualQcCommandOptions runIndividualQcCommandOptions; - public RunInferredSexCommandOptions runInferredSexCommandOptions; - public QueryKnockoutGeneCommandOptions queryKnockoutGeneCommandOptions; - public QueryKnockoutIndividualCommandOptions queryKnockoutIndividualCommandOptions; - public RunKnockoutCommandOptions runKnockoutCommandOptions; - public RunMendelianErrorCommandOptions runMendelianErrorCommandOptions; - public MetadataCommandOptions metadataCommandOptions; - public QueryMutationalSignatureCommandOptions queryMutationalSignatureCommandOptions; - public RunMutationalSignatureCommandOptions runMutationalSignatureCommandOptions; - public RunPlinkCommandOptions runPlinkCommandOptions; - public QueryCommandOptions queryCommandOptions; - public RunRelatednessCommandOptions runRelatednessCommandOptions; - public RunRvtestsCommandOptions runRvtestsCommandOptions; - public AggregationStatsSampleCommandOptions aggregationStatsSampleCommandOptions; - public RunSampleEligibilityCommandOptions runSampleEligibilityCommandOptions; - public RunSampleQcCommandOptions runSampleQcCommandOptions; - public QuerySampleCommandOptions querySampleCommandOptions; - public RunSampleCommandOptions runSampleCommandOptions; - public QuerySampleStatsCommandOptions querySampleStatsCommandOptions; - public RunSampleStatsCommandOptions runSampleStatsCommandOptions; - public RunStatsExportCommandOptions runStatsExportCommandOptions; - public RunStatsCommandOptions runStatsCommandOptions; + public JCommander jCommander; + public CommonCommandOptions commonCommandOptions; + + public AggregationStatsCommandOptions aggregationStatsCommandOptions; + public MetadataAnnotationCommandOptions metadataAnnotationCommandOptions; + public QueryAnnotationCommandOptions queryAnnotationCommandOptions; + public RunCircosCommandOptions runCircosCommandOptions; + public DeleteCohortStatsCommandOptions deleteCohortStatsCommandOptions; + public InfoCohortStatsCommandOptions infoCohortStatsCommandOptions; + public RunCohortStatsCommandOptions runCohortStatsCommandOptions; + public RunExomiserCommandOptions runExomiserCommandOptions; + public RunExportCommandOptions runExportCommandOptions; + public GenotypesFamilyCommandOptions genotypesFamilyCommandOptions; + public RunFamilyQcCommandOptions runFamilyQcCommandOptions; + public DeleteFileCommandOptions deleteFileCommandOptions; + public RunGatkCommandOptions runGatkCommandOptions; + public RunGenomePlotCommandOptions runGenomePlotCommandOptions; + public RunGwasCommandOptions runGwasCommandOptions; + public RunHrDetectCommandOptions runHrDetectCommandOptions; + public RunIndexCommandOptions runIndexCommandOptions; + public RunIndividualQcCommandOptions runIndividualQcCommandOptions; + public RunInferredSexCommandOptions runInferredSexCommandOptions; + public QueryKnockoutGeneCommandOptions queryKnockoutGeneCommandOptions; + public QueryKnockoutIndividualCommandOptions queryKnockoutIndividualCommandOptions; + public RunKnockoutCommandOptions runKnockoutCommandOptions; + public RunMendelianErrorCommandOptions runMendelianErrorCommandOptions; + public MetadataCommandOptions metadataCommandOptions; + public QueryMutationalSignatureCommandOptions queryMutationalSignatureCommandOptions; + public RunMutationalSignatureCommandOptions runMutationalSignatureCommandOptions; + public RunPlinkCommandOptions runPlinkCommandOptions; + public QueryCommandOptions queryCommandOptions; + public RunRelatednessCommandOptions runRelatednessCommandOptions; + public RunRvtestsCommandOptions runRvtestsCommandOptions; + public AggregationStatsSampleCommandOptions aggregationStatsSampleCommandOptions; + public RunSampleEligibilityCommandOptions runSampleEligibilityCommandOptions; + public RunSampleQcCommandOptions runSampleQcCommandOptions; + public QuerySampleCommandOptions querySampleCommandOptions; + public RunSampleCommandOptions runSampleCommandOptions; + public QuerySampleStatsCommandOptions querySampleStatsCommandOptions; + public RunSampleStatsCommandOptions runSampleStatsCommandOptions; + public RunStatsExportCommandOptions runStatsExportCommandOptions; + public RunStatsCommandOptions runStatsCommandOptions; public AnalysisVariantCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { - + this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); @@ -117,2500 +111,2504 @@ public AnalysisVariantCommandOptions(CommonCommandOptions commonCommandOptions, this.runSampleStatsCommandOptions = new RunSampleStatsCommandOptions(); this.runStatsExportCommandOptions = new RunStatsExportCommandOptions(); this.runStatsCommandOptions = new RunStatsCommandOptions(); - + } - + @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Calculate and fetch aggregation stats") public class AggregationStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--cohort"}, description = "Select variants with calculated stats for the selected cohorts", required = false, arity = 1) - public String cohort; - + public String cohort; + @Parameter(names = {"--cohort-stats-ref"}, description = "Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsRef; - + public String cohortStatsRef; + @Parameter(names = {"--cohort-stats-alt"}, description = "Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsAlt; - + public String cohortStatsAlt; + @Parameter(names = {"--cohort-stats-maf"}, description = "Minor Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMaf; - + public String cohortStatsMaf; + @Parameter(names = {"--cohort-stats-mgf"}, description = "Minor Genotype Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMgf; - + public String cohortStatsMgf; + @Parameter(names = {"--cohort-stats-pass"}, description = "Filter PASS frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8", required = false, arity = 1) - public String cohortStatsPass; - + public String cohortStatsPass; + @Parameter(names = {"--missing-alleles"}, description = "Number of missing alleles: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingAlleles; - + public String missingAlleles; + @Parameter(names = {"--missing-genotypes"}, description = "Number of missing genotypes: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingGenotypes; - + public String missingGenotypes; + @Parameter(names = {"--score"}, description = "Filter by variant score: [{study:}]{score}[<|>|<=|>=]{number}", required = false, arity = 1) - public String score; - + public String score; + @Parameter(names = {"--annotation-exists"}, description = "Return only annotated variants", required = false, arity = 1) - public Boolean annotationExists; - + public Boolean annotationExists; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) - public String xref; - + public String xref; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--protein-substitution"}, description = "Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant", required = false, arity = 1) - public String proteinSubstitution; - + public String proteinSubstitution; + @Parameter(names = {"--conservation"}, description = "Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1", required = false, arity = 1) - public String conservation; - + public String conservation; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--population-frequency-ref"}, description = "Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyRef; - + public String populationFrequencyRef; + @Parameter(names = {"--population-frequency-maf"}, description = "Population minor allele frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyMaf; - + public String populationFrequencyMaf; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--gene-trait-id"}, description = "List of gene trait association id. e.g. 'umls:C0007222' , 'OMIM:269600'", required = false, arity = 1) - public String geneTraitId; - + public String geneTraitId; + @Parameter(names = {"--go"}, description = "List of GO (Gene Ontology) terms. e.g. 'GO:0002020'", required = false, arity = 1) - public String go; - + public String go; + @Parameter(names = {"--expression"}, description = "List of tissues of interest. e.g. 'lung'", required = false, arity = 1) - public String expression; - + public String expression; + @Parameter(names = {"--protein-keyword"}, description = "List of Uniprot protein variant annotation keywords", required = false, arity = 1) - public String proteinKeyword; - + public String proteinKeyword; + @Parameter(names = {"--drug"}, description = "List of drug names", required = false, arity = 1) - public String drug; - + public String drug; + @Parameter(names = {"--functional-score"}, description = "Functional score: {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , cadd_raw<=0.3", required = false, arity = 1) - public String functionalScore; - + public String functionalScore; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--custom-annotation"}, description = "Custom annotation: {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}", required = false, arity = 1) - public String customAnnotation; - + public String customAnnotation; + @Parameter(names = {"--trait"}, description = "List of traits, based on ClinVar, HPO, COSMIC, i.e.: IDs, histologies, descriptions,...", required = false, arity = 1) - public String trait; - + public String trait; + @Parameter(names = {"--field"}, description = "List of facet fields separated by semicolons, e.g.: studies;type. For nested faceted fields use >>, e.g.: chromosome>>type;percentile(gerp)", required = false, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"annotation-metadata"}, commandDescription ="Read variant annotations metadata from any saved versions") public class MetadataAnnotationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--annotation-id"}, description = "Annotation identifier", required = false, arity = 1) - public String annotationId; - + public String annotationId; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + } @Parameters(commandNames = {"annotation-query"}, commandDescription ="Query variant annotations from any saved versions") public class QueryAnnotationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--annotation-id"}, description = "Annotation identifier", required = false, arity = 1) - public String annotationId; - + public String annotationId; + } @Parameters(commandNames = {"circos-run"}, commandDescription ="Generate a Circos plot for a given sample.") public class RunCircosCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--title"}, description = "The body web service title parameter", required = false, arity = 1) public String title; - + @Parameter(names = {"--density"}, description = "The body web service density parameter", required = false, arity = 1) public String density; - + @DynamicParameter(names = {"--query"}, description = "The body web service query parameter. Use: --query key=value", required = false) public java.util.Map query = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"cohort-stats-delete"}, commandDescription ="Delete cohort variant stats from a cohort.") public class DeleteCohortStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--cohort"}, description = "Cohort ID or UUID", required = false, arity = 1) - public String cohort; - + public String cohort; + } @Parameters(commandNames = {"cohort-stats-info"}, commandDescription ="Read cohort variant stats from list of cohorts.") public class InfoCohortStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--cohort"}, description = "Comma separated list of cohort IDs or UUIDs up to a maximum of 100", required = true, arity = 1) - public String cohort; - + public String cohort; + } @Parameters(commandNames = {"cohort-stats-run"}, commandDescription ="Compute cohort variant stats for the selected list of samples.") public class RunCohortStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--samples"}, description = "The body web service samples parameter", required = false, arity = 1) public String samples; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--sample-annotation"}, description = "The body web service sampleAnnotation parameter", required = false, arity = 1) public String sampleAnnotation; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } - @Parameters(commandNames = {"exomiser-run"}, commandDescription ="The Exomiser is a Java program that finds potential disease-causing variants from whole-exome or whole-genome sequencing data.") + @Parameters(commandNames = {"exomiser-run"}, commandDescription ="The Exomiser is a Java program that finds potential disease-causing" + + " variants from whole-exome or whole-genome sequencing data.") public class RunExomiserCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will" + + " ignore all the other parameters.", arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - - @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - - @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - - @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - - @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - - @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - - @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) + + @Parameter(names = {"--study", "-s"}, description = "study", required = true, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated" + + " automatically if not provided.", arity = 1) + public String jobId; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", + arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-description"}, description = "Job description", arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-tags"}, description = "Job tags", arity = 1) + public String jobTags; + + @Parameter(names = {"--sample"}, description = "Sample ID.", arity = 1) public String sample; - - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", required = false, arity = 1) - public String exomiserVersion = "14.0.0"; - - @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) + + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", arity = 1) + public String exomiserVersion; + + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", arity = 1) public String outdir; - + } @Parameters(commandNames = {"export-run"}, commandDescription ="Filter and export variants from the variant storage to a file") public class RunExportCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--type"}, description = "The body web service type parameter", required = false, arity = 1) public String type; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String panelModeOfInheritance; - + @Parameter(names = {"--panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String panelConfidence; - + @Parameter(names = {"--panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String panelRoleInCancer; - + @Parameter(names = {"--panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean panelIntersection = false; - + @Parameter(names = {"--panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String panelFeatureType; - + @Parameter(names = {"--cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String cohortStatsRef; - + @Parameter(names = {"--cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String cohortStatsAlt; - + @Parameter(names = {"--cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String cohortStatsMaf; - + @Parameter(names = {"--ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String ct; - + @Parameter(names = {"--xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String xref; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String proteinSubstitution; - + @Parameter(names = {"--conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String conservation; - + @Parameter(names = {"--population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String populationFrequencyMaf; - + @Parameter(names = {"--population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String populationFrequencyAlt; - + @Parameter(names = {"--population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String populationFrequencyRef; - + @Parameter(names = {"--transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String transcriptFlag; - + @Parameter(names = {"--functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String functionalScore; - + @Parameter(names = {"--clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String clinical; - + @Parameter(names = {"--clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String clinicalSignificance; - + @Parameter(names = {"--clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean clinicalConfirmedStatus = false; - + @Parameter(names = {"--body_project"}, description = "The body web service project parameter", required = false, arity = 1) public String bodyProject; - + @Parameter(names = {"--body_study"}, description = "The body web service study parameter", required = false, arity = 1) public String bodyStudy; - + @Parameter(names = {"--saved-filter"}, description = "The body web service savedFilter parameter", required = false, arity = 1) public String savedFilter; - + @Parameter(names = {"--chromosome"}, description = "The body web service chromosome parameter", required = false, arity = 1) public String chromosome; - + @Parameter(names = {"--reference"}, description = "The body web service reference parameter", required = false, arity = 1) public String reference; - + @Parameter(names = {"--alternate"}, description = "The body web service alternate parameter", required = false, arity = 1) public String alternate; - + @Parameter(names = {"--release"}, description = "The body web service release parameter", required = false, arity = 1) public String release; - + @Parameter(names = {"--include-study"}, description = "The body web service includeStudy parameter", required = false, arity = 1) public String includeStudy; - + @Parameter(names = {"--include-sample"}, description = "The body web service includeSample parameter", required = false, arity = 1) public String includeSample; - + @Parameter(names = {"--include-file"}, description = "The body web service includeFile parameter", required = false, arity = 1) public String includeFile; - + @Parameter(names = {"--include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) public String includeSampleData; - + @Parameter(names = {"--include-sample-id"}, description = "The body web service includeSampleId parameter", required = false, help = true, arity = 0) public boolean includeSampleId = false; - + @Parameter(names = {"--include-genotype"}, description = "The body web service includeGenotype parameter", required = false, help = true, arity = 0) public boolean includeGenotype = false; - + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + @Parameter(names = {"--qual"}, description = "The body web service qual parameter", required = false, arity = 1) public String qual; - + @Parameter(names = {"--filter"}, description = "The body web service filter parameter", required = false, arity = 1) public String filter; - + @Parameter(names = {"--file-data"}, description = "The body web service fileData parameter", required = false, arity = 1) public String fileData; - + @Parameter(names = {"--genotype"}, description = "The body web service genotype parameter", required = false, arity = 1) public String genotype; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--sample-limit"}, description = "The body web service sampleLimit parameter", required = false, arity = 1) public Integer sampleLimit; - + @Parameter(names = {"--sample-skip"}, description = "The body web service sampleSkip parameter", required = false, arity = 1) public Integer sampleSkip; - + @Parameter(names = {"--sample-data"}, description = "The body web service sampleData parameter", required = false, arity = 1) public String sampleData; - + @Parameter(names = {"--sample-annotation"}, description = "The body web service sampleAnnotation parameter", required = false, arity = 1) public String sampleAnnotation; - + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--family-members"}, description = "The body web service familyMembers parameter", required = false, arity = 1) public String familyMembers; - + @Parameter(names = {"--family-disorder"}, description = "The body web service familyDisorder parameter", required = false, arity = 1) public String familyDisorder; - + @Parameter(names = {"--family-proband"}, description = "The body web service familyProband parameter", required = false, arity = 1) public String familyProband; - + @Parameter(names = {"--family-segregation"}, description = "The body web service familySegregation parameter", required = false, arity = 1) public String familySegregation; - + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--cohort-stats-pass"}, description = "The body web service cohortStatsPass parameter", required = false, arity = 1) public String cohortStatsPass; - + @Parameter(names = {"--cohort-stats-mgf"}, description = "The body web service cohortStatsMgf parameter", required = false, arity = 1) public String cohortStatsMgf; - + @Parameter(names = {"--missing-alleles"}, description = "The body web service missingAlleles parameter", required = false, arity = 1) public String missingAlleles; - + @Parameter(names = {"--missing-genotypes"}, description = "The body web service missingGenotypes parameter", required = false, arity = 1) public String missingGenotypes; - + @Parameter(names = {"--annotation-exists"}, description = "The body web service annotationExists parameter", required = false, arity = 1) public Boolean annotationExists; - + @Parameter(names = {"--score"}, description = "The body web service score parameter", required = false, arity = 1) public String score; - + @Parameter(names = {"--polyphen"}, description = "The body web service polyphen parameter", required = false, arity = 1) public String polyphen; - + @Parameter(names = {"--sift"}, description = "The body web service sift parameter", required = false, arity = 1) public String sift; - + @Parameter(names = {"--gene-role-in-cancer"}, description = "The body web service geneRoleInCancer parameter", required = false, arity = 1) public String geneRoleInCancer; - + @Parameter(names = {"--gene-trait-id"}, description = "The body web service geneTraitId parameter", required = false, arity = 1) public String geneTraitId; - + @Parameter(names = {"--gene-trait-name"}, description = "The body web service geneTraitName parameter", required = false, arity = 1) public String geneTraitName; - + @Parameter(names = {"--trait"}, description = "The body web service trait parameter", required = false, arity = 1) public String trait; - + @Parameter(names = {"--cosmic"}, description = "The body web service cosmic parameter", required = false, arity = 1) public String cosmic; - + @Parameter(names = {"--clinvar"}, description = "The body web service clinvar parameter", required = false, arity = 1) public String clinvar; - + @Parameter(names = {"--hpo"}, description = "The body web service hpo parameter", required = false, arity = 1) public String hpo; - + @Parameter(names = {"--go"}, description = "The body web service go parameter", required = false, arity = 1) public String go; - + @Parameter(names = {"--expression"}, description = "The body web service expression parameter", required = false, arity = 1) public String expression; - + @Parameter(names = {"--protein-keyword"}, description = "The body web service proteinKeyword parameter", required = false, arity = 1) public String proteinKeyword; - + @Parameter(names = {"--drug"}, description = "The body web service drug parameter", required = false, arity = 1) public String drug; - + @Parameter(names = {"--custom-annotation"}, description = "The body web service customAnnotation parameter", required = false, arity = 1) public String customAnnotation; - + @Parameter(names = {"--unknown-genotype"}, description = "The body web service unknownGenotype parameter", required = false, arity = 1) public String unknownGenotype; - + @Parameter(names = {"--sample-metadata"}, description = "The body web service sampleMetadata parameter", required = false, help = true, arity = 0) public boolean sampleMetadata = false; - + @Parameter(names = {"--sort"}, description = "The body web service sort parameter", required = false, help = true, arity = 0) public boolean sort = false; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--output-file-name"}, description = "The body web service outputFileName parameter", required = false, arity = 1) public String outputFileName; - + @Parameter(names = {"--output-file-format"}, description = "The body web service outputFileFormat parameter", required = false, arity = 1) public String outputFileFormat; - + @Parameter(names = {"--variants-file"}, description = "The body web service variantsFile parameter", required = false, arity = 1) public String variantsFile; - + @Parameter(names = {"--body_include"}, description = "The body web service include parameter", required = false, arity = 1) public String bodyInclude; - + @Parameter(names = {"--body_exclude"}, description = "The body web service exclude parameter", required = false, arity = 1) public String bodyExclude; - + @Parameter(names = {"--limit"}, description = "The body web service limit parameter", required = false, arity = 1) public Integer limit; - + @Parameter(names = {"--skip"}, description = "The body web service skip parameter", required = false, arity = 1) public Integer skip; - + @Parameter(names = {"--summary"}, description = "The body web service summary parameter", required = false, help = true, arity = 0) public boolean summary = false; - + } @Parameters(commandNames = {"family-genotypes"}, commandDescription ="Calculate the possible genotypes for the members of a family") public class GenotypesFamilyCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--family"}, description = "Family id", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis id", required = false, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--mode-of-inheritance"}, description = "Mode of inheritance", required = true, arity = 1) - public String modeOfInheritance = "MONOALLELIC"; - + public String modeOfInheritance = "MONOALLELIC"; + @Parameter(names = {"--penetrance"}, description = "Penetrance", required = false, arity = 1) - public String penetrance = "COMPLETE"; - + public String penetrance = "COMPLETE"; + @Parameter(names = {"--disorder"}, description = "Disorder id", required = false, arity = 1) - public String disorder; - + public String disorder; + } @Parameters(commandNames = {"family-qc-run"}, commandDescription ="Run quality control (QC) for a given family. It computes the relatedness scores among the family members") public class RunFamilyQcCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--relatedness-method"}, description = "The body web service relatednessMethod parameter", required = false, arity = 1) public String relatednessMethod; - + @Parameter(names = {"--relatedness-maf"}, description = "The body web service relatednessMaf parameter", required = false, arity = 1) public String relatednessMaf; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"file-delete"}, commandDescription =" [DEPRECATED] Use operation/variant/delete") public class DeleteFileCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Files to remove", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--resume"}, description = "Resume a previously failed indexation", required = false, help = true, arity = 0) - public boolean resume = false; - + public boolean resume = false; + } @Parameters(commandNames = {"gatk-run"}, commandDescription ="GATK is a Genome Analysis Toolkit for variant discovery in high-throughput sequencing data. Supported Gatk commands: HaplotypeCaller") public class RunGatkCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @DynamicParameter(names = {"--gatk-params"}, description = "The body web service gatkParams parameter. Use: --gatk-params key=value", required = false) public java.util.Map gatkParams = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"genome-plot-run"}, commandDescription ="Generate a genome plot for a given sample.") public class RunGenomePlotCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--config-file"}, description = "The body web service configFile parameter", required = false, arity = 1) public String configFile; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"gwas-run"}, commandDescription ="Run a Genome Wide Association Study between two cohorts.") public class RunGwasCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--phenotype"}, description = "The body web service phenotype parameter", required = false, arity = 1) public String phenotype; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--index-score-id"}, description = "The body web service indexScoreId parameter", required = false, arity = 1) public String indexScoreId; - + @Parameter(names = {"--method"}, description = "The body web service method parameter", required = false, arity = 1) public String method; - + @Parameter(names = {"--fisher-mode"}, description = "The body web service fisherMode parameter", required = false, arity = 1) public String fisherMode; - + @Parameter(names = {"--case-cohort"}, description = "The body web service caseCohort parameter", required = false, arity = 1) public String caseCohort; - + @Parameter(names = {"--case-cohort-samples-annotation"}, description = "The body web service caseCohortSamplesAnnotation parameter", required = false, arity = 1) public String caseCohortSamplesAnnotation; - + @Parameter(names = {"--case-cohort-samples"}, description = "The body web service caseCohortSamples parameter", required = false, arity = 1) public String caseCohortSamples; - + @Parameter(names = {"--control-cohort"}, description = "The body web service controlCohort parameter", required = false, arity = 1) public String controlCohort; - + @Parameter(names = {"--control-cohort-samples-annotation"}, description = "The body web service controlCohortSamplesAnnotation parameter", required = false, arity = 1) public String controlCohortSamplesAnnotation; - + @Parameter(names = {"--control-cohort-samples"}, description = "The body web service controlCohortSamples parameter", required = false, arity = 1) public String controlCohortSamples; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"hr-detect-run"}, commandDescription ="Run HRDetect analysis for a given somatic sample.") public class RunHrDetectCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "ID to identify the HRDetect results.", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "Decription for these particular HRDetect results.", required = false, arity = 1) public String description; - + @Parameter(names = {"--sample-id"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) public String sampleId; - + @Parameter(names = {"--snv-fitting-id"}, description = "Mutational signature fitting ID for SNV.", required = false, arity = 1) public String snvFittingId; - + @Parameter(names = {"--sv-fitting-id"}, description = "Mutational signature fitting ID for SV.", required = false, arity = 1) public String svFittingId; - + @Parameter(names = {"--cnv-query"}, description = "CNV query", required = false, arity = 1) public String cnvQuery; - + @Parameter(names = {"--indel-query"}, description = "INDEL query", required = false, arity = 1) public String indelQuery; - + @Parameter(names = {"--snv3custom-name"}, description = "Custom signature name that will be considered as SNV3 input for HRDetect.", required = false, arity = 1) public String snv3CustomName; - + @Parameter(names = {"--snv8custom-name"}, description = "Custom signature name that will be considered as SNV8 input for HRDetect.", required = false, arity = 1) public String snv8CustomName; - + @Parameter(names = {"--sv3custom-name"}, description = "Custom signature name that will be considered as SV3 input for HRDetect.", required = false, arity = 1) public String sv3CustomName; - + @Parameter(names = {"--sv8custom-name"}, description = "Custom signature name that will be considered as SV8 input for HRDetect.", required = false, arity = 1) public String sv8CustomName; - + @Parameter(names = {"--bootstrap"}, description = "Request HRDetect with bootstrap.", required = false, arity = 1) public Boolean bootstrap; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"index-run"}, commandDescription =" [DEPRECATED] Use operation/variant/index") public class RunIndexCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; - + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; - + @Parameter(names = {"--outdir"}, description = "Output directory", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean transform = false; - + @Parameter(names = {"--gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean gvcf = false; - + @Parameter(names = {"--normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean normalizationSkip = false; - + @Parameter(names = {"--reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String referenceGenome; - + @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean family = false; - + @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean somatic = false; - + @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean load = false; - + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) public boolean forceReload = false; - + @Parameter(names = {"--load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String loadSplitData; - + @Parameter(names = {"--load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean loadMultiFileData = false; - + @Parameter(names = {"--load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String loadSampleIndex; - + @Parameter(names = {"--load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String loadArchive; - + @Parameter(names = {"--load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String loadHomRef; - + @Parameter(names = {"--post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String postLoadCheck; - + @Parameter(names = {"--include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String includeGenotypes; - + @Parameter(names = {"--include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String includeSampleData; - + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String merge; - + @Parameter(names = {"--deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String deduplicationPolicy; - + @Parameter(names = {"--calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean calculateStats = false; - + @Parameter(names = {"--aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String aggregated; - + @Parameter(names = {"--aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String aggregationMappingFile; - + @Parameter(names = {"--annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean annotate = false; - + @Parameter(names = {"--annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String annotator; - + @Parameter(names = {"--overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean overwriteAnnotations = false; - + @Parameter(names = {"--index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexSearch = false; - + @Parameter(names = {"--skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean skipIndexedFiles = false; - + } @Parameters(commandNames = {"individual-qc-run"}, commandDescription ="Run quality control (QC) for a given individual. It includes inferred sex and mendelian errors (UDP)") public class RunIndividualQcCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; - + @Parameter(names = {"--sample"}, description = "Sample ID (required when the individual has multiple samples)", required = false, arity = 1) public String sample; - + @Parameter(names = {"--inferred-sex-method"}, description = "Inferred sex method. Valid values: CoverageRatio", required = false, arity = 1) public String inferredSexMethod = "CoverageRatio"; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"inferred-sex-run"}, commandDescription ="Infer sex from chromosome mean coverages.") public class RunInferredSexCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; - + @Parameter(names = {"--sample"}, description = "Sample ID (required when the individual has multiple samples)", required = false, arity = 1) public String sample; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"knockout-gene-query"}, commandDescription ="Fetch values from KnockoutAnalysis result, by genes") public class QueryKnockoutGeneCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job"}, description = "Job ID or UUID", required = false, arity = 1) - public String job; - + public String job; + } @Parameters(commandNames = {"knockout-individual-query"}, commandDescription ="Fetch values from KnockoutAnalysis result, by individuals") public class QueryKnockoutIndividualCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job"}, description = "Job ID or UUID", required = false, arity = 1) - public String job; - + public String job; + } @Parameters(commandNames = {"knockout-run"}, commandDescription ="Obtains the list of knocked out genes for each sample.") public class RunKnockoutCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--consequence-type"}, description = "The body web service consequenceType parameter", required = false, arity = 1) public String consequenceType; - + @Parameter(names = {"--filter"}, description = "The body web service filter parameter", required = false, arity = 1) public String filter; - + @Parameter(names = {"--qual"}, description = "The body web service qual parameter", required = false, arity = 1) public String qual; - + @Parameter(names = {"--skip-genes-file"}, description = "The body web service skipGenesFile parameter", required = false, help = true, arity = 0) public boolean skipGenesFile = false; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + } @Parameters(commandNames = {"mendelian-error-run"}, commandDescription ="Run mendelian error analysis to infer uniparental disomy regions.") public class RunMendelianErrorCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--individual"}, description = "The body web service individual parameter", required = false, arity = 1) public String individual; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"metadata"}, commandDescription ="") public class MetadataCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--include-study"}, description = "List of studies to include in the result. Accepts 'all' and 'none'.", required = false, arity = 1) - public String includeStudy; - + public String includeStudy; + @Parameter(names = {"--include-file"}, description = "List of files to be returned. Accepts 'all' and 'none'. If undefined, automatically includes files used for filtering. If none, no file is included.", required = false, arity = 1) - public String includeFile; - + public String includeFile; + @Parameter(names = {"--include-sample"}, description = "List of samples to be included in the result. Accepts 'all' and 'none'. If undefined, automatically includes samples used for filtering. If none, no sample is included.", required = false, arity = 1) - public String includeSample; - + public String includeSample; + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + } @Parameters(commandNames = {"mutational-signature-query"}, commandDescription ="Run mutational signature analysis for a given sample. Use context index.") public class QueryMutationalSignatureCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--sample"}, description = "Sample name", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--type"}, description = "Variant type. Valid values: SNV, SV", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--file-data"}, description = "Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file is specified, will use all files from 'file' filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP", required = false, arity = 1) - public String fileData; - + public String fileData; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--qual"}, description = "Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4", required = false, arity = 1) - public String qual; - + public String qual; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--panel"}, description = "Filter by genes from the given disease panel", required = false, arity = 1) - public String panel; - + public String panel; + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "Filter genes from specific panels that match certain mode of inheritance. Accepted values : [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String panelModeOfInheritance; - + public String panelModeOfInheritance; + @Parameter(names = {"--panel-confidence"}, description = "Filter genes from specific panels that match certain confidence. Accepted values : [ high, medium, low, rejected ]", required = false, arity = 1) - public String panelConfidence; - + public String panelConfidence; + @Parameter(names = {"--panel-feature-type"}, description = "Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]", required = false, arity = 1) - public String panelFeatureType; - + public String panelFeatureType; + @Parameter(names = {"--panel-role-in-cancer"}, description = "Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, tumorSuppressorGene, fusion ]", required = false, arity = 1) - public String panelRoleInCancer; - + public String panelRoleInCancer; + @Parameter(names = {"--panel-intersection"}, description = "Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel.", required = false, help = true, arity = 0) - public boolean panelIntersection = false; - + public boolean panelIntersection = false; + @Parameter(names = {"--ms-id"}, description = "Signature ID.", required = false, arity = 1) - public String msId; - + public String msId; + @Parameter(names = {"--ms-description"}, description = "Signature description.", required = false, arity = 1) - public String msDescription; - + public String msDescription; + } @Parameters(commandNames = {"mutational-signature-run"}, commandDescription ="Run mutational signature analysis for a given sample.") public class RunMutationalSignatureCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "Signature ID.", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "Signature description.", required = false, arity = 1) public String description; - + @Parameter(names = {"--sample"}, description = "Sample.", required = false, arity = 1) public String sample; - + @Parameter(names = {"--query"}, description = "Signature query in JSON format, e.g: ''{\'sample\':\'NR123456_T\', \'fileData\': \'NR.123456_T_vs_NR.1234567_G.annot.vcf.gz:FILTER=PASS;CLPM<=0;ASMD>=140\'}'.", required = false, arity = 1) public String query; - + @Parameter(names = {"--fit-id"}, description = "Fiiting signature ID", required = false, arity = 1) public String fitId; - + @Parameter(names = {"--fit-method"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String fitMethod = "FitMS"; - + @Parameter(names = {"--fit-n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) public Integer fitNBoot; - + @Parameter(names = {"--fit-sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) public String fitSigVersion = "RefSigv2"; - + @Parameter(names = {"--fit-organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) public String fitOrgan; - + @Parameter(names = {"--fit-threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) public Float fitThresholdPerc = 5f; - + @Parameter(names = {"--fit-threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) public Float fitThresholdPval = 0.05f; - + @Parameter(names = {"--fit-max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) public Integer fitMaxRareSigs = 1; - + @Parameter(names = {"--fit-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) public String fitSignaturesFile; - + @Parameter(names = {"--fit-rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) public String fitRareSignaturesFile; - + @Parameter(names = {"--skip"}, description = "To skip to compute catalogue counts or the signature fitting. Use the following keywords: catalogue, fitting.", required = false, arity = 1) public String skip; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"plink-run"}, commandDescription ="Plink is a whole genome association analysis toolset, designed to perform a range of basic, large-scale analyses.") public class RunPlinkCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @DynamicParameter(names = {"--plink-params"}, description = "The body web service plinkParams parameter. Use: --plink-params key=value", required = false) public java.util.Map plinkParams = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"query"}, commandDescription ="Filter and fetch variants from indexed VCF files in the variant storage") public class QueryCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sort"}, description = "Sort the results", required = false, help = true, arity = 0) - public boolean sort = false; - + public boolean sort = false; + @Parameter(names = {"--summary"}, description = "Fast fetch of main variant parameters", required = false, help = true, arity = 0) - public boolean summary = false; - + public boolean summary = false; + @Parameter(names = {"--approximate-count"}, description = "Get an approximate count, instead of an exact total count. Reduces execution time", required = false, help = true, arity = 0) - public boolean approximateCount = false; - + public boolean approximateCount = false; + @Parameter(names = {"--approximate-count-sampling-size"}, description = "Sampling size to get the approximate count. Larger values increase accuracy but also increase execution time", required = false, arity = 1) - public Integer approximateCountSamplingSize; - + public Integer approximateCountSamplingSize; + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--reference"}, description = "Reference allele", required = false, arity = 1) - public String reference; - + public String reference; + @Parameter(names = {"--alternate"}, description = "Main alternate allele", required = false, arity = 1) - public String alternate; - + public String alternate; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--qual"}, description = "Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4", required = false, arity = 1) - public String qual; - + public String qual; + @Parameter(names = {"--file-data"}, description = "Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file is specified, will use all files from 'file' filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP", required = false, arity = 1) - public String fileData; - + public String fileData; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--genotype"}, description = "Samples with a specific genotype: {samp_1}:{gt_1}(,{gt_n})*(;{samp_n}:{gt_1}(,{gt_n})*)* e.g. HG0097:0/0;HG0098:0/1,1/1. Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT. This will automatically set 'includeSample' parameter when not provided", required = false, arity = 1) - public String genotype; - + public String genotype; + @Parameter(names = {"--sample-data"}, description = "Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will use all samples from 'sample' or 'genotype' filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10", required = false, arity = 1) - public String sampleData; - + public String sampleData; + @Parameter(names = {"--sample-annotation"}, description = "Selects some samples using metadata information from Catalog. e.g. age>20;phenotype=hpo:123,hpo:456;name=smith", required = false, arity = 1) - public String sampleAnnotation; - + public String sampleAnnotation; + @Parameter(names = {"--sample-metadata"}, description = "Return the samples metadata group by study. Sample names will appear in the same order as their corresponding genotypes.", required = false, help = true, arity = 0) - public boolean sampleMetadata = false; - + public boolean sampleMetadata = false; + @Parameter(names = {"--unknown-genotype"}, description = "Returned genotype for unknown genotypes. Common values: [0/0, 0|0, ./.]", required = false, arity = 1) - public String unknownGenotype; - + public String unknownGenotype; + @Parameter(names = {"--sample-limit"}, description = "Limit the number of samples to be included in the result", required = false, arity = 1) - public Integer sampleLimit; - + public Integer sampleLimit; + @Parameter(names = {"--sample-skip"}, description = "Skip some samples from the result. Useful for sample pagination.", required = false, arity = 1) - public Integer sampleSkip; - + public Integer sampleSkip; + @Parameter(names = {"--cohort"}, description = "Select variants with calculated stats for the selected cohorts", required = false, arity = 1) - public String cohort; - + public String cohort; + @Parameter(names = {"--cohort-stats-ref"}, description = "Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsRef; - + public String cohortStatsRef; + @Parameter(names = {"--cohort-stats-alt"}, description = "Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsAlt; - + public String cohortStatsAlt; + @Parameter(names = {"--cohort-stats-maf"}, description = "Minor Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMaf; - + public String cohortStatsMaf; + @Parameter(names = {"--cohort-stats-mgf"}, description = "Minor Genotype Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMgf; - + public String cohortStatsMgf; + @Parameter(names = {"--cohort-stats-pass"}, description = "Filter PASS frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8", required = false, arity = 1) - public String cohortStatsPass; - + public String cohortStatsPass; + @Parameter(names = {"--missing-alleles"}, description = "Number of missing alleles: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingAlleles; - + public String missingAlleles; + @Parameter(names = {"--missing-genotypes"}, description = "Number of missing genotypes: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingGenotypes; - + public String missingGenotypes; + @Parameter(names = {"--score"}, description = "Filter by variant score: [{study:}]{score}[<|>|<=|>=]{number}", required = false, arity = 1) - public String score; - + public String score; + @Parameter(names = {"--family"}, description = "Filter variants where any of the samples from the given family contains the variant (HET or HOM_ALT)", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-disorder"}, description = "Specify the disorder to use for the family segregation", required = false, arity = 1) - public String familyDisorder; - + public String familyDisorder; + @Parameter(names = {"--family-segregation"}, description = "Filter by segregation mode from a given family. Accepted values: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String familySegregation; - + public String familySegregation; + @Parameter(names = {"--family-members"}, description = "Sub set of the members of a given family", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-proband"}, description = "Specify the proband child to use for the family segregation", required = false, arity = 1) - public String familyProband; - + public String familyProband; + @Parameter(names = {"--include-study"}, description = "List of studies to include in the result. Accepts 'all' and 'none'.", required = false, arity = 1) - public String includeStudy; - + public String includeStudy; + @Parameter(names = {"--include-file"}, description = "List of files to be returned. Accepts 'all' and 'none'. If undefined, automatically includes files used for filtering. If none, no file is included.", required = false, arity = 1) - public String includeFile; - + public String includeFile; + @Parameter(names = {"--include-sample"}, description = "List of samples to be included in the result. Accepts 'all' and 'none'. If undefined, automatically includes samples used for filtering. If none, no sample is included.", required = false, arity = 1) - public String includeSample; - + public String includeSample; + @Parameter(names = {"--include-sample-data"}, description = "List of Sample Data keys (i.e. FORMAT column from VCF file) from Sample Data to include in the output. e.g: DP,AD. Accepts 'all' and 'none'.", required = false, arity = 1) - public String includeSampleData; - + public String includeSampleData; + @Parameter(names = {"--include-genotype"}, description = "Include genotypes, apart of other formats defined with includeFormat", required = false, arity = 1) - public String includeGenotype; - + public String includeGenotype; + @Parameter(names = {"--include-sample-id"}, description = "Include sampleId on each result", required = false, arity = 1) - public String includeSampleId; - + public String includeSampleId; + @Parameter(names = {"--annotation-exists"}, description = "Return only annotated variants", required = false, arity = 1) - public Boolean annotationExists; - + public Boolean annotationExists; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) - public String xref; - + public String xref; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--protein-substitution"}, description = "Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant", required = false, arity = 1) - public String proteinSubstitution; - + public String proteinSubstitution; + @Parameter(names = {"--conservation"}, description = "Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1", required = false, arity = 1) - public String conservation; - + public String conservation; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--population-frequency-ref"}, description = "Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyRef; - + public String populationFrequencyRef; + @Parameter(names = {"--population-frequency-maf"}, description = "Population minor allele frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyMaf; - + public String populationFrequencyMaf; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--gene-trait-id"}, description = "List of gene trait association id. e.g. 'umls:C0007222' , 'OMIM:269600'", required = false, arity = 1) - public String geneTraitId; - + public String geneTraitId; + @Parameter(names = {"--go"}, description = "List of GO (Gene Ontology) terms. e.g. 'GO:0002020'", required = false, arity = 1) - public String go; - + public String go; + @Parameter(names = {"--expression"}, description = "List of tissues of interest. e.g. 'lung'", required = false, arity = 1) - public String expression; - + public String expression; + @Parameter(names = {"--protein-keyword"}, description = "List of Uniprot protein variant annotation keywords", required = false, arity = 1) - public String proteinKeyword; - + public String proteinKeyword; + @Parameter(names = {"--drug"}, description = "List of drug names", required = false, arity = 1) - public String drug; - + public String drug; + @Parameter(names = {"--functional-score"}, description = "Functional score: {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , cadd_raw<=0.3", required = false, arity = 1) - public String functionalScore; - + public String functionalScore; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--custom-annotation"}, description = "Custom annotation: {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}", required = false, arity = 1) - public String customAnnotation; - + public String customAnnotation; + @Parameter(names = {"--panel"}, description = "Filter by genes from the given disease panel", required = false, arity = 1) - public String panel; - + public String panel; + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "Filter genes from specific panels that match certain mode of inheritance. Accepted values : [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String panelModeOfInheritance; - + public String panelModeOfInheritance; + @Parameter(names = {"--panel-confidence"}, description = "Filter genes from specific panels that match certain confidence. Accepted values : [ high, medium, low, rejected ]", required = false, arity = 1) - public String panelConfidence; - + public String panelConfidence; + @Parameter(names = {"--panel-role-in-cancer"}, description = "Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, tumorSuppressorGene, fusion ]", required = false, arity = 1) - public String panelRoleInCancer; - + public String panelRoleInCancer; + @Parameter(names = {"--panel-feature-type"}, description = "Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]", required = false, arity = 1) - public String panelFeatureType; - + public String panelFeatureType; + @Parameter(names = {"--panel-intersection"}, description = "Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel.", required = false, help = true, arity = 0) - public boolean panelIntersection = false; - + public boolean panelIntersection = false; + @Parameter(names = {"--trait"}, description = "List of traits, based on ClinVar, HPO, COSMIC, i.e.: IDs, histologies, descriptions,...", required = false, arity = 1) - public String trait; - + public String trait; + } @Parameters(commandNames = {"relatedness-run"}, commandDescription ="Compute a score to quantify relatedness between samples.") public class RunRelatednessCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--individuals"}, description = "The body web service individuals parameter", required = false, arity = 1) public String individuals; - + @Parameter(names = {"--samples"}, description = "The body web service samples parameter", required = false, arity = 1) public String samples; - + @Parameter(names = {"--minor-allele-freq"}, description = "The body web service minorAlleleFreq parameter", required = false, arity = 1) public String minorAlleleFreq; - + @Parameter(names = {"--method"}, description = "The body web service method parameter", required = false, arity = 1) public String method; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"rvtests-run"}, commandDescription ="Rvtests is a flexible software package for genetic association studies. Supported RvTests commands: rvtest, vcf2kinship") public class RunRvtestsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @DynamicParameter(names = {"--rvtests-params"}, description = "The body web service rvtestsParams parameter. Use: --rvtests-params key=value", required = false) public java.util.Map rvtestsParams = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"sample-aggregation-stats"}, commandDescription ="Calculate and fetch sample aggregation stats") public class AggregationStatsSampleCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--genotype"}, description = "Samples with a specific genotype: {samp_1}:{gt_1}(,{gt_n})*(;{samp_n}:{gt_1}(,{gt_n})*)* e.g. HG0097:0/0;HG0098:0/1,1/1. Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT. This will automatically set 'includeSample' parameter when not provided", required = false, arity = 1) - public String genotype; - + public String genotype; + @Parameter(names = {"--sample-annotation"}, description = "Selects some samples using metadata information from Catalog. e.g. age>20;phenotype=hpo:123,hpo:456;name=smith", required = false, arity = 1) - public String sampleAnnotation; - + public String sampleAnnotation; + @Parameter(names = {"--family"}, description = "Filter variants where any of the samples from the given family contains the variant (HET or HOM_ALT)", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-disorder"}, description = "Specify the disorder to use for the family segregation", required = false, arity = 1) - public String familyDisorder; - + public String familyDisorder; + @Parameter(names = {"--family-segregation"}, description = "Filter by segregation mode from a given family. Accepted values: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String familySegregation; - + public String familySegregation; + @Parameter(names = {"--family-members"}, description = "Sub set of the members of a given family", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-proband"}, description = "Specify the proband child to use for the family segregation", required = false, arity = 1) - public String familyProband; - + public String familyProband; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--field"}, description = "List of facet fields separated by semicolons, e.g.: studies;type. For nested faceted fields use >>, e.g.: chromosome>>type . Accepted values: chromosome, type, genotype, consequenceType, biotype, clinicalSignificance, dp, qual, filter", required = false, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"sample-eligibility-run"}, commandDescription ="Filter samples by a complex query involving metadata and variants data") public class RunSampleEligibilityCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--query"}, description = "The body web service query parameter", required = false, arity = 1) public String query; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--cohort-id"}, description = "The body web service cohortId parameter", required = false, arity = 1) public String cohortId; - + } @Parameters(commandNames = {"sample-qc-run"}, commandDescription ="Run quality control (QC) for a given sample. It includes variant stats, and if the sample is somatic, mutational signature and genome plot are calculated.") public class RunSampleQcCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) public String sample; - + @Parameter(names = {"--vs-id"}, description = "Variant stats ID.", required = false, arity = 1) public String vsId; - + @Parameter(names = {"--vs-description"}, description = "Variant stats description.", required = false, arity = 1) public String vsDescription; - + @Parameter(names = {"--vs-query-id"}, description = "The body web service id parameter", required = false, arity = 1) public String vsQueryId; - + @Parameter(names = {"--vs-query-region"}, description = "The body web service region parameter", required = false, arity = 1) public String vsQueryRegion; - + @Parameter(names = {"--vs-query-gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String vsQueryGene; - + @Parameter(names = {"--vs-query-type"}, description = "The body web service type parameter", required = false, arity = 1) public String vsQueryType; - + @Parameter(names = {"--vs-query-panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String vsQueryPanel; - + @Parameter(names = {"--vs-query-panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String vsQueryPanelModeOfInheritance; - + @Parameter(names = {"--vs-query-panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String vsQueryPanelConfidence; - + @Parameter(names = {"--vs-query-panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String vsQueryPanelRoleInCancer; - + @Parameter(names = {"--vs-query-panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean vsQueryPanelIntersection = false; - + @Parameter(names = {"--vs-query-panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String vsQueryPanelFeatureType; - + @Parameter(names = {"--vs-query-cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String vsQueryCohortStatsRef; - + @Parameter(names = {"--vs-query-cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String vsQueryCohortStatsAlt; - + @Parameter(names = {"--vs-query-cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String vsQueryCohortStatsMaf; - + @Parameter(names = {"--vs-query-ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String vsQueryCt; - + @Parameter(names = {"--vs-query-xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String vsQueryXref; - + @Parameter(names = {"--vs-query-biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String vsQueryBiotype; - + @Parameter(names = {"--vs-query-protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String vsQueryProteinSubstitution; - + @Parameter(names = {"--vs-query-conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String vsQueryConservation; - + @Parameter(names = {"--vs-query-population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String vsQueryPopulationFrequencyMaf; - + @Parameter(names = {"--vs-query-population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String vsQueryPopulationFrequencyAlt; - + @Parameter(names = {"--vs-query-population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String vsQueryPopulationFrequencyRef; - + @Parameter(names = {"--vs-query-transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String vsQueryTranscriptFlag; - + @Parameter(names = {"--vs-query-functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String vsQueryFunctionalScore; - + @Parameter(names = {"--vs-query-clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String vsQueryClinical; - + @Parameter(names = {"--vs-query-clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String vsQueryClinicalSignificance; - + @Parameter(names = {"--vs-query-clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean vsQueryClinicalConfirmedStatus = false; - + @Parameter(names = {"--ms-id"}, description = "Signature ID.", required = false, arity = 1) public String msId; - + @Parameter(names = {"--ms-description"}, description = "Signature description.", required = false, arity = 1) public String msDescription; - + @Parameter(names = {"--ms-query"}, description = "Signature query in JSON format, e.g: ''{\'sample\':\'NR123456_T\', \'fileData\': \'NR.123456_T_vs_NR.1234567_G.annot.vcf.gz:FILTER=PASS;CLPM<=0;ASMD>=140\'}'.", required = false, arity = 1) public String msQuery; - + @Parameter(names = {"--ms-fit-id"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String msFitId; - + @Parameter(names = {"--ms-fit-method"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String msFitMethod = "FitMS"; - + @Parameter(names = {"--ms-fit-n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) public Integer msFitNBoot; - + @Parameter(names = {"--ms-fit-sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) public String msFitSigVersion = "RefSigv2"; - + @Parameter(names = {"--ms-fit-organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) public String msFitOrgan; - + @Parameter(names = {"--ms-fit-threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) public Float msFitThresholdPerc = 5f; - + @Parameter(names = {"--ms-fit-threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) public Float msFitThresholdPval = 0.05f; - + @Parameter(names = {"--ms-fit-max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) public Integer msFitMaxRareSigs = 1; - + @Parameter(names = {"--ms-fit-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) public String msFitSignaturesFile; - + @Parameter(names = {"--ms-fit-rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) public String msFitRareSignaturesFile; - + @Parameter(names = {"--gp-id"}, description = "Genome plot ID.", required = false, arity = 1) public String gpId; - + @Parameter(names = {"--gp-description"}, description = "Genome plot description.", required = false, arity = 1) public String gpDescription; - + @Parameter(names = {"--gp-config-file"}, description = "Genome plot configuration file.", required = false, arity = 1) public String gpConfigFile; - + @Parameter(names = {"--skip"}, description = "Quality control metrics to skip. Valid values are: variant-stats, signature, signature-catalogue, signature-fitting, genome-plot", required = false, arity = 1) public String skip; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"sample-query"}, commandDescription ="Get sample data of a given variant") public class QuerySampleCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--variant"}, description = "Variant", required = false, arity = 1) - public String variant; - + public String variant; + @Parameter(names = {"--study", "-s"}, description = "Study where all the samples belong to", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--genotype"}, description = "Genotypes that the sample must have to be selected", required = false, arity = 1) - public String genotype; - + public String genotype; + } @Parameters(commandNames = {"sample-run"}, commandDescription ="Get samples given a set of variants") public class RunSampleCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--type"}, description = "The body web service type parameter", required = false, arity = 1) public String type; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String panelModeOfInheritance; - + @Parameter(names = {"--panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String panelConfidence; - + @Parameter(names = {"--panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String panelRoleInCancer; - + @Parameter(names = {"--panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean panelIntersection = false; - + @Parameter(names = {"--panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String panelFeatureType; - + @Parameter(names = {"--cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String cohortStatsRef; - + @Parameter(names = {"--cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String cohortStatsAlt; - + @Parameter(names = {"--cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String cohortStatsMaf; - + @Parameter(names = {"--ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String ct; - + @Parameter(names = {"--xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String xref; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String proteinSubstitution; - + @Parameter(names = {"--conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String conservation; - + @Parameter(names = {"--population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String populationFrequencyMaf; - + @Parameter(names = {"--population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String populationFrequencyAlt; - + @Parameter(names = {"--population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String populationFrequencyRef; - + @Parameter(names = {"--transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String transcriptFlag; - + @Parameter(names = {"--functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String functionalScore; - + @Parameter(names = {"--clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String clinical; - + @Parameter(names = {"--clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String clinicalSignificance; - + @Parameter(names = {"--clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean clinicalConfirmedStatus = false; - + @Parameter(names = {"--genotypes"}, description = "The body web service genotypes parameter", required = false, arity = 1) public String genotypes; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--samples-in-all-variants"}, description = "The body web service samplesInAllVariants parameter", required = false, help = true, arity = 0) public boolean samplesInAllVariants = false; - + @Parameter(names = {"--max-variants"}, description = "The body web service maxVariants parameter", required = false, arity = 1) public Integer maxVariants; - + } @Parameters(commandNames = {"sample-stats-query"}, commandDescription ="Obtain sample variant stats from a sample.") public class QuerySampleStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--sample-data"}, description = "Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will use all samples from 'sample' or 'genotype' filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10", required = false, arity = 1) - public String sampleData; - + public String sampleData; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--filter-transcript"}, description = "Do filter transcripts when obtaining transcript counts", required = false, help = true, arity = 0) - public boolean filterTranscript = false; - + public boolean filterTranscript = false; + @Parameter(names = {"--sample"}, description = "Sample ID", required = true, arity = 1) - public String sample; - + public String sample; + } @Parameters(commandNames = {"sample-stats-run"}, commandDescription ="Compute sample variant stats for the selected list of samples.") public class RunSampleStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--individual"}, description = "The body web service individual parameter", required = false, arity = 1) public String individual; - + @Parameter(names = {"--variant-query-id"}, description = "The body web service id parameter", required = false, arity = 1) public String variantQueryId; - + @Parameter(names = {"--variant-query-region"}, description = "The body web service region parameter", required = false, arity = 1) public String variantQueryRegion; - + @Parameter(names = {"--variant-query-gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String variantQueryGene; - + @Parameter(names = {"--variant-query-type"}, description = "The body web service type parameter", required = false, arity = 1) public String variantQueryType; - + @Parameter(names = {"--variant-query-panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String variantQueryPanel; - + @Parameter(names = {"--variant-query-panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String variantQueryPanelModeOfInheritance; - + @Parameter(names = {"--variant-query-panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String variantQueryPanelConfidence; - + @Parameter(names = {"--variant-query-panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String variantQueryPanelRoleInCancer; - + @Parameter(names = {"--variant-query-panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean variantQueryPanelIntersection = false; - + @Parameter(names = {"--variant-query-panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String variantQueryPanelFeatureType; - + @Parameter(names = {"--variant-query-cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String variantQueryCohortStatsRef; - + @Parameter(names = {"--variant-query-cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String variantQueryCohortStatsAlt; - + @Parameter(names = {"--variant-query-cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String variantQueryCohortStatsMaf; - + @Parameter(names = {"--variant-query-ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String variantQueryCt; - + @Parameter(names = {"--variant-query-xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String variantQueryXref; - + @Parameter(names = {"--variant-query-biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String variantQueryBiotype; - + @Parameter(names = {"--variant-query-protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String variantQueryProteinSubstitution; - + @Parameter(names = {"--variant-query-conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String variantQueryConservation; - + @Parameter(names = {"--variant-query-population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String variantQueryPopulationFrequencyMaf; - + @Parameter(names = {"--variant-query-population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String variantQueryPopulationFrequencyAlt; - + @Parameter(names = {"--variant-query-population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String variantQueryPopulationFrequencyRef; - + @Parameter(names = {"--variant-query-transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String variantQueryTranscriptFlag; - + @Parameter(names = {"--variant-query-functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String variantQueryFunctionalScore; - + @Parameter(names = {"--variant-query-clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String variantQueryClinical; - + @Parameter(names = {"--variant-query-clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String variantQueryClinicalSignificance; - + @Parameter(names = {"--variant-query-clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean variantQueryClinicalConfirmedStatus = false; - + @Parameter(names = {"--variant-query-sample-data"}, description = "The body web service sampleData parameter", required = false, arity = 1) public String variantQuerySampleData; - + @Parameter(names = {"--variant-query-file-data"}, description = "The body web service fileData parameter", required = false, arity = 1) public String variantQueryFileData; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--index-overwrite"}, description = "The body web service indexOverwrite parameter", required = false, help = true, arity = 0) public boolean indexOverwrite = false; - + @Parameter(names = {"--index-id"}, description = "The body web service indexId parameter", required = false, arity = 1) public String indexId; - + @Parameter(names = {"--index-description"}, description = "The body web service indexDescription parameter", required = false, arity = 1) public String indexDescription; - + @Parameter(names = {"--batch-size"}, description = "The body web service batchSize parameter", required = false, arity = 1) public Integer batchSize; - + } @Parameters(commandNames = {"stats-export-run"}, commandDescription ="Export calculated variant stats and frequencies") public class RunStatsExportCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--cohorts"}, description = "The body web service cohorts parameter", required = false, arity = 1) public String cohorts; - + @Parameter(names = {"--output"}, description = "The body web service output parameter", required = false, arity = 1) public String output; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--output-file-format"}, description = "The body web service outputFileFormat parameter", required = false, arity = 1) public String outputFileFormat; - + } @Parameters(commandNames = {"stats-run"}, commandDescription ="Compute variant stats for any cohort and any set of variants.") public class RunStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--samples"}, description = "The body web service samples parameter", required = false, arity = 1) public String samples; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--output-file-name"}, description = "The body web service outputFileName parameter", required = false, arity = 1) public String outputFileName; - + @Parameter(names = {"--aggregated"}, description = "The body web service aggregated parameter", required = false, arity = 1) public String aggregated; - + @Parameter(names = {"--aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) public String aggregationMappingFile; - + } } \ No newline at end of file diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 27db4eaeee9..3fcaa74c98a 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -25,12 +25,15 @@ analysis: # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used opencgaExtTools: "opencb/opencga-ext-tools" tools: - exomiser-13.1.0: + - id: "exomiser" + version: "13.1" dockerId: "exomiser/exomiser-cli:13.1.0" resources: HG38: "exomiser/2109_hg38.zip" PHENOTYPE: "exomiser/2109_phenotype.zip" - exomiser-14.0.0: + - id: "exomiser" + version: "14.0" + defaultVersion: true dockerId: "exomiser/exomiser-cli:14.0.0" resources: HG38: "exomiser/2402_hg38.zip" diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index a506527e38f..89ffedf5483 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -527,5 +527,4 @@ public class FieldConstants { public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID."; public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID."; public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version."; - public static final String EXOMISER_DEFAULT_VERSION = "14.0.0"; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index c0450569451..9f1c7b32683 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -29,7 +29,7 @@ public class Analysis { private String resourceUrl; private String opencgaExtTools; - private Map tools; + private List tools; private Execution execution; @@ -37,7 +37,7 @@ public class Analysis { public Analysis() { packages = new ArrayList<>(); - tools = new HashMap<>(); + tools = new ArrayList<>(); execution = new Execution(); frameworks = new ArrayList<>(); } @@ -78,11 +78,11 @@ public Analysis setOpencgaExtTools(String opencgaExtTools) { return this; } - public Map getTools() { + public List getTools() { return tools; } - public Analysis setTools(Map tools) { + public Analysis setTools(List tools) { this.tools = tools; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java index 65aa7dc08be..3744915b6b9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java @@ -21,6 +21,9 @@ public class AnalysisTool { + private String id; + private String version; + private boolean defaultVersion; private String dockerId; private String params; private Map resources; @@ -29,7 +32,10 @@ public AnalysisTool() { resources = new HashMap<>(); } - public AnalysisTool(String dockerId, String params, Map resources) { + public AnalysisTool(String id, String version, boolean defaultVersion, String dockerId, String params, Map resources) { + this.id = id; + this.version = version; + this.defaultVersion = defaultVersion; this.dockerId = dockerId; this.params = params; this.resources = resources; @@ -38,13 +44,43 @@ public AnalysisTool(String dockerId, String params, Map resource @Override public String toString() { final StringBuilder sb = new StringBuilder("AnalysisTool{"); - sb.append("dockerId='").append(dockerId).append('\''); + sb.append("id='").append(id).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append(", defaultVersion=").append(defaultVersion); + sb.append(", dockerId='").append(dockerId).append('\''); sb.append(", params='").append(params).append('\''); sb.append(", resources=").append(resources); sb.append('}'); return sb.toString(); } + public String getId() { + return id; + } + + public AnalysisTool setId(String id) { + this.id = id; + return this; + } + + public String getVersion() { + return version; + } + + public AnalysisTool setVersion(String version) { + this.version = version; + return this; + } + + public boolean isDefaultVersion() { + return defaultVersion; + } + + public AnalysisTool setDefaultVersion(boolean defaultVersion) { + this.defaultVersion = defaultVersion; + return this; + } + public String getDockerId() { return dockerId; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java index 61d33a06ebd..deb5f28ba00 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java @@ -4,8 +4,6 @@ import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; -import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_DEFAULT_VERSION; - public class ExomiserInterpretationAnalysisParams extends ToolParams { public static final String DESCRIPTION = "Exomiser interpretation analysis params"; @@ -13,7 +11,7 @@ public class ExomiserInterpretationAnalysisParams extends ToolParams { @DataField(id = "clinicalAnalysis", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, required = true) private String clinicalAnalysis; - @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION, defaultValue = EXOMISER_DEFAULT_VERSION) + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) private String exomiserVersion; public ExomiserInterpretationAnalysisParams() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java index 15a3954935c..88d5b3876dd 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java @@ -4,17 +4,13 @@ import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; -import java.util.Map; - -import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_DEFAULT_VERSION; - public class ExomiserWrapperParams extends ToolParams { public static final String DESCRIPTION = "Exomiser parameters"; @DataField(id = "sample", description = FieldConstants.EXOMISER_SAMPLE_DESCRIPTION, required = true) private String sample; - @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION, defaultValue = EXOMISER_DEFAULT_VERSION) + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) private String exomiserVersion; @DataField(id = "outdir", description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 75d980fc12d..485e748c3d6 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -118,12 +118,15 @@ analysis: # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used opencgaExtTools: "opencb/opencga-ext-tools" tools: - exomiser-13.1.0: + - id: "exomiser" + version: "13.1" dockerId: "exomiser/exomiser-cli:13.1.0" resources: HG38: "exomiser/2109_hg38.zip" PHENOTYPE: "exomiser/2109_phenotype.zip" - exomiser-14.0.0: + - id: "exomiser" + version: "14.0" + defaultVersion: true dockerId: "exomiser/exomiser-cli:14.0.0" resources: HG38: "exomiser/2402_hg38.zip" From 0c2fe66b0cec21263f6b70345787c2e6d229d9a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 6 Jun 2024 09:00:24 +0200 Subject: [PATCH 035/128] analysis: use the resource version instead of the exomiser version to identify the Exomiser files to download, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java modified: opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java --- .../ExomiserWrapperAnalysisExecutor.java | 21 +++++++++++-------- .../ExomiserInterpretationAnalysisTest.java | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 14753c30e59..3658c1abc67 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -377,19 +377,22 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { // Mutex management to avoid multiple downloadings at the same time // the first to come, download data, others have to wait for - File readyFile = exomiserDataPath.resolve("READY-" + exomiserVersion).toFile(); - File preparingFile = exomiserDataPath.resolve("PREPARING-" + exomiserVersion).toFile(); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, HG38_RESOURCE_KEY); + String resourceVersion = Paths.get(resource).getFileName().toString().split("[_]")[0]; + File readyFile = exomiserDataPath.resolve("READY-" + resourceVersion).toFile(); + File preparingFile = exomiserDataPath.resolve("PREPARING-" + resourceVersion).toFile(); // If all is ready, then return if (readyFile.exists()) { - logger.info("Exomiser {} data is already downloaded, so Exomiser analysis is ready to be executed.", exomiserVersion); + logger.info("Exomiser {} data {} is already downloaded, so Exomiser analysis is ready to be executed.", exomiserVersion, + resourceVersion); return exomiserDataPath; } // If it is preparing, then wait for ready and then return if (preparingFile.exists()) { long startTime = System.currentTimeMillis(); - logger.info("Exomiser {} data is downloading, waiting for it...", exomiserVersion); + logger.info("Exomiser {} data {} is downloading, waiting for it...", exomiserVersion, resourceVersion); while (!readyFile.exists()) { try { Thread.sleep(10000); @@ -400,8 +403,8 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { } long elapsedTime = System.currentTimeMillis() - startTime; if (elapsedTime > 18000000) { - throw new ToolException("Unable to run the Exomiser analysis because of Exomiser " + exomiserVersion + " data is not" - + " ready yet: maximum waiting time exceeded"); + throw new ToolException("Unable to run the Exomiser analysis because of Exomiser " + exomiserVersion + " data " + + resourceVersion + " is not ready yet: maximum waiting time exceeded"); } } logger.info("Exomiser {} data is now downloaded: Exomiser analysis is ready to be executed", exomiserVersion); @@ -413,7 +416,7 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { preparingFile.createNewFile(); } catch (IOException e) { preparingFile.delete(); - throw new ToolException("Error creating the Exomiser " + exomiserVersion + " data directory"); + throw new ToolException("Error creating the Exomiser " + exomiserVersion + " data " + resourceVersion + " directory"); } // Download resources and unzip files @@ -423,14 +426,14 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { } catch (ToolException e) { // If something wrong happened, the preparing file has to be deleted preparingFile.delete(); - throw new ToolException("Something wrong happened when preparing Exomiser " + exomiserVersion + " data", e); + throw new ToolException("Something wrong happened when preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } // Mutex management, signal exomiser data is ready try { readyFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error preparing Exomiser " + exomiserVersion + " data", e); + throw new ToolException("Error preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } preparingFile.delete(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index c24cc6ebe41..1fdbaa4b5d6 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -72,7 +72,8 @@ public void testNormalization() throws NonStandardCompliantSampleField { @Test public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { String exomiserVersion = "13.1"; - assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + exomiserVersion).toFile().exists()); + String resourceVersion = "2109"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_single")); @@ -103,7 +104,8 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE @Test public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { String exomiserVersion = "13.1"; - assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + exomiserVersion).toFile().exists()); + String resourceVersion = "2109"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_family")); From dac02335cdfb62c3f153cdc9cd00a8166a99adde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 6 Jun 2024 09:44:03 +0100 Subject: [PATCH 036/128] storage: Change default enabled normalization extensions. From NONE to ALL. #TASK-5861, #TASK-5448 --- .../storage/core/variant/VariantStorageOptions.java | 2 +- .../storage/core/variant/VariantStoragePipeline.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java index caefbb5260e..cf516491a4a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java @@ -26,7 +26,7 @@ public enum VariantStorageOptions implements ConfigurationOption { TRANSFORM_ISOLATE("transform.isolate", false), // Do not store file in metadata NORMALIZATION_SKIP("normalization.skip", false), // Do not run normalization NORMALIZATION_REFERENCE_GENOME("normalization.referenceGenome"), - NORMALIZATION_EXTENSIONS("normalization.extensions", ParamConstants.NONE), + NORMALIZATION_EXTENSIONS("normalization.extensions", ParamConstants.ALL), DEDUPLICATION_POLICY("deduplication.policy", "maxQual"), DEDUPLICATION_BUFFER_SIZE("deduplication.bufferSize", 100), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 7c0197ac034..459093e5dff 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -417,7 +417,13 @@ protected Task initNormalizer(VariantFileMetadata metadata) th .then(new VariantSorterTask(100)) // Sort before generating reference blocks .then(new VariantReferenceBlockCreatorTask(metadata.getHeader())); } - if (CollectionUtils.isNotEmpty(enabledExtensions)) { + if (CollectionUtils.isEmpty(enabledExtensions)) { + enabledExtensions = Collections.singleton(NORMALIZATION_EXTENSIONS.defaultValue()); + } + if ((enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.NONE))) { + logger.info("Skip normalization extensions"); + } else { + logger.info("Enable normalization extensions: {}", enabledExtensions); VariantNormalizerExtensionFactory extensionFactory; if (enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.ALL)) { extensionFactory = new VariantNormalizerExtensionFactory(); @@ -425,7 +431,9 @@ protected Task initNormalizer(VariantFileMetadata metadata) th extensionFactory = new VariantNormalizerExtensionFactory(new HashSet<>(enabledExtensions)); } Task extension = extensionFactory.buildExtensions(metadata); - if (extension != null) { + if (extension == null) { + logger.info("No normalization extensions can be used."); + } else { normalizer = normalizer.then(extension); } } From 61938336e296fe25ed59b844ee14b89390a4279e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 6 Jun 2024 09:46:02 +0100 Subject: [PATCH 037/128] server: Add variant/setup operation. #TASK-5861, #TASK-5448 --- .../app/cli/main/OpenCgaCompleter.java | 2 +- .../app/cli/main/OpencgaCliOptionsParser.java | 1 + ...erationsVariantStorageCommandExecutor.java | 44 +++++++++++++++++++ ...perationsVariantStorageCommandOptions.java | 43 ++++++++++++++++++ .../src/main/R/R/Operation-methods.R | 8 ++++ .../rest/clients/VariantOperationClient.java | 16 +++++++ .../src/main/javascript/VariantOperation.js | 10 +++++ .../rest_clients/variant_operation_client.py | 13 ++++++ .../models/variant/VariantSetupParams.java | 2 +- .../VariantOperationWebService.java | 18 ++++++++ 10 files changed, 155 insertions(+), 2 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index d6bd902fff1..8ee3a300f24 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -104,7 +104,7 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List operationsList = asList( "cellbase-configure","variant-aggregate","variant-annotation-delete","variant-annotation-index","variant-annotation-save","variant-configure","variant-delete","variant-family-aggregate","variant-family-index","variant-index","variant-index-launcher","variant-julie-run","variant-metadata-repair","variant-metadata-synchronize","variant-prune","variant-sample-delete","variant-sample-index","variant-sample-index-configure","variant-score-delete","variant-score-index","variant-secondary-annotation-index","variant-secondary-sample-index","configure-variant-secondary-sample-index","variant-secondary-index","variant-secondary-index-delete","variant-stats-delete","variant-stats-index","variant-study-delete") + private List operationsList = asList( "cellbase-configure","variant-aggregate","variant-annotation-delete","variant-annotation-index","variant-annotation-save","variant-configure","variant-delete","variant-family-aggregate","variant-family-index","variant-index","variant-index-launcher","variant-julie-run","variant-metadata-repair","variant-metadata-synchronize","variant-prune","variant-sample-delete","variant-sample-index","variant-sample-index-configure","variant-score-delete","variant-score-index","variant-secondary-annotation-index","variant-secondary-sample-index","configure-variant-secondary-sample-index","variant-secondary-index","variant-secondary-index-delete","variant-setup","variant-stats-delete","variant-stats-index","variant-study-delete") .stream() .map(Candidate::new) .collect(toList()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index 33d53de3bf8..eb7132854c8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -360,6 +360,7 @@ public OpencgaCliOptionsParser() { operationsVariantStorageSubCommands.addCommand("configure-variant-secondary-sample-index", operationsVariantStorageCommandOptions.configureVariantSecondarySampleIndexCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-secondary-index", operationsVariantStorageCommandOptions.secondaryIndexVariantCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-secondary-index-delete", operationsVariantStorageCommandOptions.deleteVariantSecondaryIndexCommandOptions); + operationsVariantStorageSubCommands.addCommand("variant-setup", operationsVariantStorageCommandOptions.setupVariantCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-stats-delete", operationsVariantStorageCommandOptions.deleteVariantStatsCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-stats-index", operationsVariantStorageCommandOptions.indexVariantStatsCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-study-delete", operationsVariantStorageCommandOptions.deleteVariantStudyCommandOptions); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 068b4a3792a..b2b8cb13f58 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -29,12 +29,14 @@ import org.opencb.opencga.core.models.operations.variant.VariantStatsDeleteParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; +import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.VariantConfigureParams; import org.opencb.opencga.core.models.variant.VariantFileDeleteParams; import org.opencb.opencga.core.models.variant.VariantFileIndexJobLauncherParams; import org.opencb.opencga.core.models.variant.VariantIndexParams; import org.opencb.opencga.core.models.variant.VariantPruneParams; import org.opencb.opencga.core.models.variant.VariantSampleDeleteParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.variant.VariantStudyDeleteParams; import org.opencb.opencga.core.response.QueryType; @@ -149,6 +151,9 @@ public void execute() throws Exception { case "variant-secondary-index-delete": queryResponse = deleteVariantSecondaryIndex(); break; + case "variant-setup": + queryResponse = setupVariant(); + break; case "variant-stats-delete": queryResponse = deleteVariantStats(); break; @@ -1082,6 +1087,45 @@ private RestResponse deleteVariantSecondaryIndex() throws Exception { return openCGAClient.getVariantOperationClient().deleteVariantSecondaryIndex(queryParams); } + private RestResponse setupVariant() throws Exception { + logger.debug("Executing setupVariant in Operations - Variant Storage command line"); + + OperationsVariantStorageCommandOptions.SetupVariantCommandOptions commandOptions = operationsVariantStorageCommandOptions.setupVariantCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + VariantSetupParams variantSetupParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/operation/variant/setup")); + return res; + } else if (commandOptions.jsonFile != null) { + variantSetupParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), VariantSetupParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "expectedSamples",commandOptions.expectedSamples, true); + putNestedIfNotNull(beanParams, "expectedFiles",commandOptions.expectedFiles, true); + putNestedIfNotNull(beanParams, "fileType",commandOptions.fileType, true); + putNestedIfNotEmpty(beanParams, "averageFileSize",commandOptions.averageFileSize, true); + putNestedIfNotNull(beanParams, "variantsPerSample",commandOptions.variantsPerSample, true); + putNestedIfNotNull(beanParams, "averageSamplesPerFile",commandOptions.averageSamplesPerFile, true); + putNestedIfNotNull(beanParams, "dataDistribution",commandOptions.dataDistribution, true); + putNestedIfNotNull(beanParams, "normalizeExtensions",commandOptions.normalizeExtensions, true); + + variantSetupParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), VariantSetupParams.class); + } + return openCGAClient.getVariantOperationClient().setupVariant(variantSetupParams, queryParams); + } + private RestResponse deleteVariantStats() throws Exception { logger.debug("Executing deleteVariantStats in Operations - Variant Storage command line"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index babd0e312f2..5a05076d151 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -58,6 +58,7 @@ public class OperationsVariantStorageCommandOptions { public ConfigureVariantSecondarySampleIndexCommandOptions configureVariantSecondarySampleIndexCommandOptions; public SecondaryIndexVariantCommandOptions secondaryIndexVariantCommandOptions; public DeleteVariantSecondaryIndexCommandOptions deleteVariantSecondaryIndexCommandOptions; + public SetupVariantCommandOptions setupVariantCommandOptions; public DeleteVariantStatsCommandOptions deleteVariantStatsCommandOptions; public IndexVariantStatsCommandOptions indexVariantStatsCommandOptions; public DeleteVariantStudyCommandOptions deleteVariantStudyCommandOptions; @@ -92,6 +93,7 @@ public OperationsVariantStorageCommandOptions(CommonCommandOptions commonCommand this.configureVariantSecondarySampleIndexCommandOptions = new ConfigureVariantSecondarySampleIndexCommandOptions(); this.secondaryIndexVariantCommandOptions = new SecondaryIndexVariantCommandOptions(); this.deleteVariantSecondaryIndexCommandOptions = new DeleteVariantSecondaryIndexCommandOptions(); + this.setupVariantCommandOptions = new SetupVariantCommandOptions(); this.deleteVariantStatsCommandOptions = new DeleteVariantStatsCommandOptions(); this.indexVariantStatsCommandOptions = new IndexVariantStatsCommandOptions(); this.deleteVariantStudyCommandOptions = new DeleteVariantStudyCommandOptions(); @@ -1168,6 +1170,47 @@ public class DeleteVariantSecondaryIndexCommandOptions { } + @Parameters(commandNames = {"variant-setup"}, commandDescription ="Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation.") + public class SetupVariantCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--expected-samples"}, description = "Expected number of samples that will be loaded. Used to infer some parameters. This number is only used as a hint. If the real number of samples is different, if it grows beyond expectation, or if , the loader should be able to handle it.", required = false, arity = 1) + public Integer expectedSamples; + + @Parameter(names = {"--expected-files"}, description = "Expected number of files that will be loaded. Used to infer some parameters. This number is only used as a hint. If the real number of files is different, the loader should be able to handle it.", required = false, arity = 1) + public Integer expectedFiles; + + @Parameter(names = {"--file-type"}, description = "Main type of the files that will be loaded. If the dataset contains multiple types of files, provide the one that matches most of the files.", required = false, arity = 1) + public String fileType; + + @Parameter(names = {"--average-file-size"}, description = "Average size of the files that will be loaded. This number is only used as a hint. If the real size of the files is different, the loader should be able to handle it. Accepts units. e.g. 435MB, 2GB, 86KB. If not provided, the value will be inferred from the file type.", required = false, arity = 1) + public String averageFileSize; + + @Parameter(names = {"--variants-per-sample"}, description = "Number of variants per sample. This number is only used as a hint. If the real number of variants per sample is different, the loader should be able to handle it. If not provided, the value will be inferred from the file type.", required = false, arity = 1) + public Integer variantsPerSample; + + @Parameter(names = {"--average-samples-per-file"}, description = "Average number of samples per file. This number is only used as a hint. If the real number of samples per file is different, the loader should be able to handle it. If not provided, the value will be inferred from the expectedSamples and expectedFiles and dataDistribution.", required = false, arity = 1) + public Float averageSamplesPerFile; + + @Parameter(names = {"--data-distribution"}, description = "Data distribution of the files. This parameter is used to infer the number of samples per file.", required = false, arity = 1) + public String dataDistribution; + + @Parameter(names = {"--normalize-extensions"}, description = "List of normalization extensions that will be used to normalize the files.", required = false, arity = 1) + public String normalizeExtensions; + + } + @Parameters(commandNames = {"variant-stats-delete"}, commandDescription ="Deletes the VariantStats of a cohort/s from the database") public class DeleteVariantStatsCommandOptions { diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index ecd586a9ed7..955ee42251d 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -44,6 +44,7 @@ #' | configureVariantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index/configure | study, skipRebuild, body | #' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | #' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, study, samples | +#' | setupVariant | /{apiVersion}/operation/variant/setup | study, body | #' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | @@ -343,6 +344,13 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params subcategory="variant/secondaryIndex", subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/operation/variant/setup: + #' Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param data Variant setup params. + setupVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", + subcategoryId=NULL, action="setup", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/operation/variant/stats/delete: #' Deletes the VariantStats of a cohort/s from the database. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index 9731eb2bcc0..dca0ab56c38 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -35,12 +35,14 @@ import org.opencb.opencga.core.models.operations.variant.VariantStatsDeleteParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; +import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.VariantConfigureParams; import org.opencb.opencga.core.models.variant.VariantFileDeleteParams; import org.opencb.opencga.core.models.variant.VariantFileIndexJobLauncherParams; import org.opencb.opencga.core.models.variant.VariantIndexParams; import org.opencb.opencga.core.models.variant.VariantPruneParams; import org.opencb.opencga.core.models.variant.VariantSampleDeleteParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.variant.VariantStudyDeleteParams; import org.opencb.opencga.core.response.RestResponse; @@ -514,6 +516,20 @@ public RestResponse deleteVariantSecondaryIndex(ObjectMap params) throws Cl return execute("operation", null, "variant/secondaryIndex", null, "delete", params, DELETE, Job.class); } + /** + * Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + * @param data Variant setup params. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse setupVariant(VariantSetupParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("operation", null, "variant", null, "setup", params, POST, VariantSetupResult.class); + } + /** * Deletes the VariantStats of a cohort/s from the database. * @param data Variant stats delete params. diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index 6f2ce1afafa..bed52da1d12 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -404,6 +404,16 @@ export default class VariantOperation extends OpenCGAParentClass { return this._delete("operation", null, "variant/secondaryIndex", null, "delete", params); } + /** Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + * @param {Object} [data] - Variant setup params. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + setupVariant(data, params) { + return this._post("operation", null, "variant", null, "setup", data, params); + } + /** Deletes the VariantStats of a cohort/s from the database * @param {Object} data - Variant stats delete params. * @param {Object} [params] - The Object containing the following optional parameters: diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index e726e9dd2db..bd59da1d268 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -481,6 +481,19 @@ def delete_variant_secondary_index(self, **options): return self._delete(category='operation', resource='delete', subcategory='variant/secondaryIndex', **options) + def setup_variant(self, data=None, **options): + """ + Execute Variant Setup to allow using the variant engine. This setup is + necessary before starting any variant operation. + PATH: /{apiVersion}/operation/variant/setup + + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param dict data: Variant setup params. + """ + + return self._post(category='operation', resource='setup', subcategory='variant', data=data, **options) + def delete_variant_stats(self, data=None, **options): """ Deletes the VariantStats of a cohort/s from the database. diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java index 2c323846d50..c027bbf3df7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -40,7 +40,7 @@ public class VariantSetupParams extends ToolParams { @DataField(description = "Data distribution of the files. This parameter is used to infer the number of samples per file.") private DataDistribution dataDistribution; - @DataField(description = "List of normalization extensions") + @DataField(description = "List of normalization extensions that will be used to normalize the files.") private List normalizeExtensions; public VariantSetupParams(VariantSetupParams params) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java index 77574d73ad9..3c6ef25b077 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java @@ -29,6 +29,7 @@ import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.operations.variant.*; +import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.*; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.core.tools.annotations.Api; @@ -101,6 +102,23 @@ public Response variantConfigure( }); } + @POST + @Path("/variant/setup") + @ApiOperation(value = "Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation.", + response = VariantSetupResult.class) + public Response variantConfigure( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = "Variant setup params") VariantSetupParams params) { + return run(() -> { + StopWatch stopWatch = StopWatch.createStarted(); + VariantSetupResult result = variantManager.variantSetup(study, params, token); + return new DataResult<>() + .setResults(Collections.singletonList(result)) + .setNumResults(1) + .setTime(((int) stopWatch.getTime(TimeUnit.MILLISECONDS))); + }); + } + @POST @Path("/variant/index") @ApiOperation(value = VariantIndexOperationTool.DESCRIPTION, response = Job.class) From dc297304de60e44bf1717d0f2a3d53d2ea379bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Thu, 6 Jun 2024 19:19:18 +0200 Subject: [PATCH 038/128] analysis: set dinamically the ClinVar white list depending on the file exists in the Exomiser data, #TASK-6297, TASK-6255 In addition: - the user is set in the docker command line according to the user/group of the job dir. - the assembly is checked (a little step to support GRCh37) On branch TASK-6255 Changes to be committed: modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java modified: opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java modified: opencga-analysis/src/test/resources/exomiser/application.properties modified: opencga-app/app/analysis/exomiser/application.properties modified: opencga-core/src/main/resources/configuration.yml --- .../ExomiserInterpretationAnalysis.java | 13 +++++-- .../DockerWrapperAnalysisExecutor.java | 9 +++++ .../exomiser/ExomiserWrapperAnalysis.java | 1 + .../ExomiserWrapperAnalysisExecutor.java | 35 ++++++++++++++++--- .../ExomiserInterpretationAnalysisTest.java | 12 ++++--- .../resources/exomiser/application.properties | 2 +- .../analysis/exomiser/application.properties | 2 +- .../src/main/resources/configuration.yml | 2 ++ 8 files changed, 64 insertions(+), 12 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 0a2b4934c0c..6e94d30138c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -275,8 +275,17 @@ private List getPrimaryFindings() throws IOException, StorageEn // Convert variants to clinical variants for (Variant variant : variantResults.getResults()) { ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant); - List exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv - .get(variant.toStringSimple()))); + List exomiserTranscripts = new ArrayList<>(); + if (normalizedToTsv.containsKey(variant.toStringSimple())) { + if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) { + exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple()))); + } else { + logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(), + normalizedToTsv.get(variant.toStringSimple())); + } + } else { + logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple()); + } for (String[] fields : variantTsvMap.get(variant.toStringSimple())) { ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]); Map attributes = getAttributesFromTsv(fields); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index 486571e8f36..c2dd3b5249e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -7,6 +7,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; +import org.opencb.commons.utils.DockerUtils; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.exceptions.ToolException; @@ -91,6 +92,14 @@ protected StringBuilder initCommandLine() { return new StringBuilder("docker run --log-driver=none -a stdin -a stdout -a stderr "); } + protected StringBuilder initCommandLine(String user) { + StringBuilder sb = initCommandLine(); + if (StringUtils.isNotEmpty(user)) { + sb.append("--user ").append(user); + } + return sb; + } + protected Map appendMounts(List> inputFilenames, StringBuilder sb) { Map mountMap = new HashMap<>(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index 63630a0d64d..a4e5dae34c9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -38,6 +38,7 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy { public final static String EXOMISER_PREFIX = "exomiser-"; // It must match the resources key in the exomiser/tool section in the configuration file + public final static String HG19_RESOURCE_KEY = "HG19"; public final static String HG38_RESOURCE_KEY = "HG38"; public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 3658c1abc67..06bfe9752a2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -9,6 +9,7 @@ import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.exec.Command; +import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; @@ -51,6 +52,7 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut // These constants must match in the file application.properties to be replaced private static final String HG38_DATA_VERSION_MARK = "put_here_hg38_data_version"; private static final String PHENOTYPE_DATA_VERSION_MARK = "put_here_phenotype_data_version"; + private static final String CLINVAR_WHITELIST_MARK = "put_here_clinvar_whitelist"; private String studyId; private String sampleId; @@ -59,7 +61,7 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void run() throws ToolException { + public void run() throws ToolException, IOException, CatalogException { // Check HPOs, it will use a set to avoid duplicate HPOs, // and it will check both phenotypes and disorders logger.info("{}: Checking individual for sample {} in study {}", ID, sampleId, studyId); @@ -67,6 +69,16 @@ public void run() throws ToolException { Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, getVariantStorageManager().getCatalogManager(), getToken()); + // Check assembly + String assembly = IndividualQcUtils.getAssembly(studyId, getVariantStorageManager().getCatalogManager(), getToken()); + if (assembly.equalsIgnoreCase("GRCh38")) { + assembly = "hg38"; +// } else if (assembly.equalsIgnoreCase("GRCh37")) { +// assembly = "hg19"; + } else { + throw new ToolException("Invalid assembly '" + assembly + "'. Supported assemblies are: GRCh38"); + } + // Set father and mother if necessary (family ?) if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { individual.setFather(IndividualQcUtils.getIndividualById(studyId, individual.getFather().getId(), @@ -164,10 +176,22 @@ public void run() throws ToolException { try { Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), target.toFile()); + // Update hg38 data version Command cmd = new Command("sed -i \"s/" + HG38_DATA_VERSION_MARK + "/" + getHg38DataVersion() + "/g\" " + target); cmd.run(); + // Update phenotype data version cmd = new Command("sed -i \"s/" + PHENOTYPE_DATA_VERSION_MARK + "/" + getPhenotypeDataVersion() + "/g\" " + target); cmd.run(); + // Update clinvar whitelist + String whitelist; + String clinvarWhitelistFilename = getHg38DataVersion() + "_hg38_clinvar_whitelist.tsv.gz"; + if (Files.exists(exomiserDataPath.resolve(getHg38DataVersion() + "_" + assembly).resolve(clinvarWhitelistFilename))) { + whitelist = "exomiser.hg38.variant-white-list-path=" + clinvarWhitelistFilename; + } else { + whitelist = "#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz"; + } + cmd = new Command("sed -i \"s/" + CLINVAR_WHITELIST_MARK + "/" + whitelist + "/g\" " + target); + cmd.run(); } catch (IOException e) { throw new ToolException("Error copying Exomiser properties file", e); } @@ -181,7 +205,8 @@ public void run() throws ToolException { } // Build the docker command line to run Exomiser - StringBuilder sb = initCommandLine(); + String[] userAndGroup = FileUtils.getUserAndGroup(getOutDir(), true); + StringBuilder sb = initCommandLine(userAndGroup[0] + ":" + userAndGroup[1]); // Append mounts sb.append(" --mount type=bind,source=" + exomiserDataPath + ",target=/data") @@ -197,11 +222,13 @@ public void run() throws ToolException { sb.append(" --ped /jobdir/").append(pedigreeFile.getName()); } sb.append(" --vcf /jobdir/" + vcfPath.getFileName()) - .append(" --assembly hg38 --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) + .append(" --assembly ").append(assembly) + .append(" --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); // Execute command and redirect stdout and stderr to the files logger.info("{}: Docker command line: {}", ID, sb); + System.out.println(sb); runCommandLine(sb.toString()); } @@ -377,7 +404,7 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { // Mutex management to avoid multiple downloadings at the same time // the first to come, download data, others have to wait for - String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, HG38_RESOURCE_KEY); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); String resourceVersion = Paths.get(resource).getFileName().toString().split("[_]")[0]; File readyFile = exomiserDataPath.resolve("READY-" + resourceVersion).toFile(); File preparingFile = exomiserDataPath.resolve("PREPARING-" + resourceVersion).toFile(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index 1fdbaa4b5d6..38d2159c6a9 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -71,8 +71,10 @@ public void testNormalization() throws NonStandardCompliantSampleField { @Test public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { - String exomiserVersion = "13.1"; - String resourceVersion = "2109"; +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); @@ -103,8 +105,10 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE @Test public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { - String exomiserVersion = "13.1"; - String resourceVersion = "2109"; +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); diff --git a/opencga-analysis/src/test/resources/exomiser/application.properties b/opencga-analysis/src/test/resources/exomiser/application.properties index ee367632b7f..ee9a2b7d7ed 100644 --- a/opencga-analysis/src/test/resources/exomiser/application.properties +++ b/opencga-analysis/src/test/resources/exomiser/application.properties @@ -40,6 +40,6 @@ exomiser.data-directory=/data exomiser.hg38.data-version=put_here_hg38_data_version #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +put_here_clinvar_whitelist exomiser.phenotype.data-version=put_here_phenotype_data_version logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/application.properties b/opencga-app/app/analysis/exomiser/application.properties index ee367632b7f..ee9a2b7d7ed 100644 --- a/opencga-app/app/analysis/exomiser/application.properties +++ b/opencga-app/app/analysis/exomiser/application.properties @@ -40,6 +40,6 @@ exomiser.data-directory=/data exomiser.hg38.data-version=put_here_hg38_data_version #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +put_here_clinvar_whitelist exomiser.phenotype.data-version=put_here_phenotype_data_version logging.file.name=/jobdir/exomiser.log diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 485e748c3d6..3c373fce695 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -122,6 +122,7 @@ analysis: version: "13.1" dockerId: "exomiser/exomiser-cli:13.1.0" resources: + HG19: "exomiser/2109_hg19.zip" HG38: "exomiser/2109_hg38.zip" PHENOTYPE: "exomiser/2109_phenotype.zip" - id: "exomiser" @@ -129,6 +130,7 @@ analysis: defaultVersion: true dockerId: "exomiser/exomiser-cli:14.0.0" resources: + HG19: "exomiser/2402_hg19.zip" HG38: "exomiser/2402_hg38.zip" PHENOTYPE: "exomiser/2402_phenotype.zip" execution: From 2077cc5e500d2d3f342e0ec68135bf8fdefd4d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 7 Jun 2024 10:48:25 +0200 Subject: [PATCH 039/128] analysis: improve log messages, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java --- .../ExomiserWrapperAnalysisExecutor.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 06bfe9752a2..7887106f868 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -64,7 +64,7 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut public void run() throws ToolException, IOException, CatalogException { // Check HPOs, it will use a set to avoid duplicate HPOs, // and it will check both phenotypes and disorders - logger.info("{}: Checking individual for sample {} in study {}", ID, sampleId, studyId); + logger.info("Checking individual for sample {} in study {}", sampleId, studyId); Set hpos = new HashSet<>(); Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, getVariantStorageManager().getCatalogManager(), getToken()); @@ -89,7 +89,7 @@ public void run() throws ToolException, IOException, CatalogException { getVariantStorageManager().getCatalogManager(), getToken())); } - logger.info("{}: Individual found: {}", ID, individual.getId()); + logger.info("Individual found: {}", individual.getId()); if (CollectionUtils.isNotEmpty(individual.getPhenotypes())) { for (Phenotype phenotype : individual.getPhenotypes()) { if (phenotype.getId().startsWith("HP:")) { @@ -109,7 +109,7 @@ public void run() throws ToolException, IOException, CatalogException { throw new ToolException("Missing phenotypes, i.e. HPO terms, for individual/sample (" + individual.getId() + "/" + sampleId + ")"); } - logger.info("{}: Getting HPO for individual {}: {}", ID, individual.getId(), StringUtils.join(hpos, ",")); + logger.info("Getting HPO for individual {}: {}", individual.getId(), StringUtils.join(hpos, ",")); List samples = new ArrayList<>(); samples.add(sampleId); @@ -150,8 +150,8 @@ public void run() throws ToolException, IOException, CatalogException { QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id,studies.samples"); - logger.info("{}: Exomiser exports variants using the query: {}", ID, query.toJson()); - logger.info("{}: Exomiser exports variants using the query options: {}", ID, queryOptions.toJson()); + logger.info("Exomiser exports variants using the query: {}", query.toJson()); + logger.info("Exomiser exports variants using the query options: {}", queryOptions.toJson()); try { getVariantStorageManager().exportData(vcfPath.toString(), VariantWriterFactory.VariantOutputFormat.VCF_GZ, null, query, @@ -206,7 +206,9 @@ public void run() throws ToolException, IOException, CatalogException { // Build the docker command line to run Exomiser String[] userAndGroup = FileUtils.getUserAndGroup(getOutDir(), true); - StringBuilder sb = initCommandLine(userAndGroup[0] + ":" + userAndGroup[1]); + String dockerUser = userAndGroup[0] + ":" + userAndGroup[1]; + logger.info("Docker user: {}", dockerUser); + StringBuilder sb = initCommandLine(dockerUser); // Append mounts sb.append(" --mount type=bind,source=" + exomiserDataPath + ",target=/data") @@ -227,8 +229,7 @@ public void run() throws ToolException, IOException, CatalogException { .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); // Execute command and redirect stdout and stderr to the files - logger.info("{}: Docker command line: {}", ID, sb); - System.out.println(sb); + logger.info("Docker command line: {}", sb); runCommandLine(sb.toString()); } @@ -516,7 +517,7 @@ private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws } else { url = getConfiguration().getAnalysis().getResourceUrl() + resource; } - logger.info("{}: Downloading Exomiser data: {} in {}", ID, url, exomiserDataPath); + logger.info("Downloading Exomiser data: {} in {}", url, exomiserDataPath); try { ResourceUtils.downloadThirdParty(new URL(url), exomiserDataPath); filename = Paths.get(url).getFileName().toString(); @@ -539,7 +540,7 @@ private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws } // Free disk space - logger.info("{}: Deleting Exomiser data: {}", ID, filename); + logger.info("Deleting Exomiser data: {}", filename); exomiserDataPath.resolve(filename).toFile().delete(); } From 53c4c272deface3998b29f0b39033a7c6b623890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 7 Jun 2024 11:36:55 +0200 Subject: [PATCH 040/128] core: improve description of the Exomiser version parameter, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java --- .../java/org/opencb/opencga/core/api/FieldConstants.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 89ffedf5483..95fc16c6838 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -45,7 +45,7 @@ public class FieldConstants { public static final String ORGANIZATION_ADMINS_DESCRIPTION = "Administrative users of the organization."; public static final String ORGANIZATION_PROJECTS_DESCRIPTION = "Projects the organization holds."; public static final String ORGANIZATION_NOTES_DESCRIPTION = "Notes of organization scope."; -// public static final String ORGANIZATION_AUTHENTICATION_ORIGINS_DESCRIPTION = "Authentication origins used by the organization. This " + // public static final String ORGANIZATION_AUTHENTICATION_ORIGINS_DESCRIPTION = "Authentication origins used by the organization. This " // + "contains all the configuration necessary to be able to communicate with the external authentication origins."; public static final String ORGANIZATION_CONFIGURATION_DESCRIPTION = "Organization configuration information."; public static final String ORGANIZATION_INTERNAL_DESCRIPTION = "Organization internal information."; @@ -497,7 +497,7 @@ public class FieldConstants { public static final String HRDETECT_CNV_QUERY_DESCRIPTION = "CNV query"; public static final String HRDETECT_INDEL_QUERY_DESCRIPTION = "INDEL query"; public static final String HRDETECT_SNV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV3 input for" - + " HRDetect."; + + " HRDetect."; public static final String HRDETECT_SNV8_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV8 input for" + " HRDetect."; public static final String HRDETECT_SV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SV3 input for" @@ -526,5 +526,7 @@ public class FieldConstants { // Exomiser public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID."; public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID."; - public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version."; + public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version in the format X.Y where X is the major version and Y the" + + " minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration" + + " file to view all installed Exomiser versions and identify the default version."; } From a7702361ec11b038fb9f39d090bd18e50132bc61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 7 Jun 2024 11:44:07 +0200 Subject: [PATCH 041/128] client: generate clients, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java modified: opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java modified: opencga-client/src/main/R/R/Admin-methods.R modified: opencga-client/src/main/R/R/Alignment-methods.R modified: opencga-client/src/main/R/R/AllGenerics.R modified: opencga-client/src/main/R/R/Clinical-methods.R modified: opencga-client/src/main/R/R/Cohort-methods.R modified: opencga-client/src/main/R/R/Family-methods.R modified: opencga-client/src/main/R/R/File-methods.R modified: opencga-client/src/main/R/R/GA4GH-methods.R modified: opencga-client/src/main/R/R/Individual-methods.R modified: opencga-client/src/main/R/R/Job-methods.R modified: opencga-client/src/main/R/R/Meta-methods.R modified: opencga-client/src/main/R/R/Operation-methods.R modified: opencga-client/src/main/R/R/Organization-methods.R modified: opencga-client/src/main/R/R/Panel-methods.R modified: opencga-client/src/main/R/R/Project-methods.R modified: opencga-client/src/main/R/R/Sample-methods.R modified: opencga-client/src/main/R/R/Study-methods.R modified: opencga-client/src/main/R/R/User-methods.R modified: opencga-client/src/main/R/R/Variant-methods.R modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java modified: opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java modified: opencga-client/src/main/javascript/Admin.js modified: opencga-client/src/main/javascript/Alignment.js modified: opencga-client/src/main/javascript/ClinicalAnalysis.js modified: opencga-client/src/main/javascript/Cohort.js modified: opencga-client/src/main/javascript/DiseasePanel.js modified: opencga-client/src/main/javascript/Family.js modified: opencga-client/src/main/javascript/File.js modified: opencga-client/src/main/javascript/GA4GH.js modified: opencga-client/src/main/javascript/Individual.js modified: opencga-client/src/main/javascript/Job.js modified: opencga-client/src/main/javascript/Meta.js modified: opencga-client/src/main/javascript/Organization.js modified: opencga-client/src/main/javascript/Project.js modified: opencga-client/src/main/javascript/Sample.js modified: opencga-client/src/main/javascript/Study.js modified: opencga-client/src/main/javascript/User.js modified: opencga-client/src/main/javascript/Variant.js modified: opencga-client/src/main/javascript/VariantOperation.js modified: opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/family_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/file_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/job_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/project_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/study_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/user_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py modified: opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py --- .../app/cli/main/OpenCgaCompleter.java | 2 +- .../app/cli/main/OpencgaCliOptionsParser.java | 2 +- .../AnalysisClinicalCommandOptions.java | 2466 ++++++++--------- .../AnalysisVariantCommandOptions.java | 2426 ++++++++-------- opencga-client/src/main/R/R/Admin-methods.R | 2 +- .../src/main/R/R/Alignment-methods.R | 2 +- opencga-client/src/main/R/R/AllGenerics.R | 20 +- .../src/main/R/R/Clinical-methods.R | 4 +- opencga-client/src/main/R/R/Cohort-methods.R | 4 +- opencga-client/src/main/R/R/Family-methods.R | 4 +- opencga-client/src/main/R/R/File-methods.R | 4 +- opencga-client/src/main/R/R/GA4GH-methods.R | 2 +- .../src/main/R/R/Individual-methods.R | 4 +- opencga-client/src/main/R/R/Job-methods.R | 4 +- opencga-client/src/main/R/R/Meta-methods.R | 2 +- .../src/main/R/R/Operation-methods.R | 2 +- .../src/main/R/R/Organization-methods.R | 4 +- opencga-client/src/main/R/R/Panel-methods.R | 4 +- opencga-client/src/main/R/R/Project-methods.R | 2 +- opencga-client/src/main/R/R/Sample-methods.R | 4 +- opencga-client/src/main/R/R/Study-methods.R | 4 +- opencga-client/src/main/R/R/User-methods.R | 2 +- opencga-client/src/main/R/R/Variant-methods.R | 2 +- .../client/rest/clients/AdminClient.java | 2 +- .../client/rest/clients/AlignmentClient.java | 2 +- .../rest/clients/ClinicalAnalysisClient.java | 2 +- .../client/rest/clients/CohortClient.java | 2 +- .../rest/clients/DiseasePanelClient.java | 2 +- .../client/rest/clients/FamilyClient.java | 2 +- .../client/rest/clients/FileClient.java | 2 +- .../client/rest/clients/GA4GHClient.java | 2 +- .../client/rest/clients/IndividualClient.java | 2 +- .../client/rest/clients/JobClient.java | 2 +- .../client/rest/clients/MetaClient.java | 2 +- .../rest/clients/OrganizationClient.java | 2 +- .../client/rest/clients/ProjectClient.java | 2 +- .../client/rest/clients/SampleClient.java | 2 +- .../client/rest/clients/StudyClient.java | 2 +- .../client/rest/clients/UserClient.java | 2 +- .../client/rest/clients/VariantClient.java | 2 +- .../rest/clients/VariantOperationClient.java | 2 +- opencga-client/src/main/javascript/Admin.js | 2 +- .../src/main/javascript/Alignment.js | 2 +- .../src/main/javascript/ClinicalAnalysis.js | 2 +- opencga-client/src/main/javascript/Cohort.js | 2 +- .../src/main/javascript/DiseasePanel.js | 2 +- opencga-client/src/main/javascript/Family.js | 2 +- opencga-client/src/main/javascript/File.js | 2 +- opencga-client/src/main/javascript/GA4GH.js | 2 +- .../src/main/javascript/Individual.js | 2 +- opencga-client/src/main/javascript/Job.js | 2 +- opencga-client/src/main/javascript/Meta.js | 2 +- .../src/main/javascript/Organization.js | 2 +- opencga-client/src/main/javascript/Project.js | 2 +- opencga-client/src/main/javascript/Sample.js | 2 +- opencga-client/src/main/javascript/Study.js | 2 +- opencga-client/src/main/javascript/User.js | 2 +- opencga-client/src/main/javascript/Variant.js | 2 +- .../src/main/javascript/VariantOperation.js | 2 +- .../pyopencga/rest_clients/admin_client.py | 2 +- .../rest_clients/alignment_client.py | 2 +- .../rest_clients/clinical_analysis_client.py | 2 +- .../pyopencga/rest_clients/cohort_client.py | 2 +- .../rest_clients/disease_panel_client.py | 2 +- .../pyopencga/rest_clients/family_client.py | 2 +- .../pyopencga/rest_clients/file_client.py | 2 +- .../pyopencga/rest_clients/ga4gh_client.py | 2 +- .../rest_clients/individual_client.py | 2 +- .../pyopencga/rest_clients/job_client.py | 2 +- .../pyopencga/rest_clients/meta_client.py | 2 +- .../rest_clients/organization_client.py | 2 +- .../pyopencga/rest_clients/project_client.py | 2 +- .../pyopencga/rest_clients/sample_client.py | 2 +- .../pyopencga/rest_clients/study_client.py | 2 +- .../pyopencga/rest_clients/user_client.py | 2 +- .../pyopencga/rest_clients/variant_client.py | 2 +- .../rest_clients/variant_operation_client.py | 2 +- 77 files changed, 2540 insertions(+), 2540 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index e83115c8d1c..cdeae77bdd4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2024-05-23 OpenCB +* Copyright 2015-2024-06-07 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index a00af1b0ca1..bc0129698f7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2024-05-23 OpenCB +* Copyright 2015-2024-06-07 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index 43b5345b02b..d0169a28ea4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -14,14 +14,14 @@ /* - * WARNING: AUTOGENERATED CODE - * - * This code was generated by a tool. - * - * Manual changes to this file may cause unexpected behavior in your application. - * Manual changes to this file will be overwritten if the code is regenerated. - * - */ +* WARNING: AUTOGENERATED CODE +* +* This code was generated by a tool. +* +* Manual changes to this file may cause unexpected behavior in your application. +* Manual changes to this file will be overwritten if the code is regenerated. +* +*/ /** * This class contains methods for the Analysis - Clinical command line. @@ -30,48 +30,48 @@ @Parameters(commandNames = {"clinical"}, commandDescription = "Analysis - Clinical commands") public class AnalysisClinicalCommandOptions { - public JCommander jCommander; - public CommonCommandOptions commonCommandOptions; - - public UpdateAclCommandOptions updateAclCommandOptions; - public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; - public UpdateClinicalConfigurationCommandOptions updateClinicalConfigurationCommandOptions; - public CreateCommandOptions createCommandOptions; - public DistinctCommandOptions distinctCommandOptions; - public DistinctInterpretationCommandOptions distinctInterpretationCommandOptions; - public SearchInterpretationCommandOptions searchInterpretationCommandOptions; - public InfoInterpretationCommandOptions infoInterpretationCommandOptions; - public RunInterpreterCancerTieringCommandOptions runInterpreterCancerTieringCommandOptions; - public RunInterpreterExomiserCommandOptions runInterpreterExomiserCommandOptions; - public RunInterpreterTeamCommandOptions runInterpreterTeamCommandOptions; - public RunInterpreterTieringCommandOptions runInterpreterTieringCommandOptions; - public RunInterpreterZettaCommandOptions runInterpreterZettaCommandOptions; - public LoadCommandOptions loadCommandOptions; - public AggregationStatsRgaCommandOptions aggregationStatsRgaCommandOptions; - public QueryRgaGeneCommandOptions queryRgaGeneCommandOptions; - public SummaryRgaGeneCommandOptions summaryRgaGeneCommandOptions; - public RunRgaIndexCommandOptions runRgaIndexCommandOptions; - public QueryRgaIndividualCommandOptions queryRgaIndividualCommandOptions; - public SummaryRgaIndividualCommandOptions summaryRgaIndividualCommandOptions; - public QueryRgaVariantCommandOptions queryRgaVariantCommandOptions; - public SummaryRgaVariantCommandOptions summaryRgaVariantCommandOptions; - public SearchCommandOptions searchCommandOptions; - public QueryVariantCommandOptions queryVariantCommandOptions; - public AclCommandOptions aclCommandOptions; - public DeleteCommandOptions deleteCommandOptions; - public UpdateCommandOptions updateCommandOptions; - public UpdateAnnotationSetsAnnotationsCommandOptions updateAnnotationSetsAnnotationsCommandOptions; - public InfoCommandOptions infoCommandOptions; - public CreateInterpretationCommandOptions createInterpretationCommandOptions; - public ClearInterpretationCommandOptions clearInterpretationCommandOptions; - public DeleteInterpretationCommandOptions deleteInterpretationCommandOptions; - public RevertInterpretationCommandOptions revertInterpretationCommandOptions; - public UpdateInterpretationCommandOptions updateInterpretationCommandOptions; - public UpdateReportCommandOptions updateReportCommandOptions; + public JCommander jCommander; + public CommonCommandOptions commonCommandOptions; + + public UpdateAclCommandOptions updateAclCommandOptions; + public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; + public UpdateClinicalConfigurationCommandOptions updateClinicalConfigurationCommandOptions; + public CreateCommandOptions createCommandOptions; + public DistinctCommandOptions distinctCommandOptions; + public DistinctInterpretationCommandOptions distinctInterpretationCommandOptions; + public SearchInterpretationCommandOptions searchInterpretationCommandOptions; + public InfoInterpretationCommandOptions infoInterpretationCommandOptions; + public RunInterpreterCancerTieringCommandOptions runInterpreterCancerTieringCommandOptions; + public RunInterpreterExomiserCommandOptions runInterpreterExomiserCommandOptions; + public RunInterpreterTeamCommandOptions runInterpreterTeamCommandOptions; + public RunInterpreterTieringCommandOptions runInterpreterTieringCommandOptions; + public RunInterpreterZettaCommandOptions runInterpreterZettaCommandOptions; + public LoadCommandOptions loadCommandOptions; + public AggregationStatsRgaCommandOptions aggregationStatsRgaCommandOptions; + public QueryRgaGeneCommandOptions queryRgaGeneCommandOptions; + public SummaryRgaGeneCommandOptions summaryRgaGeneCommandOptions; + public RunRgaIndexCommandOptions runRgaIndexCommandOptions; + public QueryRgaIndividualCommandOptions queryRgaIndividualCommandOptions; + public SummaryRgaIndividualCommandOptions summaryRgaIndividualCommandOptions; + public QueryRgaVariantCommandOptions queryRgaVariantCommandOptions; + public SummaryRgaVariantCommandOptions summaryRgaVariantCommandOptions; + public SearchCommandOptions searchCommandOptions; + public QueryVariantCommandOptions queryVariantCommandOptions; + public AclCommandOptions aclCommandOptions; + public DeleteCommandOptions deleteCommandOptions; + public UpdateCommandOptions updateCommandOptions; + public UpdateAnnotationSetsAnnotationsCommandOptions updateAnnotationSetsAnnotationsCommandOptions; + public InfoCommandOptions infoCommandOptions; + public CreateInterpretationCommandOptions createInterpretationCommandOptions; + public ClearInterpretationCommandOptions clearInterpretationCommandOptions; + public DeleteInterpretationCommandOptions deleteInterpretationCommandOptions; + public RevertInterpretationCommandOptions revertInterpretationCommandOptions; + public UpdateInterpretationCommandOptions updateInterpretationCommandOptions; + public UpdateReportCommandOptions updateReportCommandOptions; public AnalysisClinicalCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { - + this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.updateAclCommandOptions = new UpdateAclCommandOptions(); @@ -109,2269 +109,2267 @@ public AnalysisClinicalCommandOptions(CommonCommandOptions commonCommandOptions, this.revertInterpretationCommandOptions = new RevertInterpretationCommandOptions(); this.updateInterpretationCommandOptions = new UpdateInterpretationCommandOptions(); this.updateReportCommandOptions = new UpdateReportCommandOptions(); - + } - + @Parameters(commandNames = {"acl-update"}, commandDescription ="Update the set of permissions granted for the member") public class UpdateAclCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--members"}, description = "Comma separated list of user or group IDs", required = true, arity = 1) - public String members; - + public String members; + @Parameter(names = {"--action"}, description = "Action to be performed [ADD, SET, REMOVE or RESET].", required = true, arity = 1) - public String action = "ADD"; - + public String action = "ADD"; + @Parameter(names = {"--propagate"}, description = "Propagate permissions to related families, individuals, samples and files", required = false, help = true, arity = 0) - public boolean propagate = false; - + public boolean propagate = false; + @Parameter(names = {"--permissions"}, description = "The body web service permissions parameter", required = true, arity = 1) public String permissions; - + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + } @Parameters(commandNames = {"annotation-sets-load"}, commandDescription ="Load annotation sets from a TSV file") public class LoadAnnotationSetsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) - public String variableSetId; - + public String variableSetId; + @Parameter(names = {"--path"}, description = "Path where the TSV file is located in OpenCGA or where it should be located.", required = true, arity = 1) - public String path; - + public String path; + @Parameter(names = {"--parents"}, description = "Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated).", required = false, help = true, arity = 0) - public boolean parents = false; - + public boolean parents = false; + @Parameter(names = {"--annotation-set-id"}, description = "Annotation set id. If not provided, variableSetId will be used.", required = false, arity = 1) - public String annotationSetId; - + public String annotationSetId; + @Parameter(names = {"--content"}, description = "The body web service content parameter", required = false, arity = 1) public String content; - + } @Parameters(commandNames = {"clinical-configuration-update"}, commandDescription ="Update Clinical Analysis configuration.") public class UpdateClinicalConfigurationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @DynamicParameter(names = {"--interpretation-default-filter"}, description = "The body web service defaultFilter parameter. Use: --interpretation-default-filter key=value", required = false) public java.util.Map interpretationDefaultFilter = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"create"}, commandDescription ="Create a new clinical analysis") public class CreateCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--skip-create-default-interpretation"}, description = "Flag to skip creating and initialise an empty default primary interpretation (Id will be '{clinicalAnalysisId}.1'). This flag is only considered if no Interpretation object is passed.", required = false, help = true, arity = 0) - public boolean skipCreateDefaultInterpretation = false; - + public boolean skipCreateDefaultInterpretation = false; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = true, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--type"}, description = "Enum param allowed values: SINGLE, FAMILY, CANCER, COHORT, AUTOCOMPARATIVE", required = false, arity = 1) public String type; - + @Parameter(names = {"--disorder-id"}, description = "The body web service id parameter", required = false, arity = 1) public String disorderId; - + @Parameter(names = {"--proband-id"}, description = "The body web service id parameter", required = false, arity = 1) public String probandId; - + @Parameter(names = {"--family-id"}, description = "The body web service id parameter", required = false, arity = 1) public String familyId; - + @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) public Boolean panelLock; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--report-title"}, description = "Report title.", required = false, arity = 1) public String reportTitle; - + @Parameter(names = {"--report-overview"}, description = "Report overview.", required = false, arity = 1) public String reportOverview; - + @Parameter(names = {"--report-logo"}, description = "Report logo.", required = false, arity = 1) public String reportLogo; - + @Parameter(names = {"--report-signed-by"}, description = "Indicates who has signed the report.", required = false, arity = 1) public String reportSignedBy; - + @Parameter(names = {"--report-signature"}, description = "Report signature.", required = false, arity = 1) public String reportSignature; - + @Parameter(names = {"--report-date"}, description = "Report date.", required = false, arity = 1) public String reportDate; - + @Parameter(names = {"--request-id"}, description = "The body web service id parameter", required = false, arity = 1) public String requestId; - + @Parameter(names = {"--request-justification"}, description = "The body web service justification parameter", required = false, arity = 1) public String requestJustification; - + @Parameter(names = {"--request-date"}, description = "The body web service date parameter", required = false, arity = 1) public String requestDate; - + @DynamicParameter(names = {"--request-attributes"}, description = "The body web service attributes parameter. Use: --request-attributes key=value", required = false) public java.util.Map requestAttributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--responsible-id"}, description = "The body web service id parameter", required = false, arity = 1) public String responsibleId; - + @Parameter(names = {"--responsible-name"}, description = "The body web service name parameter", required = false, arity = 1) public String responsibleName; - + @Parameter(names = {"--responsible-email"}, description = "The body web service email parameter", required = false, arity = 1) public String responsibleEmail; - + @Parameter(names = {"--responsible-organization"}, description = "The body web service organization parameter", required = false, arity = 1) public String responsibleOrganization; - + @Parameter(names = {"--responsible-department"}, description = "The body web service department parameter", required = false, arity = 1) public String responsibleDepartment; - + @Parameter(names = {"--responsible-address"}, description = "The body web service address parameter", required = false, arity = 1) public String responsibleAddress; - + @Parameter(names = {"--responsible-city"}, description = "The body web service city parameter", required = false, arity = 1) public String responsibleCity; - + @Parameter(names = {"--responsible-postcode"}, description = "The body web service postcode parameter", required = false, arity = 1) public String responsiblePostcode; - + @Parameter(names = {"--interpretation-description"}, description = "The body web service description parameter", required = false, arity = 1) public String interpretationDescription; - + @Parameter(names = {"--interpretation-clinical-analysis-id"}, description = "The body web service clinicalAnalysisId parameter", required = false, arity = 1) public String interpretationClinicalAnalysisId; - + @Parameter(names = {"--interpretation-creation-date"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String interpretationCreationDate; - + @Parameter(names = {"--interpretation-modification-date"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String interpretationModificationDate; - + @Parameter(names = {"--interpretation-locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean interpretationLocked; - + @DynamicParameter(names = {"--interpretation-attributes"}, description = "The body web service attributes parameter. Use: --interpretation-attributes key=value", required = false) public java.util.Map interpretationAttributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--quality-control-summary"}, description = "Enum param allowed values: HIGH, MEDIUM, LOW, DISCARD, NEEDS_REVIEW, UNKNOWN", required = false, arity = 1) public String qualityControlSummary; - + @Parameter(names = {"--quality-control-comments"}, description = "The body web service comments parameter", required = false, arity = 1) public String qualityControlComments; - + @Parameter(names = {"--quality-control-files"}, description = "The body web service files parameter", required = false, arity = 1) public String qualityControlFiles; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--due-date"}, description = "The body web service dueDate parameter", required = false, arity = 1) public String dueDate; - + @Parameter(names = {"--priority-id"}, description = "The body web service id parameter", required = false, arity = 1) public String priorityId; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + } @Parameters(commandNames = {"distinct"}, commandDescription ="Clinical Analysis distinct method") public class DistinctCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Clinical Analysis UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--type"}, description = "Clinical Analysis type", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--disorder"}, description = "Clinical Analysis disorder. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String disorder; - + public String disorder; + @Parameter(names = {"--files"}, description = "Clinical Analysis files", required = false, arity = 1) - public String files; - + public String files; + @Parameter(names = {"--sample"}, description = "Sample associated to the proband or any member of a family", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--individual"}, description = "Proband or any member of a family", required = false, arity = 1) - public String individual; - + public String individual; + @Parameter(names = {"--proband"}, description = "Clinical Analysis proband", required = false, arity = 1) - public String proband; - + public String proband; + @Parameter(names = {"--proband-samples"}, description = "Clinical Analysis proband samples", required = false, arity = 1) - public String probandSamples; - + public String probandSamples; + @Parameter(names = {"--family"}, description = "Clinical Analysis family", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-members"}, description = "Clinical Analysis family members", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-member-samples"}, description = "Clinical Analysis family members samples", required = false, arity = 1) - public String familyMemberSamples; - + public String familyMemberSamples; + @Parameter(names = {"--panels"}, description = "Clinical Analysis panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--locked"}, description = "Locked Clinical Analyses", required = false, arity = 1) - public Boolean locked; - + public Boolean locked; + @Parameter(names = {"--analyst-id"}, description = "Clinical Analysis analyst id", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--priority"}, description = "Clinical Analysis priority", required = false, arity = 1) - public String priority; - + public String priority; + @Parameter(names = {"--flags"}, description = "Clinical Analysis flags", required = false, arity = 1) - public String flags; - + public String flags; + @Parameter(names = {"--creation-date", "--cd"}, description = "Clinical Analysis Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Clinical Analysis Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--due-date"}, description = "Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String dueDate; - + public String dueDate; + @Parameter(names = {"--quality-control-summary"}, description = "Clinical Analysis quality control summary", required = false, arity = 1) - public String qualityControlSummary; - + public String qualityControlSummary; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - + public String annotation; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + @Parameter(names = {"--field"}, description = "Comma separated list of fields for which to obtain the distinct values", required = true, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"interpretation-distinct"}, commandDescription ="Interpretation distinct method") public class DistinctInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) - public String clinicalAnalysisId; - + public String clinicalAnalysisId; + @Parameter(names = {"--analyst-id"}, description = "Analyst ID", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--method-name"}, description = "Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String methodName; - + public String methodName; + @Parameter(names = {"--panels"}, description = "Interpretation panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--primary-findings"}, description = "Interpretation primary findings", required = false, arity = 1) - public String primaryFindings; - + public String primaryFindings; + @Parameter(names = {"--secondary-findings"}, description = "Interpretation secondary findings", required = false, arity = 1) - public String secondaryFindings; - + public String secondaryFindings; + @Parameter(names = {"--creation-date", "--cd"}, description = "Interpretation Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Interpretation Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + @Parameter(names = {"--field"}, description = "Comma separated list of fields for which to obtain the distinct values", required = true, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"interpretation-search"}, commandDescription ="Search clinical interpretations") public class SearchInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--sort"}, description = "Sort the results", required = false, help = true, arity = 0) - public boolean sort = false; - + public boolean sort = false; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) - public String clinicalAnalysisId; - + public String clinicalAnalysisId; + @Parameter(names = {"--analyst-id"}, description = "Analyst ID", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--method-name"}, description = "Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String methodName; - + public String methodName; + @Parameter(names = {"--panels"}, description = "Interpretation panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--primary-findings"}, description = "Interpretation primary findings", required = false, arity = 1) - public String primaryFindings; - + public String primaryFindings; + @Parameter(names = {"--secondary-findings"}, description = "Interpretation secondary findings", required = false, arity = 1) - public String secondaryFindings; - + public String secondaryFindings; + @Parameter(names = {"--creation-date", "--cd"}, description = "Interpretation Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Interpretation Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + } @Parameters(commandNames = {"interpretation-info"}, commandDescription ="Clinical interpretation information") public class InfoInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--interpretations"}, description = "Comma separated list of clinical interpretation IDs up to a maximum of 100", required = true, arity = 1) - public String interpretations; - + public String interpretations; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--version"}, description = "Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not supported if multiple interpretation ids are provided.", required = false, arity = 1) - public String version; - + public String version; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + } @Parameters(commandNames = {"interpreter-cancer-tiering-run"}, commandDescription ="Run cancer tiering interpretation analysis") public class RunInterpreterCancerTieringCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--discarded-variants"}, description = "The body web service discardedVariants parameter", required = false, arity = 1) public String discardedVariants; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"interpreter-exomiser-run"}, commandDescription ="Run exomiser interpretation analysis") public class RunInterpreterExomiserCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated" - + " automatically if not provided.", arity = 1) - public String jobId; - - @Parameter(names = {"--job-description"}, description = "Job description", arity = 1) - public String jobDescription; - - @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", - arity = 1) - public String jobDependsOn; - - @Parameter(names = {"--job-tags"}, description = "Job tags", arity = 1) - public String jobTags; - - @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = true, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = false, arity = 1) public String clinicalAnalysis; - - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", arity = 1) + + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) public String exomiserVersion; - + } @Parameters(commandNames = {"interpreter-team-run"}, commandDescription ="Run TEAM interpretation analysis") public class RunInterpreterTeamCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--panels"}, description = "The body web service panels parameter", required = false, arity = 1) public String panels; - + @Parameter(names = {"--family-segregation"}, description = "The body web service familySegregation parameter", required = false, arity = 1) public String familySegregation; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"interpreter-tiering-run"}, commandDescription ="Run tiering interpretation analysis") public class RunInterpreterTieringCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--panels"}, description = "The body web service panels parameter", required = false, arity = 1) public String panels; - + @Parameter(names = {"--penetrance"}, description = "The body web service penetrance parameter", required = false, arity = 1) public String penetrance; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"interpreter-zetta-run"}, commandDescription ="Run Zetta interpretation analysis") public class RunInterpreterZettaCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; - + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--type"}, description = "The body web service type parameter", required = false, arity = 1) public String type; - + @Parameter(names = {"--body_study"}, description = "The body web service study parameter", required = false, arity = 1) public String bodyStudy; - + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + @Parameter(names = {"--filter"}, description = "The body web service filter parameter", required = false, arity = 1) public String filter; - + @Parameter(names = {"--qual"}, description = "The body web service qual parameter", required = false, arity = 1) public String qual; - + @Parameter(names = {"--file-data"}, description = "The body web service fileData parameter", required = false, arity = 1) public String fileData; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--sample-data"}, description = "The body web service sampleData parameter", required = false, arity = 1) public String sampleData; - + @Parameter(names = {"--sample-annotation"}, description = "The body web service sampleAnnotation parameter", required = false, arity = 1) public String sampleAnnotation; - + @Parameter(names = {"--sample-metadata"}, description = "The body web service sampleMetadata parameter", required = false, arity = 1) public String sampleMetadata; - + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String cohortStatsRef; - + @Parameter(names = {"--cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String cohortStatsAlt; - + @Parameter(names = {"--cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String cohortStatsMaf; - + @Parameter(names = {"--cohort-stats-mgf"}, description = "The body web service cohortStatsMgf parameter", required = false, arity = 1) public String cohortStatsMgf; - + @Parameter(names = {"--cohort-stats-pass"}, description = "The body web service cohortStatsPass parameter", required = false, arity = 1) public String cohortStatsPass; - + @Parameter(names = {"--score"}, description = "The body web service score parameter", required = false, arity = 1) public String score; - + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--family-disorder"}, description = "The body web service familyDisorder parameter", required = false, arity = 1) public String familyDisorder; - + @Parameter(names = {"--family-segregation"}, description = "The body web service familySegregation parameter", required = false, arity = 1) public String familySegregation; - + @Parameter(names = {"--family-members"}, description = "The body web service familyMembers parameter", required = false, arity = 1) public String familyMembers; - + @Parameter(names = {"--family-proband"}, description = "The body web service familyProband parameter", required = false, arity = 1) public String familyProband; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String ct; - + @Parameter(names = {"--xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String xref; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String proteinSubstitution; - + @Parameter(names = {"--conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String conservation; - + @Parameter(names = {"--population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String populationFrequencyAlt; - + @Parameter(names = {"--population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String populationFrequencyRef; - + @Parameter(names = {"--population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String populationFrequencyMaf; - + @Parameter(names = {"--transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String transcriptFlag; - + @Parameter(names = {"--gene-trait-id"}, description = "The body web service geneTraitId parameter", required = false, arity = 1) public String geneTraitId; - + @Parameter(names = {"--go"}, description = "The body web service go parameter", required = false, arity = 1) public String go; - + @Parameter(names = {"--expression"}, description = "The body web service expression parameter", required = false, arity = 1) public String expression; - + @Parameter(names = {"--protein-keyword"}, description = "The body web service proteinKeyword parameter", required = false, arity = 1) public String proteinKeyword; - + @Parameter(names = {"--drug"}, description = "The body web service drug parameter", required = false, arity = 1) public String drug; - + @Parameter(names = {"--functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String functionalScore; - + @Parameter(names = {"--clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String clinical; - + @Parameter(names = {"--clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String clinicalSignificance; - + @Parameter(names = {"--clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean clinicalConfirmedStatus = false; - + @Parameter(names = {"--custom-annotation"}, description = "The body web service customAnnotation parameter", required = false, arity = 1) public String customAnnotation; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String panelModeOfInheritance; - + @Parameter(names = {"--panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String panelConfidence; - + @Parameter(names = {"--panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String panelRoleInCancer; - + @Parameter(names = {"--trait"}, description = "The body web service trait parameter", required = false, arity = 1) public String trait; - + @Parameter(names = {"--primary"}, description = "The body web service primary parameter", required = false, help = true, arity = 0) public boolean primary = false; - + } @Parameters(commandNames = {"load"}, commandDescription ="Load clinical analyses from a file") public class LoadCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + } @Parameters(commandNames = {"rga-aggregation-stats"}, commandDescription ="RGA aggregation stats") public class AggregationStatsRgaCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: clinicalSignificances;type. For nested fields use >>, e.g.: type>>clinicalSignificances;knockoutType", required = true, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"rga-gene-query"}, commandDescription ="Query gene RGA") public class QueryRgaGeneCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--include-individual"}, description = "Include only the comma separated list of individuals to the response", required = false, arity = 1) - public String includeIndividual; - + public String includeIndividual; + @Parameter(names = {"--skip-individual"}, description = "Number of individuals to skip", required = false, arity = 1) - public Integer skipIndividual; - + public Integer skipIndividual; + @Parameter(names = {"--limit-individual"}, description = "Limit number of individuals returned (default: 1000)", required = false, arity = 1) - public Integer limitIndividual; - + public Integer limitIndividual; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-gene-summary"}, commandDescription ="RGA gene summary stats") public class SummaryRgaGeneCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-index-run"}, commandDescription ="Generate Recessive Gene Analysis secondary index") public class RunRgaIndexCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--auxiliar-index"}, description = "Index auxiliar collection to improve performance assuming RGA is completely indexed.", required = false, help = true, arity = 0) - public boolean auxiliarIndex = false; - + public boolean auxiliarIndex = false; + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + } @Parameters(commandNames = {"rga-individual-query"}, commandDescription ="Query individual RGA") public class QueryRgaIndividualCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-individual-summary"}, commandDescription ="RGA individual summary stats") public class SummaryRgaIndividualCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-variant-query"}, commandDescription ="Query variant RGA") public class QueryRgaVariantCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--include-individual"}, description = "Include only the comma separated list of individuals to the response", required = false, arity = 1) - public String includeIndividual; - + public String includeIndividual; + @Parameter(names = {"--skip-individual"}, description = "Number of individuals to skip", required = false, arity = 1) - public Integer skipIndividual; - + public Integer skipIndividual; + @Parameter(names = {"--limit-individual"}, description = "Limit number of individuals returned (default: 1000)", required = false, arity = 1) - public Integer limitIndividual; - + public Integer limitIndividual; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"rga-variant-summary"}, commandDescription ="RGA variant summary stats") public class SummaryRgaVariantCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sample-id"}, description = "Filter by sample id.", required = false, arity = 1) - public String sampleId; - + public String sampleId; + @Parameter(names = {"--individual-id"}, description = "Filter by individual id.", required = false, arity = 1) - public String individualId; - + public String individualId; + @Parameter(names = {"--sex"}, description = "Filter by sex.", required = false, arity = 1) - public String sex; - + public String sex; + @Parameter(names = {"--phenotypes"}, description = "Filter by phenotypes.", required = false, arity = 1) - public String phenotypes; - + public String phenotypes; + @Parameter(names = {"--disorders"}, description = "Filter by disorders.", required = false, arity = 1) - public String disorders; - + public String disorders; + @Parameter(names = {"--num-parents"}, description = "Filter by the number of parents registered.", required = false, arity = 1) - public String numParents; - + public String numParents; + @Parameter(names = {"--gene-id"}, description = "Filter by gene id.", required = false, arity = 1) - public String geneId; - + public String geneId; + @Parameter(names = {"--gene-name"}, description = "Filter by gene name.", required = false, arity = 1) - public String geneName; - + public String geneName; + @Parameter(names = {"--chromosome"}, description = "Filter by chromosome.", required = false, arity = 1) - public String chromosome; - + public String chromosome; + @Parameter(names = {"--start"}, description = "Filter by start position.", required = false, arity = 1) - public String start; - + public String start; + @Parameter(names = {"--end"}, description = "Filter by end position.", required = false, arity = 1) - public String end; - + public String end; + @Parameter(names = {"--transcript-id"}, description = "Filter by transcript id.", required = false, arity = 1) - public String transcriptId; - + public String transcriptId; + @Parameter(names = {"--variants"}, description = "Filter by variant id.", required = false, arity = 1) - public String variants; - + public String variants; + @Parameter(names = {"--db-snps"}, description = "Filter by DB_SNP id.", required = false, arity = 1) - public String dbSnps; - + public String dbSnps; + @Parameter(names = {"--knockout-type"}, description = "Filter by knockout type.", required = false, arity = 1) - public String knockoutType; - + public String knockoutType; + @Parameter(names = {"--filter"}, description = "Filter by filter (PASS, NOT_PASS).", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--type"}, description = "Filter by variant type.", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--clinical-significance"}, description = "Filter by clinical significance.", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--population-frequency"}, description = "Filter by population frequency.", required = false, arity = 1) - public String populationFrequency; - + public String populationFrequency; + @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) - public String consequenceType; - + public String consequenceType; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + } @Parameters(commandNames = {"search"}, commandDescription ="Clinical analysis search.") public class SearchCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) - public boolean flattenAnnotations = false; - + public boolean flattenAnnotations = false; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--id"}, description = "Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--uuid"}, description = "Comma separated list of Clinical Analysis UUIDs up to a maximum of 100", required = false, arity = 1) - public String uuid; - + public String uuid; + @Parameter(names = {"--type"}, description = "Clinical Analysis type", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--disorder"}, description = "Clinical Analysis disorder. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) - public String disorder; - + public String disorder; + @Parameter(names = {"--files"}, description = "Clinical Analysis files", required = false, arity = 1) - public String files; - + public String files; + @Parameter(names = {"--sample"}, description = "Sample associated to the proband or any member of a family", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--individual"}, description = "Proband or any member of a family", required = false, arity = 1) - public String individual; - + public String individual; + @Parameter(names = {"--proband"}, description = "Clinical Analysis proband", required = false, arity = 1) - public String proband; - + public String proband; + @Parameter(names = {"--proband-samples"}, description = "Clinical Analysis proband samples", required = false, arity = 1) - public String probandSamples; - + public String probandSamples; + @Parameter(names = {"--family"}, description = "Clinical Analysis family", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-members"}, description = "Clinical Analysis family members", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-member-samples"}, description = "Clinical Analysis family members samples", required = false, arity = 1) - public String familyMemberSamples; - + public String familyMemberSamples; + @Parameter(names = {"--panels"}, description = "Clinical Analysis panels", required = false, arity = 1) - public String panels; - + public String panels; + @Parameter(names = {"--locked"}, description = "Locked Clinical Analyses", required = false, arity = 1) - public Boolean locked; - + public Boolean locked; + @Parameter(names = {"--analyst-id"}, description = "Clinical Analysis analyst id", required = false, arity = 1) - public String analystId; - + public String analystId; + @Parameter(names = {"--priority"}, description = "Clinical Analysis priority", required = false, arity = 1) - public String priority; - + public String priority; + @Parameter(names = {"--flags"}, description = "Clinical Analysis flags", required = false, arity = 1) - public String flags; - + public String flags; + @Parameter(names = {"--creation-date", "--cd"}, description = "Clinical Analysis Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String creationDate; - + public String creationDate; + @Parameter(names = {"--modification-date", "--md"}, description = "Clinical Analysis Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String modificationDate; - + public String modificationDate; + @Parameter(names = {"--due-date"}, description = "Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) - public String dueDate; - + public String dueDate; + @Parameter(names = {"--quality-control-summary"}, description = "Clinical Analysis quality control summary", required = false, arity = 1) - public String qualityControlSummary; - + public String qualityControlSummary; + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) - public String release; - + public String release; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) - public String status; - + public String status; + @Parameter(names = {"--internal-status"}, description = "Filter by internal status", required = false, arity = 1) - public String internalStatus; - + public String internalStatus; + @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - + public String annotation; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + } @Parameters(commandNames = {"variant-query"}, commandDescription ="Fetch clinical variants") public class QueryVariantCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--approximate-count"}, description = "Get an approximate count, instead of an exact total count. Reduces execution time", required = false, help = true, arity = 0) - public boolean approximateCount = false; - + public boolean approximateCount = false; + @Parameter(names = {"--approximate-count-sampling-size"}, description = "Sampling size to get the approximate count. Larger values increase accuracy but also increase execution time", required = false, arity = 1) - public Integer approximateCountSamplingSize; - + public Integer approximateCountSamplingSize; + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--include-interpretation"}, description = "Interpretation ID to include the fields related to this interpretation", required = false, arity = 1) - public String includeInterpretation; - + public String includeInterpretation; + @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--qual"}, description = "Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4", required = false, arity = 1) - public String qual; - + public String qual; + @Parameter(names = {"--file-data"}, description = "Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file is specified, will use all files from 'file' filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP", required = false, arity = 1) - public String fileData; - + public String fileData; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--sample-data"}, description = "Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will use all samples from 'sample' or 'genotype' filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10", required = false, arity = 1) - public String sampleData; - + public String sampleData; + @Parameter(names = {"--sample-annotation"}, description = "Selects some samples using metadata information from Catalog. e.g. age>20;phenotype=hpo:123,hpo:456;name=smith", required = false, arity = 1) - public String sampleAnnotation; - + public String sampleAnnotation; + @Parameter(names = {"--cohort"}, description = "Select variants with calculated stats for the selected cohorts", required = false, arity = 1) - public String cohort; - + public String cohort; + @Parameter(names = {"--cohort-stats-ref"}, description = "Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsRef; - + public String cohortStatsRef; + @Parameter(names = {"--cohort-stats-alt"}, description = "Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsAlt; - + public String cohortStatsAlt; + @Parameter(names = {"--cohort-stats-maf"}, description = "Minor Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMaf; - + public String cohortStatsMaf; + @Parameter(names = {"--cohort-stats-mgf"}, description = "Minor Genotype Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMgf; - + public String cohortStatsMgf; + @Parameter(names = {"--cohort-stats-pass"}, description = "Filter PASS frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8", required = false, arity = 1) - public String cohortStatsPass; - + public String cohortStatsPass; + @Parameter(names = {"--missing-alleles"}, description = "Number of missing alleles: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingAlleles; - + public String missingAlleles; + @Parameter(names = {"--missing-genotypes"}, description = "Number of missing genotypes: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingGenotypes; - + public String missingGenotypes; + @Parameter(names = {"--score"}, description = "Filter by variant score: [{study:}]{score}[<|>|<=|>=]{number}", required = false, arity = 1) - public String score; - + public String score; + @Parameter(names = {"--family"}, description = "Filter variants where any of the samples from the given family contains the variant (HET or HOM_ALT)", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-disorder"}, description = "Specify the disorder to use for the family segregation", required = false, arity = 1) - public String familyDisorder; - + public String familyDisorder; + @Parameter(names = {"--family-segregation"}, description = "Filter by segregation mode from a given family. Accepted values: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String familySegregation; - + public String familySegregation; + @Parameter(names = {"--family-members"}, description = "Sub set of the members of a given family", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-proband"}, description = "Specify the proband child to use for the family segregation", required = false, arity = 1) - public String familyProband; - + public String familyProband; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) - public String xref; - + public String xref; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--protein-substitution"}, description = "Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant", required = false, arity = 1) - public String proteinSubstitution; - + public String proteinSubstitution; + @Parameter(names = {"--conservation"}, description = "Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1", required = false, arity = 1) - public String conservation; - + public String conservation; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--population-frequency-ref"}, description = "Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyRef; - + public String populationFrequencyRef; + @Parameter(names = {"--population-frequency-maf"}, description = "Population minor allele frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyMaf; - + public String populationFrequencyMaf; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--gene-trait-id"}, description = "List of gene trait association id. e.g. 'umls:C0007222' , 'OMIM:269600'", required = false, arity = 1) - public String geneTraitId; - + public String geneTraitId; + @Parameter(names = {"--go"}, description = "List of GO (Gene Ontology) terms. e.g. 'GO:0002020'", required = false, arity = 1) - public String go; - + public String go; + @Parameter(names = {"--expression"}, description = "List of tissues of interest. e.g. 'lung'", required = false, arity = 1) - public String expression; - + public String expression; + @Parameter(names = {"--protein-keyword"}, description = "List of Uniprot protein variant annotation keywords", required = false, arity = 1) - public String proteinKeyword; - + public String proteinKeyword; + @Parameter(names = {"--drug"}, description = "List of drug names", required = false, arity = 1) - public String drug; - + public String drug; + @Parameter(names = {"--functional-score"}, description = "Functional score: {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , cadd_raw<=0.3", required = false, arity = 1) - public String functionalScore; - + public String functionalScore; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--custom-annotation"}, description = "Custom annotation: {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}", required = false, arity = 1) - public String customAnnotation; - + public String customAnnotation; + @Parameter(names = {"--panel"}, description = "Filter by genes from the given disease panel", required = false, arity = 1) - public String panel; - + public String panel; + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "Filter genes from specific panels that match certain mode of inheritance. Accepted values : [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String panelModeOfInheritance; - + public String panelModeOfInheritance; + @Parameter(names = {"--panel-confidence"}, description = "Filter genes from specific panels that match certain confidence. Accepted values : [ high, medium, low, rejected ]", required = false, arity = 1) - public String panelConfidence; - + public String panelConfidence; + @Parameter(names = {"--panel-role-in-cancer"}, description = "Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, tumorSuppressorGene, fusion ]", required = false, arity = 1) - public String panelRoleInCancer; - + public String panelRoleInCancer; + @Parameter(names = {"--panel-feature-type"}, description = "Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]", required = false, arity = 1) - public String panelFeatureType; - + public String panelFeatureType; + @Parameter(names = {"--panel-intersection"}, description = "Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel.", required = false, help = true, arity = 0) - public boolean panelIntersection = false; - + public boolean panelIntersection = false; + @Parameter(names = {"--trait"}, description = "List of traits, based on ClinVar, HPO, COSMIC, i.e.: IDs, histologies, descriptions,...", required = false, arity = 1) - public String trait; - + public String trait; + } @Parameters(commandNames = {"acl"}, commandDescription ="Returns the acl of the clinical analyses. If member is provided, it will only return the acl for the member.") public class AclCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) - public String clinicalAnalyses; - + public String clinicalAnalyses; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--member"}, description = "User or group ID", required = false, arity = 1) - public String member; - + public String member; + @Parameter(names = {"--silent"}, description = "Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason", required = false, help = true, arity = 0) - public boolean silent = false; - + public boolean silent = false; + } @Parameters(commandNames = {"delete"}, commandDescription ="Delete clinical analyses") public class DeleteCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--force"}, description = "Force deletion if the ClinicalAnalysis contains interpretations or is locked", required = false, help = true, arity = 0) - public boolean force = false; - + public boolean force = false; + @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) - public String clinicalAnalyses; - + public String clinicalAnalyses; + } @Parameters(commandNames = {"update"}, commandDescription ="Update clinical analysis attributes") public class UpdateCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs", required = true, arity = 1) - public String clinicalAnalyses; - + public String clinicalAnalyses; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--analysts-action"}, description = "Action to be performed if the array of analysts is being updated.", required = false, arity = 1) - public String analystsAction = "ADD"; - + public String analystsAction = "ADD"; + @Parameter(names = {"--annotation-sets-action"}, description = "Action to be performed if the array of annotationSets is being updated.", required = false, arity = 1) - public String annotationSetsAction = "ADD"; - + public String annotationSetsAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--type"}, description = "Enum param allowed values: SINGLE, FAMILY, CANCER, COHORT, AUTOCOMPARATIVE", required = false, arity = 1) public String type; - + @Parameter(names = {"--disorder-id"}, description = "The body web service id parameter", required = false, arity = 1) public String disorderId; - + @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) public Boolean panelLock; - + @Parameter(names = {"--proband-id"}, description = "The body web service id parameter", required = false, arity = 1) public String probandId; - + @Parameter(names = {"--family-id"}, description = "The body web service id parameter", required = false, arity = 1) public String familyId; - + @Parameter(names = {"--locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean locked; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--report-title"}, description = "Report title.", required = false, arity = 1) public String reportTitle; - + @Parameter(names = {"--report-overview"}, description = "Report overview.", required = false, arity = 1) public String reportOverview; - + @Parameter(names = {"--report-logo"}, description = "Report logo.", required = false, arity = 1) public String reportLogo; - + @Parameter(names = {"--report-signed-by"}, description = "Indicates who has signed the report.", required = false, arity = 1) public String reportSignedBy; - + @Parameter(names = {"--report-signature"}, description = "Report signature.", required = false, arity = 1) public String reportSignature; - + @Parameter(names = {"--report-date"}, description = "Report date.", required = false, arity = 1) public String reportDate; - + @Parameter(names = {"--request-id"}, description = "The body web service id parameter", required = false, arity = 1) public String requestId; - + @Parameter(names = {"--request-justification"}, description = "The body web service justification parameter", required = false, arity = 1) public String requestJustification; - + @Parameter(names = {"--request-date"}, description = "The body web service date parameter", required = false, arity = 1) public String requestDate; - + @DynamicParameter(names = {"--request-attributes"}, description = "The body web service attributes parameter. Use: --request-attributes key=value", required = false) public java.util.Map requestAttributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--responsible-id"}, description = "The body web service id parameter", required = false, arity = 1) public String responsibleId; - + @Parameter(names = {"--responsible-name"}, description = "The body web service name parameter", required = false, arity = 1) public String responsibleName; - + @Parameter(names = {"--responsible-email"}, description = "The body web service email parameter", required = false, arity = 1) public String responsibleEmail; - + @Parameter(names = {"--responsible-organization"}, description = "The body web service organization parameter", required = false, arity = 1) public String responsibleOrganization; - + @Parameter(names = {"--responsible-department"}, description = "The body web service department parameter", required = false, arity = 1) public String responsibleDepartment; - + @Parameter(names = {"--responsible-address"}, description = "The body web service address parameter", required = false, arity = 1) public String responsibleAddress; - + @Parameter(names = {"--responsible-city"}, description = "The body web service city parameter", required = false, arity = 1) public String responsibleCity; - + @Parameter(names = {"--responsible-postcode"}, description = "The body web service postcode parameter", required = false, arity = 1) public String responsiblePostcode; - + @Parameter(names = {"--quality-control-summary"}, description = "Enum param allowed values: HIGH, MEDIUM, LOW, DISCARD, NEEDS_REVIEW, UNKNOWN", required = false, arity = 1) public String qualityControlSummary; - + @Parameter(names = {"--quality-control-comments"}, description = "The body web service comments parameter", required = false, arity = 1) public String qualityControlComments; - + @Parameter(names = {"--quality-control-files"}, description = "The body web service files parameter", required = false, arity = 1) public String qualityControlFiles; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--due-date"}, description = "The body web service dueDate parameter", required = false, arity = 1) public String dueDate; - + @Parameter(names = {"--priority-id"}, description = "The body web service id parameter", required = false, arity = 1) public String priorityId; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + } @Parameters(commandNames = {"annotation-sets-annotations-update"}, commandDescription ="Update annotations from an annotationSet") public class UpdateAnnotationSetsAnnotationsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) - public String annotationSet; - + public String annotationSet; + @Parameter(names = {"--action"}, description = "Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any.", required = false, arity = 1) - public String action = "ADD"; - + public String action = "ADD"; + } @Parameters(commandNames = {"info"}, commandDescription ="Clinical analysis info") public class InfoCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) - public boolean flattenAnnotations = false; - + public boolean flattenAnnotations = false; + @Parameter(names = {"--clinical-analysis"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) - public boolean deleted = false; - + public boolean deleted = false; + } @Parameters(commandNames = {"interpretation-create"}, commandDescription ="Create a new Interpretation") public class CreateInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study id", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--set-as"}, description = "Set interpretation as", required = false, arity = 1) - public String setAs = "SECONDARY"; - + public String setAs = "SECONDARY"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--clinical-analysis-id"}, description = "The body web service clinicalAnalysisId parameter", required = false, arity = 1) public String clinicalAnalysisId; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--method-name"}, description = "The body web service name parameter", required = false, arity = 1) public String methodName; - + @Parameter(names = {"--method-version"}, description = "The body web service version parameter", required = false, arity = 1) public String methodVersion; - + @Parameter(names = {"--method-commit"}, description = "The body web service commit parameter", required = false, arity = 1) public String methodCommit; - + @Parameter(names = {"--locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean locked; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"interpretation-clear"}, commandDescription ="Clear the fields of the main interpretation of the Clinical Analysis") public class ClearInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--interpretations"}, description = "Interpretation IDs of the Clinical Analysis", required = true, arity = 1) - public String interpretations; - + public String interpretations; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + } @Parameters(commandNames = {"interpretation-delete"}, commandDescription ="Delete interpretation") public class DeleteInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--interpretations"}, description = "Interpretation IDs of the Clinical Analysis", required = true, arity = 1) - public String interpretations; - + public String interpretations; + @Parameter(names = {"--set-as-primary"}, description = "Interpretation id to set as primary from the list of secondaries in case of deleting the actual primary one", required = false, arity = 1) - public String setAsPrimary; - + public String setAsPrimary; + } @Parameters(commandNames = {"interpretation-revert"}, commandDescription ="Revert to a previous interpretation version") public class RevertInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--interpretation"}, description = "Interpretation ID", required = true, arity = 1) - public String interpretation; - + public String interpretation; + @Parameter(names = {"--version"}, description = "Version to revert to", required = true, arity = 1) - public Integer version; - + public Integer version; + } @Parameters(commandNames = {"interpretation-update"}, commandDescription ="Update interpretation fields") public class UpdateInterpretationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--set-as"}, description = "Set interpretation as", required = false, arity = 1) - public String setAs; - + public String setAs; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--interpretation"}, description = "Interpretation ID", required = true, arity = 1) - public String interpretation; - + public String interpretation; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; - + @Parameter(names = {"--method-name"}, description = "The body web service name parameter", required = false, arity = 1) public String methodName; - + @Parameter(names = {"--method-version"}, description = "The body web service version parameter", required = false, arity = 1) public String methodVersion; - + @Parameter(names = {"--method-commit"}, description = "The body web service commit parameter", required = false, arity = 1) public String methodCommit; - + @Parameter(names = {"--creation-date", "--cd"}, description = "The body web service creationDate parameter", required = false, arity = 1) public String creationDate; - + @Parameter(names = {"--modification-date", "--md"}, description = "The body web service modificationDate parameter", required = false, arity = 1) public String modificationDate; - + @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; - + @Parameter(names = {"--locked"}, description = "The body web service locked parameter", required = false, arity = 1) public Boolean locked; - + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"report-update"}, commandDescription ="Update clinical analysis report") public class UpdateReportCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--supporting-evidences-action"}, description = "Action to be performed if the array of supporting evidences is being updated.", required = false, arity = 1) - public String supportingEvidencesAction = "ADD"; - + public String supportingEvidencesAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) - public boolean includeResult = false; - + public boolean includeResult = false; + @Parameter(names = {"--title"}, description = "Report title.", required = false, arity = 1) public String title; - + @Parameter(names = {"--overview"}, description = "Report overview.", required = false, arity = 1) public String overview; - + @Parameter(names = {"--discussion-author"}, description = "The body web service author parameter", required = false, arity = 1) public String discussionAuthor; - + @Parameter(names = {"--discussion-date"}, description = "The body web service date parameter", required = false, arity = 1) public String discussionDate; - + @Parameter(names = {"--discussion-text"}, description = "The body web service text parameter", required = false, arity = 1) public String discussionText; - + @Parameter(names = {"--logo"}, description = "Report logo.", required = false, arity = 1) public String logo; - + @Parameter(names = {"--signed-by"}, description = "Indicates who has signed the report.", required = false, arity = 1) public String signedBy; - + @Parameter(names = {"--signature"}, description = "Report signature.", required = false, arity = 1) public String signature; - + @Parameter(names = {"--date"}, description = "Report date.", required = false, arity = 1) public String date; - + } } \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index a1dba3af944..42663503e8f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -1,21 +1,27 @@ package org.opencb.opencga.app.cli.main.options; -import com.beust.jcommander.*; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.DynamicParameter; +import com.beust.jcommander.ParametersDelegate; import java.util.HashMap; +import java.util.Map; +import java.util.List; -import static org.opencb.opencga.app.cli.GeneralCliOptions.CommonCommandOptions; +import static org.opencb.opencga.app.cli.GeneralCliOptions.*; /* - * WARNING: AUTOGENERATED CODE - * - * This code was generated by a tool. - * - * Manual changes to this file may cause unexpected behavior in your application. - * Manual changes to this file will be overwritten if the code is regenerated. - * - */ +* WARNING: AUTOGENERATED CODE +* +* This code was generated by a tool. +* +* Manual changes to this file may cause unexpected behavior in your application. +* Manual changes to this file will be overwritten if the code is regenerated. +* +*/ /** * This class contains methods for the Analysis - Variant command line. @@ -24,52 +30,52 @@ @Parameters(commandNames = {"variant"}, commandDescription = "Analysis - Variant commands") public class AnalysisVariantCommandOptions { - public JCommander jCommander; - public CommonCommandOptions commonCommandOptions; - - public AggregationStatsCommandOptions aggregationStatsCommandOptions; - public MetadataAnnotationCommandOptions metadataAnnotationCommandOptions; - public QueryAnnotationCommandOptions queryAnnotationCommandOptions; - public RunCircosCommandOptions runCircosCommandOptions; - public DeleteCohortStatsCommandOptions deleteCohortStatsCommandOptions; - public InfoCohortStatsCommandOptions infoCohortStatsCommandOptions; - public RunCohortStatsCommandOptions runCohortStatsCommandOptions; - public RunExomiserCommandOptions runExomiserCommandOptions; - public RunExportCommandOptions runExportCommandOptions; - public GenotypesFamilyCommandOptions genotypesFamilyCommandOptions; - public RunFamilyQcCommandOptions runFamilyQcCommandOptions; - public DeleteFileCommandOptions deleteFileCommandOptions; - public RunGatkCommandOptions runGatkCommandOptions; - public RunGenomePlotCommandOptions runGenomePlotCommandOptions; - public RunGwasCommandOptions runGwasCommandOptions; - public RunHrDetectCommandOptions runHrDetectCommandOptions; - public RunIndexCommandOptions runIndexCommandOptions; - public RunIndividualQcCommandOptions runIndividualQcCommandOptions; - public RunInferredSexCommandOptions runInferredSexCommandOptions; - public QueryKnockoutGeneCommandOptions queryKnockoutGeneCommandOptions; - public QueryKnockoutIndividualCommandOptions queryKnockoutIndividualCommandOptions; - public RunKnockoutCommandOptions runKnockoutCommandOptions; - public RunMendelianErrorCommandOptions runMendelianErrorCommandOptions; - public MetadataCommandOptions metadataCommandOptions; - public QueryMutationalSignatureCommandOptions queryMutationalSignatureCommandOptions; - public RunMutationalSignatureCommandOptions runMutationalSignatureCommandOptions; - public RunPlinkCommandOptions runPlinkCommandOptions; - public QueryCommandOptions queryCommandOptions; - public RunRelatednessCommandOptions runRelatednessCommandOptions; - public RunRvtestsCommandOptions runRvtestsCommandOptions; - public AggregationStatsSampleCommandOptions aggregationStatsSampleCommandOptions; - public RunSampleEligibilityCommandOptions runSampleEligibilityCommandOptions; - public RunSampleQcCommandOptions runSampleQcCommandOptions; - public QuerySampleCommandOptions querySampleCommandOptions; - public RunSampleCommandOptions runSampleCommandOptions; - public QuerySampleStatsCommandOptions querySampleStatsCommandOptions; - public RunSampleStatsCommandOptions runSampleStatsCommandOptions; - public RunStatsExportCommandOptions runStatsExportCommandOptions; - public RunStatsCommandOptions runStatsCommandOptions; + public JCommander jCommander; + public CommonCommandOptions commonCommandOptions; + + public AggregationStatsCommandOptions aggregationStatsCommandOptions; + public MetadataAnnotationCommandOptions metadataAnnotationCommandOptions; + public QueryAnnotationCommandOptions queryAnnotationCommandOptions; + public RunCircosCommandOptions runCircosCommandOptions; + public DeleteCohortStatsCommandOptions deleteCohortStatsCommandOptions; + public InfoCohortStatsCommandOptions infoCohortStatsCommandOptions; + public RunCohortStatsCommandOptions runCohortStatsCommandOptions; + public RunExomiserCommandOptions runExomiserCommandOptions; + public RunExportCommandOptions runExportCommandOptions; + public GenotypesFamilyCommandOptions genotypesFamilyCommandOptions; + public RunFamilyQcCommandOptions runFamilyQcCommandOptions; + public DeleteFileCommandOptions deleteFileCommandOptions; + public RunGatkCommandOptions runGatkCommandOptions; + public RunGenomePlotCommandOptions runGenomePlotCommandOptions; + public RunGwasCommandOptions runGwasCommandOptions; + public RunHrDetectCommandOptions runHrDetectCommandOptions; + public RunIndexCommandOptions runIndexCommandOptions; + public RunIndividualQcCommandOptions runIndividualQcCommandOptions; + public RunInferredSexCommandOptions runInferredSexCommandOptions; + public QueryKnockoutGeneCommandOptions queryKnockoutGeneCommandOptions; + public QueryKnockoutIndividualCommandOptions queryKnockoutIndividualCommandOptions; + public RunKnockoutCommandOptions runKnockoutCommandOptions; + public RunMendelianErrorCommandOptions runMendelianErrorCommandOptions; + public MetadataCommandOptions metadataCommandOptions; + public QueryMutationalSignatureCommandOptions queryMutationalSignatureCommandOptions; + public RunMutationalSignatureCommandOptions runMutationalSignatureCommandOptions; + public RunPlinkCommandOptions runPlinkCommandOptions; + public QueryCommandOptions queryCommandOptions; + public RunRelatednessCommandOptions runRelatednessCommandOptions; + public RunRvtestsCommandOptions runRvtestsCommandOptions; + public AggregationStatsSampleCommandOptions aggregationStatsSampleCommandOptions; + public RunSampleEligibilityCommandOptions runSampleEligibilityCommandOptions; + public RunSampleQcCommandOptions runSampleQcCommandOptions; + public QuerySampleCommandOptions querySampleCommandOptions; + public RunSampleCommandOptions runSampleCommandOptions; + public QuerySampleStatsCommandOptions querySampleStatsCommandOptions; + public RunSampleStatsCommandOptions runSampleStatsCommandOptions; + public RunStatsExportCommandOptions runStatsExportCommandOptions; + public RunStatsCommandOptions runStatsCommandOptions; public AnalysisVariantCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { - + this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); @@ -111,2504 +117,2500 @@ public AnalysisVariantCommandOptions(CommonCommandOptions commonCommandOptions, this.runSampleStatsCommandOptions = new RunSampleStatsCommandOptions(); this.runStatsExportCommandOptions = new RunStatsExportCommandOptions(); this.runStatsCommandOptions = new RunStatsCommandOptions(); - + } - + @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Calculate and fetch aggregation stats") public class AggregationStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--cohort"}, description = "Select variants with calculated stats for the selected cohorts", required = false, arity = 1) - public String cohort; - + public String cohort; + @Parameter(names = {"--cohort-stats-ref"}, description = "Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsRef; - + public String cohortStatsRef; + @Parameter(names = {"--cohort-stats-alt"}, description = "Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsAlt; - + public String cohortStatsAlt; + @Parameter(names = {"--cohort-stats-maf"}, description = "Minor Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMaf; - + public String cohortStatsMaf; + @Parameter(names = {"--cohort-stats-mgf"}, description = "Minor Genotype Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMgf; - + public String cohortStatsMgf; + @Parameter(names = {"--cohort-stats-pass"}, description = "Filter PASS frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8", required = false, arity = 1) - public String cohortStatsPass; - + public String cohortStatsPass; + @Parameter(names = {"--missing-alleles"}, description = "Number of missing alleles: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingAlleles; - + public String missingAlleles; + @Parameter(names = {"--missing-genotypes"}, description = "Number of missing genotypes: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingGenotypes; - + public String missingGenotypes; + @Parameter(names = {"--score"}, description = "Filter by variant score: [{study:}]{score}[<|>|<=|>=]{number}", required = false, arity = 1) - public String score; - + public String score; + @Parameter(names = {"--annotation-exists"}, description = "Return only annotated variants", required = false, arity = 1) - public Boolean annotationExists; - + public Boolean annotationExists; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) - public String xref; - + public String xref; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--protein-substitution"}, description = "Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant", required = false, arity = 1) - public String proteinSubstitution; - + public String proteinSubstitution; + @Parameter(names = {"--conservation"}, description = "Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1", required = false, arity = 1) - public String conservation; - + public String conservation; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--population-frequency-ref"}, description = "Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyRef; - + public String populationFrequencyRef; + @Parameter(names = {"--population-frequency-maf"}, description = "Population minor allele frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyMaf; - + public String populationFrequencyMaf; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--gene-trait-id"}, description = "List of gene trait association id. e.g. 'umls:C0007222' , 'OMIM:269600'", required = false, arity = 1) - public String geneTraitId; - + public String geneTraitId; + @Parameter(names = {"--go"}, description = "List of GO (Gene Ontology) terms. e.g. 'GO:0002020'", required = false, arity = 1) - public String go; - + public String go; + @Parameter(names = {"--expression"}, description = "List of tissues of interest. e.g. 'lung'", required = false, arity = 1) - public String expression; - + public String expression; + @Parameter(names = {"--protein-keyword"}, description = "List of Uniprot protein variant annotation keywords", required = false, arity = 1) - public String proteinKeyword; - + public String proteinKeyword; + @Parameter(names = {"--drug"}, description = "List of drug names", required = false, arity = 1) - public String drug; - + public String drug; + @Parameter(names = {"--functional-score"}, description = "Functional score: {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , cadd_raw<=0.3", required = false, arity = 1) - public String functionalScore; - + public String functionalScore; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--custom-annotation"}, description = "Custom annotation: {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}", required = false, arity = 1) - public String customAnnotation; - + public String customAnnotation; + @Parameter(names = {"--trait"}, description = "List of traits, based on ClinVar, HPO, COSMIC, i.e.: IDs, histologies, descriptions,...", required = false, arity = 1) - public String trait; - + public String trait; + @Parameter(names = {"--field"}, description = "List of facet fields separated by semicolons, e.g.: studies;type. For nested faceted fields use >>, e.g.: chromosome>>type;percentile(gerp)", required = false, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"annotation-metadata"}, commandDescription ="Read variant annotations metadata from any saved versions") public class MetadataAnnotationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--annotation-id"}, description = "Annotation identifier", required = false, arity = 1) - public String annotationId; - + public String annotationId; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + } @Parameters(commandNames = {"annotation-query"}, commandDescription ="Query variant annotations from any saved versions") public class QueryAnnotationCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--annotation-id"}, description = "Annotation identifier", required = false, arity = 1) - public String annotationId; - + public String annotationId; + } @Parameters(commandNames = {"circos-run"}, commandDescription ="Generate a Circos plot for a given sample.") public class RunCircosCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--title"}, description = "The body web service title parameter", required = false, arity = 1) public String title; - + @Parameter(names = {"--density"}, description = "The body web service density parameter", required = false, arity = 1) public String density; - + @DynamicParameter(names = {"--query"}, description = "The body web service query parameter. Use: --query key=value", required = false) public java.util.Map query = new HashMap<>(); //Dynamic parameters must be initialized; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"cohort-stats-delete"}, commandDescription ="Delete cohort variant stats from a cohort.") public class DeleteCohortStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--cohort"}, description = "Cohort ID or UUID", required = false, arity = 1) - public String cohort; - + public String cohort; + } @Parameters(commandNames = {"cohort-stats-info"}, commandDescription ="Read cohort variant stats from list of cohorts.") public class InfoCohortStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--cohort"}, description = "Comma separated list of cohort IDs or UUIDs up to a maximum of 100", required = true, arity = 1) - public String cohort; - + public String cohort; + } @Parameters(commandNames = {"cohort-stats-run"}, commandDescription ="Compute cohort variant stats for the selected list of samples.") public class RunCohortStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--samples"}, description = "The body web service samples parameter", required = false, arity = 1) public String samples; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--sample-annotation"}, description = "The body web service sampleAnnotation parameter", required = false, arity = 1) public String sampleAnnotation; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } - @Parameters(commandNames = {"exomiser-run"}, commandDescription ="The Exomiser is a Java program that finds potential disease-causing" - + " variants from whole-exome or whole-genome sequencing data.") + @Parameters(commandNames = {"exomiser-run"}, commandDescription ="The Exomiser is a Java program that finds potential disease-causing variants from whole-exome or whole-genome sequencing data.") public class RunExomiserCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will" - + " ignore all the other parameters.", arity = 1) + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - - @Parameter(names = {"--study", "-s"}, description = "study", required = true, arity = 1) - public String study; - - @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated" - + " automatically if not provided.", arity = 1) - public String jobId; - - @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", - arity = 1) - public String jobDependsOn; - - @Parameter(names = {"--job-description"}, description = "Job description", arity = 1) - public String jobDescription; - - @Parameter(names = {"--job-tags"}, description = "Job tags", arity = 1) - public String jobTags; - - @Parameter(names = {"--sample"}, description = "Sample ID.", arity = 1) + + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; - - @Parameter(names = {"--exomiser-version"}, description = "Exomiser version.", arity = 1) + + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) public String exomiserVersion; - - @Parameter(names = {"--outdir"}, description = "Output dir for the job.", arity = 1) + + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"export-run"}, commandDescription ="Filter and export variants from the variant storage to a file") public class RunExportCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--type"}, description = "The body web service type parameter", required = false, arity = 1) public String type; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String panelModeOfInheritance; - + @Parameter(names = {"--panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String panelConfidence; - + @Parameter(names = {"--panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String panelRoleInCancer; - + @Parameter(names = {"--panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean panelIntersection = false; - + @Parameter(names = {"--panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String panelFeatureType; - + @Parameter(names = {"--cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String cohortStatsRef; - + @Parameter(names = {"--cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String cohortStatsAlt; - + @Parameter(names = {"--cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String cohortStatsMaf; - + @Parameter(names = {"--ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String ct; - + @Parameter(names = {"--xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String xref; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String proteinSubstitution; - + @Parameter(names = {"--conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String conservation; - + @Parameter(names = {"--population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String populationFrequencyMaf; - + @Parameter(names = {"--population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String populationFrequencyAlt; - + @Parameter(names = {"--population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String populationFrequencyRef; - + @Parameter(names = {"--transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String transcriptFlag; - + @Parameter(names = {"--functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String functionalScore; - + @Parameter(names = {"--clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String clinical; - + @Parameter(names = {"--clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String clinicalSignificance; - + @Parameter(names = {"--clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean clinicalConfirmedStatus = false; - + @Parameter(names = {"--body_project"}, description = "The body web service project parameter", required = false, arity = 1) public String bodyProject; - + @Parameter(names = {"--body_study"}, description = "The body web service study parameter", required = false, arity = 1) public String bodyStudy; - + @Parameter(names = {"--saved-filter"}, description = "The body web service savedFilter parameter", required = false, arity = 1) public String savedFilter; - + @Parameter(names = {"--chromosome"}, description = "The body web service chromosome parameter", required = false, arity = 1) public String chromosome; - + @Parameter(names = {"--reference"}, description = "The body web service reference parameter", required = false, arity = 1) public String reference; - + @Parameter(names = {"--alternate"}, description = "The body web service alternate parameter", required = false, arity = 1) public String alternate; - + @Parameter(names = {"--release"}, description = "The body web service release parameter", required = false, arity = 1) public String release; - + @Parameter(names = {"--include-study"}, description = "The body web service includeStudy parameter", required = false, arity = 1) public String includeStudy; - + @Parameter(names = {"--include-sample"}, description = "The body web service includeSample parameter", required = false, arity = 1) public String includeSample; - + @Parameter(names = {"--include-file"}, description = "The body web service includeFile parameter", required = false, arity = 1) public String includeFile; - + @Parameter(names = {"--include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) public String includeSampleData; - + @Parameter(names = {"--include-sample-id"}, description = "The body web service includeSampleId parameter", required = false, help = true, arity = 0) public boolean includeSampleId = false; - + @Parameter(names = {"--include-genotype"}, description = "The body web service includeGenotype parameter", required = false, help = true, arity = 0) public boolean includeGenotype = false; - + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; - + @Parameter(names = {"--qual"}, description = "The body web service qual parameter", required = false, arity = 1) public String qual; - + @Parameter(names = {"--filter"}, description = "The body web service filter parameter", required = false, arity = 1) public String filter; - + @Parameter(names = {"--file-data"}, description = "The body web service fileData parameter", required = false, arity = 1) public String fileData; - + @Parameter(names = {"--genotype"}, description = "The body web service genotype parameter", required = false, arity = 1) public String genotype; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--sample-limit"}, description = "The body web service sampleLimit parameter", required = false, arity = 1) public Integer sampleLimit; - + @Parameter(names = {"--sample-skip"}, description = "The body web service sampleSkip parameter", required = false, arity = 1) public Integer sampleSkip; - + @Parameter(names = {"--sample-data"}, description = "The body web service sampleData parameter", required = false, arity = 1) public String sampleData; - + @Parameter(names = {"--sample-annotation"}, description = "The body web service sampleAnnotation parameter", required = false, arity = 1) public String sampleAnnotation; - + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--family-members"}, description = "The body web service familyMembers parameter", required = false, arity = 1) public String familyMembers; - + @Parameter(names = {"--family-disorder"}, description = "The body web service familyDisorder parameter", required = false, arity = 1) public String familyDisorder; - + @Parameter(names = {"--family-proband"}, description = "The body web service familyProband parameter", required = false, arity = 1) public String familyProband; - + @Parameter(names = {"--family-segregation"}, description = "The body web service familySegregation parameter", required = false, arity = 1) public String familySegregation; - + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--cohort-stats-pass"}, description = "The body web service cohortStatsPass parameter", required = false, arity = 1) public String cohortStatsPass; - + @Parameter(names = {"--cohort-stats-mgf"}, description = "The body web service cohortStatsMgf parameter", required = false, arity = 1) public String cohortStatsMgf; - + @Parameter(names = {"--missing-alleles"}, description = "The body web service missingAlleles parameter", required = false, arity = 1) public String missingAlleles; - + @Parameter(names = {"--missing-genotypes"}, description = "The body web service missingGenotypes parameter", required = false, arity = 1) public String missingGenotypes; - + @Parameter(names = {"--annotation-exists"}, description = "The body web service annotationExists parameter", required = false, arity = 1) public Boolean annotationExists; - + @Parameter(names = {"--score"}, description = "The body web service score parameter", required = false, arity = 1) public String score; - + @Parameter(names = {"--polyphen"}, description = "The body web service polyphen parameter", required = false, arity = 1) public String polyphen; - + @Parameter(names = {"--sift"}, description = "The body web service sift parameter", required = false, arity = 1) public String sift; - + @Parameter(names = {"--gene-role-in-cancer"}, description = "The body web service geneRoleInCancer parameter", required = false, arity = 1) public String geneRoleInCancer; - + @Parameter(names = {"--gene-trait-id"}, description = "The body web service geneTraitId parameter", required = false, arity = 1) public String geneTraitId; - + @Parameter(names = {"--gene-trait-name"}, description = "The body web service geneTraitName parameter", required = false, arity = 1) public String geneTraitName; - + @Parameter(names = {"--trait"}, description = "The body web service trait parameter", required = false, arity = 1) public String trait; - + @Parameter(names = {"--cosmic"}, description = "The body web service cosmic parameter", required = false, arity = 1) public String cosmic; - + @Parameter(names = {"--clinvar"}, description = "The body web service clinvar parameter", required = false, arity = 1) public String clinvar; - + @Parameter(names = {"--hpo"}, description = "The body web service hpo parameter", required = false, arity = 1) public String hpo; - + @Parameter(names = {"--go"}, description = "The body web service go parameter", required = false, arity = 1) public String go; - + @Parameter(names = {"--expression"}, description = "The body web service expression parameter", required = false, arity = 1) public String expression; - + @Parameter(names = {"--protein-keyword"}, description = "The body web service proteinKeyword parameter", required = false, arity = 1) public String proteinKeyword; - + @Parameter(names = {"--drug"}, description = "The body web service drug parameter", required = false, arity = 1) public String drug; - + @Parameter(names = {"--custom-annotation"}, description = "The body web service customAnnotation parameter", required = false, arity = 1) public String customAnnotation; - + @Parameter(names = {"--unknown-genotype"}, description = "The body web service unknownGenotype parameter", required = false, arity = 1) public String unknownGenotype; - + @Parameter(names = {"--sample-metadata"}, description = "The body web service sampleMetadata parameter", required = false, help = true, arity = 0) public boolean sampleMetadata = false; - + @Parameter(names = {"--sort"}, description = "The body web service sort parameter", required = false, help = true, arity = 0) public boolean sort = false; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--output-file-name"}, description = "The body web service outputFileName parameter", required = false, arity = 1) public String outputFileName; - + @Parameter(names = {"--output-file-format"}, description = "The body web service outputFileFormat parameter", required = false, arity = 1) public String outputFileFormat; - + @Parameter(names = {"--variants-file"}, description = "The body web service variantsFile parameter", required = false, arity = 1) public String variantsFile; - + @Parameter(names = {"--body_include"}, description = "The body web service include parameter", required = false, arity = 1) public String bodyInclude; - + @Parameter(names = {"--body_exclude"}, description = "The body web service exclude parameter", required = false, arity = 1) public String bodyExclude; - + @Parameter(names = {"--limit"}, description = "The body web service limit parameter", required = false, arity = 1) public Integer limit; - + @Parameter(names = {"--skip"}, description = "The body web service skip parameter", required = false, arity = 1) public Integer skip; - + @Parameter(names = {"--summary"}, description = "The body web service summary parameter", required = false, help = true, arity = 0) public boolean summary = false; - + } @Parameters(commandNames = {"family-genotypes"}, commandDescription ="Calculate the possible genotypes for the members of a family") public class GenotypesFamilyCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--family"}, description = "Family id", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis id", required = false, arity = 1) - public String clinicalAnalysis; - + public String clinicalAnalysis; + @Parameter(names = {"--mode-of-inheritance"}, description = "Mode of inheritance", required = true, arity = 1) - public String modeOfInheritance = "MONOALLELIC"; - + public String modeOfInheritance = "MONOALLELIC"; + @Parameter(names = {"--penetrance"}, description = "Penetrance", required = false, arity = 1) - public String penetrance = "COMPLETE"; - + public String penetrance = "COMPLETE"; + @Parameter(names = {"--disorder"}, description = "Disorder id", required = false, arity = 1) - public String disorder; - + public String disorder; + } @Parameters(commandNames = {"family-qc-run"}, commandDescription ="Run quality control (QC) for a given family. It computes the relatedness scores among the family members") public class RunFamilyQcCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--relatedness-method"}, description = "The body web service relatednessMethod parameter", required = false, arity = 1) public String relatednessMethod; - + @Parameter(names = {"--relatedness-maf"}, description = "The body web service relatednessMaf parameter", required = false, arity = 1) public String relatednessMaf; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"file-delete"}, commandDescription =" [DEPRECATED] Use operation/variant/delete") public class DeleteFileCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Files to remove", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--resume"}, description = "Resume a previously failed indexation", required = false, help = true, arity = 0) - public boolean resume = false; - + public boolean resume = false; + } @Parameters(commandNames = {"gatk-run"}, commandDescription ="GATK is a Genome Analysis Toolkit for variant discovery in high-throughput sequencing data. Supported Gatk commands: HaplotypeCaller") public class RunGatkCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @DynamicParameter(names = {"--gatk-params"}, description = "The body web service gatkParams parameter. Use: --gatk-params key=value", required = false) public java.util.Map gatkParams = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"genome-plot-run"}, commandDescription ="Generate a genome plot for a given sample.") public class RunGenomePlotCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; - + @Parameter(names = {"--config-file"}, description = "The body web service configFile parameter", required = false, arity = 1) public String configFile; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"gwas-run"}, commandDescription ="Run a Genome Wide Association Study between two cohorts.") public class RunGwasCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--phenotype"}, description = "The body web service phenotype parameter", required = false, arity = 1) public String phenotype; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--index-score-id"}, description = "The body web service indexScoreId parameter", required = false, arity = 1) public String indexScoreId; - + @Parameter(names = {"--method"}, description = "The body web service method parameter", required = false, arity = 1) public String method; - + @Parameter(names = {"--fisher-mode"}, description = "The body web service fisherMode parameter", required = false, arity = 1) public String fisherMode; - + @Parameter(names = {"--case-cohort"}, description = "The body web service caseCohort parameter", required = false, arity = 1) public String caseCohort; - + @Parameter(names = {"--case-cohort-samples-annotation"}, description = "The body web service caseCohortSamplesAnnotation parameter", required = false, arity = 1) public String caseCohortSamplesAnnotation; - + @Parameter(names = {"--case-cohort-samples"}, description = "The body web service caseCohortSamples parameter", required = false, arity = 1) public String caseCohortSamples; - + @Parameter(names = {"--control-cohort"}, description = "The body web service controlCohort parameter", required = false, arity = 1) public String controlCohort; - + @Parameter(names = {"--control-cohort-samples-annotation"}, description = "The body web service controlCohortSamplesAnnotation parameter", required = false, arity = 1) public String controlCohortSamplesAnnotation; - + @Parameter(names = {"--control-cohort-samples"}, description = "The body web service controlCohortSamples parameter", required = false, arity = 1) public String controlCohortSamples; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"hr-detect-run"}, commandDescription ="Run HRDetect analysis for a given somatic sample.") public class RunHrDetectCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "ID to identify the HRDetect results.", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "Decription for these particular HRDetect results.", required = false, arity = 1) public String description; - + @Parameter(names = {"--sample-id"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) public String sampleId; - + @Parameter(names = {"--snv-fitting-id"}, description = "Mutational signature fitting ID for SNV.", required = false, arity = 1) public String snvFittingId; - + @Parameter(names = {"--sv-fitting-id"}, description = "Mutational signature fitting ID for SV.", required = false, arity = 1) public String svFittingId; - + @Parameter(names = {"--cnv-query"}, description = "CNV query", required = false, arity = 1) public String cnvQuery; - + @Parameter(names = {"--indel-query"}, description = "INDEL query", required = false, arity = 1) public String indelQuery; - + @Parameter(names = {"--snv3custom-name"}, description = "Custom signature name that will be considered as SNV3 input for HRDetect.", required = false, arity = 1) public String snv3CustomName; - + @Parameter(names = {"--snv8custom-name"}, description = "Custom signature name that will be considered as SNV8 input for HRDetect.", required = false, arity = 1) public String snv8CustomName; - + @Parameter(names = {"--sv3custom-name"}, description = "Custom signature name that will be considered as SV3 input for HRDetect.", required = false, arity = 1) public String sv3CustomName; - + @Parameter(names = {"--sv8custom-name"}, description = "Custom signature name that will be considered as SV8 input for HRDetect.", required = false, arity = 1) public String sv8CustomName; - + @Parameter(names = {"--bootstrap"}, description = "Request HRDetect with bootstrap.", required = false, arity = 1) public Boolean bootstrap; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"index-run"}, commandDescription =" [DEPRECATED] Use operation/variant/index") public class RunIndexCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; - + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; - + @Parameter(names = {"--outdir"}, description = "Output directory", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean transform = false; - + @Parameter(names = {"--gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean gvcf = false; - + @Parameter(names = {"--normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean normalizationSkip = false; - + @Parameter(names = {"--reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String referenceGenome; - + @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean family = false; - + @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean somatic = false; - + @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean load = false; - + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) public boolean forceReload = false; - + @Parameter(names = {"--load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String loadSplitData; - + @Parameter(names = {"--load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean loadMultiFileData = false; - + @Parameter(names = {"--load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String loadSampleIndex; - + @Parameter(names = {"--load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String loadArchive; - + @Parameter(names = {"--load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String loadHomRef; - + @Parameter(names = {"--post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String postLoadCheck; - + @Parameter(names = {"--include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String includeGenotypes; - + @Parameter(names = {"--include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String includeSampleData; - + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String merge; - + @Parameter(names = {"--deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String deduplicationPolicy; - + @Parameter(names = {"--calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean calculateStats = false; - + @Parameter(names = {"--aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String aggregated; - + @Parameter(names = {"--aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String aggregationMappingFile; - + @Parameter(names = {"--annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean annotate = false; - + @Parameter(names = {"--annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String annotator; - + @Parameter(names = {"--overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean overwriteAnnotations = false; - + @Parameter(names = {"--index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexSearch = false; - + @Parameter(names = {"--skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean skipIndexedFiles = false; - + } @Parameters(commandNames = {"individual-qc-run"}, commandDescription ="Run quality control (QC) for a given individual. It includes inferred sex and mendelian errors (UDP)") public class RunIndividualQcCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; - + @Parameter(names = {"--sample"}, description = "Sample ID (required when the individual has multiple samples)", required = false, arity = 1) public String sample; - + @Parameter(names = {"--inferred-sex-method"}, description = "Inferred sex method. Valid values: CoverageRatio", required = false, arity = 1) public String inferredSexMethod = "CoverageRatio"; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"inferred-sex-run"}, commandDescription ="Infer sex from chromosome mean coverages.") public class RunInferredSexCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; - + @Parameter(names = {"--sample"}, description = "Sample ID (required when the individual has multiple samples)", required = false, arity = 1) public String sample; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"knockout-gene-query"}, commandDescription ="Fetch values from KnockoutAnalysis result, by genes") public class QueryKnockoutGeneCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job"}, description = "Job ID or UUID", required = false, arity = 1) - public String job; - + public String job; + } @Parameters(commandNames = {"knockout-individual-query"}, commandDescription ="Fetch values from KnockoutAnalysis result, by individuals") public class QueryKnockoutIndividualCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job"}, description = "Job ID or UUID", required = false, arity = 1) - public String job; - + public String job; + } @Parameters(commandNames = {"knockout-run"}, commandDescription ="Obtains the list of knocked out genes for each sample.") public class RunKnockoutCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--consequence-type"}, description = "The body web service consequenceType parameter", required = false, arity = 1) public String consequenceType; - + @Parameter(names = {"--filter"}, description = "The body web service filter parameter", required = false, arity = 1) public String filter; - + @Parameter(names = {"--qual"}, description = "The body web service qual parameter", required = false, arity = 1) public String qual; - + @Parameter(names = {"--skip-genes-file"}, description = "The body web service skipGenesFile parameter", required = false, help = true, arity = 0) public boolean skipGenesFile = false; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + } @Parameters(commandNames = {"mendelian-error-run"}, commandDescription ="Run mendelian error analysis to infer uniparental disomy regions.") public class RunMendelianErrorCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; - + @Parameter(names = {"--individual"}, description = "The body web service individual parameter", required = false, arity = 1) public String individual; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"metadata"}, commandDescription ="") public class MetadataCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--include-study"}, description = "List of studies to include in the result. Accepts 'all' and 'none'.", required = false, arity = 1) - public String includeStudy; - + public String includeStudy; + @Parameter(names = {"--include-file"}, description = "List of files to be returned. Accepts 'all' and 'none'. If undefined, automatically includes files used for filtering. If none, no file is included.", required = false, arity = 1) - public String includeFile; - + public String includeFile; + @Parameter(names = {"--include-sample"}, description = "List of samples to be included in the result. Accepts 'all' and 'none'. If undefined, automatically includes samples used for filtering. If none, no sample is included.", required = false, arity = 1) - public String includeSample; - + public String includeSample; + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + } @Parameters(commandNames = {"mutational-signature-query"}, commandDescription ="Run mutational signature analysis for a given sample. Use context index.") public class QueryMutationalSignatureCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--sample"}, description = "Sample name", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--type"}, description = "Variant type. Valid values: SNV, SV", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--file-data"}, description = "Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file is specified, will use all files from 'file' filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP", required = false, arity = 1) - public String fileData; - + public String fileData; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--qual"}, description = "Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4", required = false, arity = 1) - public String qual; - + public String qual; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--panel"}, description = "Filter by genes from the given disease panel", required = false, arity = 1) - public String panel; - + public String panel; + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "Filter genes from specific panels that match certain mode of inheritance. Accepted values : [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String panelModeOfInheritance; - + public String panelModeOfInheritance; + @Parameter(names = {"--panel-confidence"}, description = "Filter genes from specific panels that match certain confidence. Accepted values : [ high, medium, low, rejected ]", required = false, arity = 1) - public String panelConfidence; - + public String panelConfidence; + @Parameter(names = {"--panel-feature-type"}, description = "Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]", required = false, arity = 1) - public String panelFeatureType; - + public String panelFeatureType; + @Parameter(names = {"--panel-role-in-cancer"}, description = "Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, tumorSuppressorGene, fusion ]", required = false, arity = 1) - public String panelRoleInCancer; - + public String panelRoleInCancer; + @Parameter(names = {"--panel-intersection"}, description = "Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel.", required = false, help = true, arity = 0) - public boolean panelIntersection = false; - + public boolean panelIntersection = false; + @Parameter(names = {"--ms-id"}, description = "Signature ID.", required = false, arity = 1) - public String msId; - + public String msId; + @Parameter(names = {"--ms-description"}, description = "Signature description.", required = false, arity = 1) - public String msDescription; - + public String msDescription; + } @Parameters(commandNames = {"mutational-signature-run"}, commandDescription ="Run mutational signature analysis for a given sample.") public class RunMutationalSignatureCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "Signature ID.", required = false, arity = 1) public String id; - + @Parameter(names = {"--description"}, description = "Signature description.", required = false, arity = 1) public String description; - + @Parameter(names = {"--sample"}, description = "Sample.", required = false, arity = 1) public String sample; - + @Parameter(names = {"--query"}, description = "Signature query in JSON format, e.g: ''{\'sample\':\'NR123456_T\', \'fileData\': \'NR.123456_T_vs_NR.1234567_G.annot.vcf.gz:FILTER=PASS;CLPM<=0;ASMD>=140\'}'.", required = false, arity = 1) public String query; - + @Parameter(names = {"--fit-id"}, description = "Fiiting signature ID", required = false, arity = 1) public String fitId; - + @Parameter(names = {"--fit-method"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String fitMethod = "FitMS"; - + @Parameter(names = {"--fit-n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) public Integer fitNBoot; - + @Parameter(names = {"--fit-sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) public String fitSigVersion = "RefSigv2"; - + @Parameter(names = {"--fit-organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) public String fitOrgan; - + @Parameter(names = {"--fit-threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) public Float fitThresholdPerc = 5f; - + @Parameter(names = {"--fit-threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) public Float fitThresholdPval = 0.05f; - + @Parameter(names = {"--fit-max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) public Integer fitMaxRareSigs = 1; - + @Parameter(names = {"--fit-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) public String fitSignaturesFile; - + @Parameter(names = {"--fit-rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) public String fitRareSignaturesFile; - + @Parameter(names = {"--skip"}, description = "To skip to compute catalogue counts or the signature fitting. Use the following keywords: catalogue, fitting.", required = false, arity = 1) public String skip; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"plink-run"}, commandDescription ="Plink is a whole genome association analysis toolset, designed to perform a range of basic, large-scale analyses.") public class RunPlinkCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @DynamicParameter(names = {"--plink-params"}, description = "The body web service plinkParams parameter. Use: --plink-params key=value", required = false) public java.util.Map plinkParams = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"query"}, commandDescription ="Filter and fetch variants from indexed VCF files in the variant storage") public class QueryCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - + public String include; + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - + public String exclude; + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) - public boolean count = false; - + public boolean count = false; + @Parameter(names = {"--sort"}, description = "Sort the results", required = false, help = true, arity = 0) - public boolean sort = false; - + public boolean sort = false; + @Parameter(names = {"--summary"}, description = "Fast fetch of main variant parameters", required = false, help = true, arity = 0) - public boolean summary = false; - + public boolean summary = false; + @Parameter(names = {"--approximate-count"}, description = "Get an approximate count, instead of an exact total count. Reduces execution time", required = false, help = true, arity = 0) - public boolean approximateCount = false; - + public boolean approximateCount = false; + @Parameter(names = {"--approximate-count-sampling-size"}, description = "Sampling size to get the approximate count. Larger values increase accuracy but also increase execution time", required = false, arity = 1) - public Integer approximateCountSamplingSize; - + public Integer approximateCountSamplingSize; + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) - public String id; - + public String id; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--reference"}, description = "Reference allele", required = false, arity = 1) - public String reference; - + public String reference; + @Parameter(names = {"--alternate"}, description = "Main alternate allele", required = false, arity = 1) - public String alternate; - + public String alternate; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--qual"}, description = "Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4", required = false, arity = 1) - public String qual; - + public String qual; + @Parameter(names = {"--file-data"}, description = "Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file is specified, will use all files from 'file' filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP", required = false, arity = 1) - public String fileData; - + public String fileData; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--genotype"}, description = "Samples with a specific genotype: {samp_1}:{gt_1}(,{gt_n})*(;{samp_n}:{gt_1}(,{gt_n})*)* e.g. HG0097:0/0;HG0098:0/1,1/1. Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT. This will automatically set 'includeSample' parameter when not provided", required = false, arity = 1) - public String genotype; - + public String genotype; + @Parameter(names = {"--sample-data"}, description = "Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will use all samples from 'sample' or 'genotype' filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10", required = false, arity = 1) - public String sampleData; - + public String sampleData; + @Parameter(names = {"--sample-annotation"}, description = "Selects some samples using metadata information from Catalog. e.g. age>20;phenotype=hpo:123,hpo:456;name=smith", required = false, arity = 1) - public String sampleAnnotation; - + public String sampleAnnotation; + @Parameter(names = {"--sample-metadata"}, description = "Return the samples metadata group by study. Sample names will appear in the same order as their corresponding genotypes.", required = false, help = true, arity = 0) - public boolean sampleMetadata = false; - + public boolean sampleMetadata = false; + @Parameter(names = {"--unknown-genotype"}, description = "Returned genotype for unknown genotypes. Common values: [0/0, 0|0, ./.]", required = false, arity = 1) - public String unknownGenotype; - + public String unknownGenotype; + @Parameter(names = {"--sample-limit"}, description = "Limit the number of samples to be included in the result", required = false, arity = 1) - public Integer sampleLimit; - + public Integer sampleLimit; + @Parameter(names = {"--sample-skip"}, description = "Skip some samples from the result. Useful for sample pagination.", required = false, arity = 1) - public Integer sampleSkip; - + public Integer sampleSkip; + @Parameter(names = {"--cohort"}, description = "Select variants with calculated stats for the selected cohorts", required = false, arity = 1) - public String cohort; - + public String cohort; + @Parameter(names = {"--cohort-stats-ref"}, description = "Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsRef; - + public String cohortStatsRef; + @Parameter(names = {"--cohort-stats-alt"}, description = "Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsAlt; - + public String cohortStatsAlt; + @Parameter(names = {"--cohort-stats-maf"}, description = "Minor Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMaf; - + public String cohortStatsMaf; + @Parameter(names = {"--cohort-stats-mgf"}, description = "Minor Genotype Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4", required = false, arity = 1) - public String cohortStatsMgf; - + public String cohortStatsMgf; + @Parameter(names = {"--cohort-stats-pass"}, description = "Filter PASS frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8", required = false, arity = 1) - public String cohortStatsPass; - + public String cohortStatsPass; + @Parameter(names = {"--missing-alleles"}, description = "Number of missing alleles: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingAlleles; - + public String missingAlleles; + @Parameter(names = {"--missing-genotypes"}, description = "Number of missing genotypes: [{study:}]{cohort}[<|>|<=|>=]{number}", required = false, arity = 1) - public String missingGenotypes; - + public String missingGenotypes; + @Parameter(names = {"--score"}, description = "Filter by variant score: [{study:}]{score}[<|>|<=|>=]{number}", required = false, arity = 1) - public String score; - + public String score; + @Parameter(names = {"--family"}, description = "Filter variants where any of the samples from the given family contains the variant (HET or HOM_ALT)", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-disorder"}, description = "Specify the disorder to use for the family segregation", required = false, arity = 1) - public String familyDisorder; - + public String familyDisorder; + @Parameter(names = {"--family-segregation"}, description = "Filter by segregation mode from a given family. Accepted values: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String familySegregation; - + public String familySegregation; + @Parameter(names = {"--family-members"}, description = "Sub set of the members of a given family", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-proband"}, description = "Specify the proband child to use for the family segregation", required = false, arity = 1) - public String familyProband; - + public String familyProband; + @Parameter(names = {"--include-study"}, description = "List of studies to include in the result. Accepts 'all' and 'none'.", required = false, arity = 1) - public String includeStudy; - + public String includeStudy; + @Parameter(names = {"--include-file"}, description = "List of files to be returned. Accepts 'all' and 'none'. If undefined, automatically includes files used for filtering. If none, no file is included.", required = false, arity = 1) - public String includeFile; - + public String includeFile; + @Parameter(names = {"--include-sample"}, description = "List of samples to be included in the result. Accepts 'all' and 'none'. If undefined, automatically includes samples used for filtering. If none, no sample is included.", required = false, arity = 1) - public String includeSample; - + public String includeSample; + @Parameter(names = {"--include-sample-data"}, description = "List of Sample Data keys (i.e. FORMAT column from VCF file) from Sample Data to include in the output. e.g: DP,AD. Accepts 'all' and 'none'.", required = false, arity = 1) - public String includeSampleData; - + public String includeSampleData; + @Parameter(names = {"--include-genotype"}, description = "Include genotypes, apart of other formats defined with includeFormat", required = false, arity = 1) - public String includeGenotype; - + public String includeGenotype; + @Parameter(names = {"--include-sample-id"}, description = "Include sampleId on each result", required = false, arity = 1) - public String includeSampleId; - + public String includeSampleId; + @Parameter(names = {"--annotation-exists"}, description = "Return only annotated variants", required = false, arity = 1) - public Boolean annotationExists; - + public Boolean annotationExists; + @Parameter(names = {"--gene"}, description = "List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter", required = false, arity = 1) - public String gene; - + public String gene; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) - public String xref; - + public String xref; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--protein-substitution"}, description = "Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant", required = false, arity = 1) - public String proteinSubstitution; - + public String proteinSubstitution; + @Parameter(names = {"--conservation"}, description = "Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1", required = false, arity = 1) - public String conservation; - + public String conservation; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--population-frequency-ref"}, description = "Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyRef; - + public String populationFrequencyRef; + @Parameter(names = {"--population-frequency-maf"}, description = "Population minor allele frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyMaf; - + public String populationFrequencyMaf; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--gene-trait-id"}, description = "List of gene trait association id. e.g. 'umls:C0007222' , 'OMIM:269600'", required = false, arity = 1) - public String geneTraitId; - + public String geneTraitId; + @Parameter(names = {"--go"}, description = "List of GO (Gene Ontology) terms. e.g. 'GO:0002020'", required = false, arity = 1) - public String go; - + public String go; + @Parameter(names = {"--expression"}, description = "List of tissues of interest. e.g. 'lung'", required = false, arity = 1) - public String expression; - + public String expression; + @Parameter(names = {"--protein-keyword"}, description = "List of Uniprot protein variant annotation keywords", required = false, arity = 1) - public String proteinKeyword; - + public String proteinKeyword; + @Parameter(names = {"--drug"}, description = "List of drug names", required = false, arity = 1) - public String drug; - + public String drug; + @Parameter(names = {"--functional-score"}, description = "Functional score: {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , cadd_raw<=0.3", required = false, arity = 1) - public String functionalScore; - + public String functionalScore; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--custom-annotation"}, description = "Custom annotation: {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}", required = false, arity = 1) - public String customAnnotation; - + public String customAnnotation; + @Parameter(names = {"--panel"}, description = "Filter by genes from the given disease panel", required = false, arity = 1) - public String panel; - + public String panel; + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "Filter genes from specific panels that match certain mode of inheritance. Accepted values : [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String panelModeOfInheritance; - + public String panelModeOfInheritance; + @Parameter(names = {"--panel-confidence"}, description = "Filter genes from specific panels that match certain confidence. Accepted values : [ high, medium, low, rejected ]", required = false, arity = 1) - public String panelConfidence; - + public String panelConfidence; + @Parameter(names = {"--panel-role-in-cancer"}, description = "Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, tumorSuppressorGene, fusion ]", required = false, arity = 1) - public String panelRoleInCancer; - + public String panelRoleInCancer; + @Parameter(names = {"--panel-feature-type"}, description = "Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]", required = false, arity = 1) - public String panelFeatureType; - + public String panelFeatureType; + @Parameter(names = {"--panel-intersection"}, description = "Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel.", required = false, help = true, arity = 0) - public boolean panelIntersection = false; - + public boolean panelIntersection = false; + @Parameter(names = {"--trait"}, description = "List of traits, based on ClinVar, HPO, COSMIC, i.e.: IDs, histologies, descriptions,...", required = false, arity = 1) - public String trait; - + public String trait; + } @Parameters(commandNames = {"relatedness-run"}, commandDescription ="Compute a score to quantify relatedness between samples.") public class RunRelatednessCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--individuals"}, description = "The body web service individuals parameter", required = false, arity = 1) public String individuals; - + @Parameter(names = {"--samples"}, description = "The body web service samples parameter", required = false, arity = 1) public String samples; - + @Parameter(names = {"--minor-allele-freq"}, description = "The body web service minorAlleleFreq parameter", required = false, arity = 1) public String minorAlleleFreq; - + @Parameter(names = {"--method"}, description = "The body web service method parameter", required = false, arity = 1) public String method; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"rvtests-run"}, commandDescription ="Rvtests is a flexible software package for genetic association studies. Supported RvTests commands: rvtest, vcf2kinship") public class RunRvtestsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @DynamicParameter(names = {"--rvtests-params"}, description = "The body web service rvtestsParams parameter. Use: --rvtests-params key=value", required = false) public java.util.Map rvtestsParams = new HashMap<>(); //Dynamic parameters must be initialized; - + } @Parameters(commandNames = {"sample-aggregation-stats"}, commandDescription ="Calculate and fetch sample aggregation stats") public class AggregationStatsSampleCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) - public String savedFilter; - + public String savedFilter; + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - - @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String study; + + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--sample"}, description = "Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. ", required = false, arity = 1) - public String sample; - + public String sample; + @Parameter(names = {"--genotype"}, description = "Samples with a specific genotype: {samp_1}:{gt_1}(,{gt_n})*(;{samp_n}:{gt_1}(,{gt_n})*)* e.g. HG0097:0/0;HG0098:0/1,1/1. Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT. This will automatically set 'includeSample' parameter when not provided", required = false, arity = 1) - public String genotype; - + public String genotype; + @Parameter(names = {"--sample-annotation"}, description = "Selects some samples using metadata information from Catalog. e.g. age>20;phenotype=hpo:123,hpo:456;name=smith", required = false, arity = 1) - public String sampleAnnotation; - + public String sampleAnnotation; + @Parameter(names = {"--family"}, description = "Filter variants where any of the samples from the given family contains the variant (HET or HOM_ALT)", required = false, arity = 1) - public String family; - + public String family; + @Parameter(names = {"--family-disorder"}, description = "Specify the disorder to use for the family segregation", required = false, arity = 1) - public String familyDisorder; - + public String familyDisorder; + @Parameter(names = {"--family-segregation"}, description = "Filter by segregation mode from a given family. Accepted values: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]", required = false, arity = 1) - public String familySegregation; - + public String familySegregation; + @Parameter(names = {"--family-members"}, description = "Sub set of the members of a given family", required = false, arity = 1) - public String familyMembers; - + public String familyMembers; + @Parameter(names = {"--family-proband"}, description = "Specify the proband child to use for the family segregation", required = false, arity = 1) - public String familyProband; - + public String familyProband; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--field"}, description = "List of facet fields separated by semicolons, e.g.: studies;type. For nested faceted fields use >>, e.g.: chromosome>>type . Accepted values: chromosome, type, genotype, consequenceType, biotype, clinicalSignificance, dp, qual, filter", required = false, arity = 1) - public String field; - + public String field; + } @Parameters(commandNames = {"sample-eligibility-run"}, commandDescription ="Filter samples by a complex query involving metadata and variants data") public class RunSampleEligibilityCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--query"}, description = "The body web service query parameter", required = false, arity = 1) public String query; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--cohort-id"}, description = "The body web service cohortId parameter", required = false, arity = 1) public String cohortId; - + } @Parameters(commandNames = {"sample-qc-run"}, commandDescription ="Run quality control (QC) for a given sample. It includes variant stats, and if the sample is somatic, mutational signature and genome plot are calculated.") public class RunSampleQcCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) public String sample; - + @Parameter(names = {"--vs-id"}, description = "Variant stats ID.", required = false, arity = 1) public String vsId; - + @Parameter(names = {"--vs-description"}, description = "Variant stats description.", required = false, arity = 1) public String vsDescription; - + @Parameter(names = {"--vs-query-id"}, description = "The body web service id parameter", required = false, arity = 1) public String vsQueryId; - + @Parameter(names = {"--vs-query-region"}, description = "The body web service region parameter", required = false, arity = 1) public String vsQueryRegion; - + @Parameter(names = {"--vs-query-gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String vsQueryGene; - + @Parameter(names = {"--vs-query-type"}, description = "The body web service type parameter", required = false, arity = 1) public String vsQueryType; - + @Parameter(names = {"--vs-query-panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String vsQueryPanel; - + @Parameter(names = {"--vs-query-panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String vsQueryPanelModeOfInheritance; - + @Parameter(names = {"--vs-query-panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String vsQueryPanelConfidence; - + @Parameter(names = {"--vs-query-panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String vsQueryPanelRoleInCancer; - + @Parameter(names = {"--vs-query-panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean vsQueryPanelIntersection = false; - + @Parameter(names = {"--vs-query-panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String vsQueryPanelFeatureType; - + @Parameter(names = {"--vs-query-cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String vsQueryCohortStatsRef; - + @Parameter(names = {"--vs-query-cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String vsQueryCohortStatsAlt; - + @Parameter(names = {"--vs-query-cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String vsQueryCohortStatsMaf; - + @Parameter(names = {"--vs-query-ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String vsQueryCt; - + @Parameter(names = {"--vs-query-xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String vsQueryXref; - + @Parameter(names = {"--vs-query-biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String vsQueryBiotype; - + @Parameter(names = {"--vs-query-protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String vsQueryProteinSubstitution; - + @Parameter(names = {"--vs-query-conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String vsQueryConservation; - + @Parameter(names = {"--vs-query-population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String vsQueryPopulationFrequencyMaf; - + @Parameter(names = {"--vs-query-population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String vsQueryPopulationFrequencyAlt; - + @Parameter(names = {"--vs-query-population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String vsQueryPopulationFrequencyRef; - + @Parameter(names = {"--vs-query-transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String vsQueryTranscriptFlag; - + @Parameter(names = {"--vs-query-functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String vsQueryFunctionalScore; - + @Parameter(names = {"--vs-query-clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String vsQueryClinical; - + @Parameter(names = {"--vs-query-clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String vsQueryClinicalSignificance; - + @Parameter(names = {"--vs-query-clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean vsQueryClinicalConfirmedStatus = false; - + @Parameter(names = {"--ms-id"}, description = "Signature ID.", required = false, arity = 1) public String msId; - + @Parameter(names = {"--ms-description"}, description = "Signature description.", required = false, arity = 1) public String msDescription; - + @Parameter(names = {"--ms-query"}, description = "Signature query in JSON format, e.g: ''{\'sample\':\'NR123456_T\', \'fileData\': \'NR.123456_T_vs_NR.1234567_G.annot.vcf.gz:FILTER=PASS;CLPM<=0;ASMD>=140\'}'.", required = false, arity = 1) public String msQuery; - + @Parameter(names = {"--ms-fit-id"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String msFitId; - + @Parameter(names = {"--ms-fit-method"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String msFitMethod = "FitMS"; - + @Parameter(names = {"--ms-fit-n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) public Integer msFitNBoot; - + @Parameter(names = {"--ms-fit-sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) public String msFitSigVersion = "RefSigv2"; - + @Parameter(names = {"--ms-fit-organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) public String msFitOrgan; - + @Parameter(names = {"--ms-fit-threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) public Float msFitThresholdPerc = 5f; - + @Parameter(names = {"--ms-fit-threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) public Float msFitThresholdPval = 0.05f; - + @Parameter(names = {"--ms-fit-max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) public Integer msFitMaxRareSigs = 1; - + @Parameter(names = {"--ms-fit-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) public String msFitSignaturesFile; - + @Parameter(names = {"--ms-fit-rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) public String msFitRareSignaturesFile; - + @Parameter(names = {"--gp-id"}, description = "Genome plot ID.", required = false, arity = 1) public String gpId; - + @Parameter(names = {"--gp-description"}, description = "Genome plot description.", required = false, arity = 1) public String gpDescription; - + @Parameter(names = {"--gp-config-file"}, description = "Genome plot configuration file.", required = false, arity = 1) public String gpConfigFile; - + @Parameter(names = {"--skip"}, description = "Quality control metrics to skip. Valid values are: variant-stats, signature, signature-catalogue, signature-fitting, genome-plot", required = false, arity = 1) public String skip; - + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; - + } @Parameters(commandNames = {"sample-query"}, commandDescription ="Get sample data of a given variant") public class QuerySampleCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - + public Integer limit; + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - + public Integer skip; + @Parameter(names = {"--variant"}, description = "Variant", required = false, arity = 1) - public String variant; - + public String variant; + @Parameter(names = {"--study", "-s"}, description = "Study where all the samples belong to", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--genotype"}, description = "Genotypes that the sample must have to be selected", required = false, arity = 1) - public String genotype; - + public String genotype; + } @Parameters(commandNames = {"sample-run"}, commandDescription ="Get samples given a set of variants") public class RunSampleCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--type"}, description = "The body web service type parameter", required = false, arity = 1) public String type; - + @Parameter(names = {"--panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String panel; - + @Parameter(names = {"--panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String panelModeOfInheritance; - + @Parameter(names = {"--panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String panelConfidence; - + @Parameter(names = {"--panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String panelRoleInCancer; - + @Parameter(names = {"--panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean panelIntersection = false; - + @Parameter(names = {"--panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String panelFeatureType; - + @Parameter(names = {"--cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String cohortStatsRef; - + @Parameter(names = {"--cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String cohortStatsAlt; - + @Parameter(names = {"--cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String cohortStatsMaf; - + @Parameter(names = {"--ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String ct; - + @Parameter(names = {"--xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String xref; - + @Parameter(names = {"--biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String biotype; - + @Parameter(names = {"--protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String proteinSubstitution; - + @Parameter(names = {"--conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String conservation; - + @Parameter(names = {"--population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String populationFrequencyMaf; - + @Parameter(names = {"--population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String populationFrequencyAlt; - + @Parameter(names = {"--population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String populationFrequencyRef; - + @Parameter(names = {"--transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String transcriptFlag; - + @Parameter(names = {"--functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String functionalScore; - + @Parameter(names = {"--clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String clinical; - + @Parameter(names = {"--clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String clinicalSignificance; - + @Parameter(names = {"--clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean clinicalConfirmedStatus = false; - + @Parameter(names = {"--genotypes"}, description = "The body web service genotypes parameter", required = false, arity = 1) public String genotypes; - + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--samples-in-all-variants"}, description = "The body web service samplesInAllVariants parameter", required = false, help = true, arity = 0) public boolean samplesInAllVariants = false; - + @Parameter(names = {"--max-variants"}, description = "The body web service maxVariants parameter", required = false, arity = 1) public Integer maxVariants; - + } @Parameters(commandNames = {"sample-stats-query"}, commandDescription ="Obtain sample variant stats from a sample.") public class QuerySampleStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) - public String region; - + public String region; + @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) - public String type; - + public String type; + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) - public String file; - + public String file; + @Parameter(names = {"--filter"}, description = "Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX", required = false, arity = 1) - public String filter; - + public String filter; + @Parameter(names = {"--sample-data"}, description = "Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will use all samples from 'sample' or 'genotype' filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10", required = false, arity = 1) - public String sampleData; - + public String sampleData; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) - public String ct; - + public String ct; + @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) - public String biotype; - + public String biotype; + @Parameter(names = {"--transcript-flag"}, description = "List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, TSO500", required = false, arity = 1) - public String transcriptFlag; - + public String transcriptFlag; + @Parameter(names = {"--population-frequency-alt"}, description = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01", required = false, arity = 1) - public String populationFrequencyAlt; - + public String populationFrequencyAlt; + @Parameter(names = {"--clinical"}, description = "Clinical source: clinvar, cosmic", required = false, arity = 1) - public String clinical; - + public String clinical; + @Parameter(names = {"--clinical-significance"}, description = "Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic", required = false, arity = 1) - public String clinicalSignificance; - + public String clinicalSignificance; + @Parameter(names = {"--clinical-confirmed-status"}, description = "Clinical confirmed status", required = false, help = true, arity = 0) - public boolean clinicalConfirmedStatus = false; - + public boolean clinicalConfirmedStatus = false; + @Parameter(names = {"--filter-transcript"}, description = "Do filter transcripts when obtaining transcript counts", required = false, help = true, arity = 0) - public boolean filterTranscript = false; - + public boolean filterTranscript = false; + @Parameter(names = {"--sample"}, description = "Sample ID", required = true, arity = 1) - public String sample; - + public String sample; + } @Parameters(commandNames = {"sample-stats-run"}, commandDescription ="Compute sample variant stats for the selected list of samples.") public class RunSampleStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; - + @Parameter(names = {"--individual"}, description = "The body web service individual parameter", required = false, arity = 1) public String individual; - + @Parameter(names = {"--variant-query-id"}, description = "The body web service id parameter", required = false, arity = 1) public String variantQueryId; - + @Parameter(names = {"--variant-query-region"}, description = "The body web service region parameter", required = false, arity = 1) public String variantQueryRegion; - + @Parameter(names = {"--variant-query-gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String variantQueryGene; - + @Parameter(names = {"--variant-query-type"}, description = "The body web service type parameter", required = false, arity = 1) public String variantQueryType; - + @Parameter(names = {"--variant-query-panel"}, description = "The body web service panel parameter", required = false, arity = 1) public String variantQueryPanel; - + @Parameter(names = {"--variant-query-panel-mode-of-inheritance"}, description = "The body web service panelModeOfInheritance parameter", required = false, arity = 1) public String variantQueryPanelModeOfInheritance; - + @Parameter(names = {"--variant-query-panel-confidence"}, description = "The body web service panelConfidence parameter", required = false, arity = 1) public String variantQueryPanelConfidence; - + @Parameter(names = {"--variant-query-panel-role-in-cancer"}, description = "The body web service panelRoleInCancer parameter", required = false, arity = 1) public String variantQueryPanelRoleInCancer; - + @Parameter(names = {"--variant-query-panel-intersection"}, description = "The body web service panelIntersection parameter", required = false, help = true, arity = 0) public boolean variantQueryPanelIntersection = false; - + @Parameter(names = {"--variant-query-panel-feature-type"}, description = "The body web service panelFeatureType parameter", required = false, arity = 1) public String variantQueryPanelFeatureType; - + @Parameter(names = {"--variant-query-cohort-stats-ref"}, description = "The body web service cohortStatsRef parameter", required = false, arity = 1) public String variantQueryCohortStatsRef; - + @Parameter(names = {"--variant-query-cohort-stats-alt"}, description = "The body web service cohortStatsAlt parameter", required = false, arity = 1) public String variantQueryCohortStatsAlt; - + @Parameter(names = {"--variant-query-cohort-stats-maf"}, description = "The body web service cohortStatsMaf parameter", required = false, arity = 1) public String variantQueryCohortStatsMaf; - + @Parameter(names = {"--variant-query-ct"}, description = "The body web service ct parameter", required = false, arity = 1) public String variantQueryCt; - + @Parameter(names = {"--variant-query-xref"}, description = "The body web service xref parameter", required = false, arity = 1) public String variantQueryXref; - + @Parameter(names = {"--variant-query-biotype"}, description = "The body web service biotype parameter", required = false, arity = 1) public String variantQueryBiotype; - + @Parameter(names = {"--variant-query-protein-substitution"}, description = "The body web service proteinSubstitution parameter", required = false, arity = 1) public String variantQueryProteinSubstitution; - + @Parameter(names = {"--variant-query-conservation"}, description = "The body web service conservation parameter", required = false, arity = 1) public String variantQueryConservation; - + @Parameter(names = {"--variant-query-population-frequency-maf"}, description = "The body web service populationFrequencyMaf parameter", required = false, arity = 1) public String variantQueryPopulationFrequencyMaf; - + @Parameter(names = {"--variant-query-population-frequency-alt"}, description = "The body web service populationFrequencyAlt parameter", required = false, arity = 1) public String variantQueryPopulationFrequencyAlt; - + @Parameter(names = {"--variant-query-population-frequency-ref"}, description = "The body web service populationFrequencyRef parameter", required = false, arity = 1) public String variantQueryPopulationFrequencyRef; - + @Parameter(names = {"--variant-query-transcript-flag"}, description = "The body web service transcriptFlag parameter", required = false, arity = 1) public String variantQueryTranscriptFlag; - + @Parameter(names = {"--variant-query-functional-score"}, description = "The body web service functionalScore parameter", required = false, arity = 1) public String variantQueryFunctionalScore; - + @Parameter(names = {"--variant-query-clinical"}, description = "The body web service clinical parameter", required = false, arity = 1) public String variantQueryClinical; - + @Parameter(names = {"--variant-query-clinical-significance"}, description = "The body web service clinicalSignificance parameter", required = false, arity = 1) public String variantQueryClinicalSignificance; - + @Parameter(names = {"--variant-query-clinical-confirmed-status"}, description = "The body web service clinicalConfirmedStatus parameter", required = false, help = true, arity = 0) public boolean variantQueryClinicalConfirmedStatus = false; - + @Parameter(names = {"--variant-query-sample-data"}, description = "The body web service sampleData parameter", required = false, arity = 1) public String variantQuerySampleData; - + @Parameter(names = {"--variant-query-file-data"}, description = "The body web service fileData parameter", required = false, arity = 1) public String variantQueryFileData; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--index"}, description = "The body web service index parameter", required = false, help = true, arity = 0) public boolean index = false; - + @Parameter(names = {"--index-overwrite"}, description = "The body web service indexOverwrite parameter", required = false, help = true, arity = 0) public boolean indexOverwrite = false; - + @Parameter(names = {"--index-id"}, description = "The body web service indexId parameter", required = false, arity = 1) public String indexId; - + @Parameter(names = {"--index-description"}, description = "The body web service indexDescription parameter", required = false, arity = 1) public String indexDescription; - + @Parameter(names = {"--batch-size"}, description = "The body web service batchSize parameter", required = false, arity = 1) public Integer batchSize; - + } @Parameters(commandNames = {"stats-export-run"}, commandDescription ="Export calculated variant stats and frequencies") public class RunStatsExportCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) - public String project; - + public String project; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--cohorts"}, description = "The body web service cohorts parameter", required = false, arity = 1) public String cohorts; - + @Parameter(names = {"--output"}, description = "The body web service output parameter", required = false, arity = 1) public String output; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--output-file-format"}, description = "The body web service outputFileFormat parameter", required = false, arity = 1) public String outputFileFormat; - + } @Parameters(commandNames = {"stats-run"}, commandDescription ="Compute variant stats for any cohort and any set of variants.") public class RunStatsCommandOptions { - + @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) public String jsonFile; - + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - + public String study; + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) - public String jobId; - + public String jobId; + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) - public String jobDescription; - + public String jobDescription; + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) - public String jobDependsOn; - + public String jobDependsOn; + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) - public String jobTags; - + public String jobTags; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; - + @Parameter(names = {"--samples"}, description = "The body web service samples parameter", required = false, arity = 1) public String samples; - + @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) public String region; - + @Parameter(names = {"--gene"}, description = "The body web service gene parameter", required = false, arity = 1) public String gene; - + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; - + @Parameter(names = {"--output-file-name"}, description = "The body web service outputFileName parameter", required = false, arity = 1) public String outputFileName; - + @Parameter(names = {"--aggregated"}, description = "The body web service aggregated parameter", required = false, arity = 1) public String aggregated; - + @Parameter(names = {"--aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) public String aggregationMappingFile; - + } } \ No newline at end of file diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index 1aef5e23bb1..ae94b1ef729 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index 036b855a053..68ac3049ac6 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index 2a76b8da965..6c2248ff3b2 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -1,6 +1,6 @@ # ############################################################################## ## OrganizationClient -setGeneric("organizationClient", function(OpencgaR, organization, id, endpointName, params=NULL, ...) +setGeneric("organizationClient", function(OpencgaR, id, organization, endpointName, params=NULL, ...) standardGeneric("organizationClient")) # ############################################################################## @@ -15,42 +15,42 @@ setGeneric("projectClient", function(OpencgaR, projects, project, endpointName, # ############################################################################## ## StudyClient -setGeneric("studyClient", function(OpencgaR, members, templateId, group, studies, study, variableSet, id, endpointName, params=NULL, ...) +setGeneric("studyClient", function(OpencgaR, group, members, variableSet, study, id, templateId, studies, endpointName, params=NULL, ...) standardGeneric("studyClient")) # ############################################################################## ## FileClient -setGeneric("fileClient", function(OpencgaR, members, files, file, annotationSet, folder, endpointName, params=NULL, ...) +setGeneric("fileClient", function(OpencgaR, members, annotationSet, file, files, folder, endpointName, params=NULL, ...) standardGeneric("fileClient")) # ############################################################################## ## JobClient -setGeneric("jobClient", function(OpencgaR, members, jobs, job, endpointName, params=NULL, ...) +setGeneric("jobClient", function(OpencgaR, jobs, members, job, endpointName, params=NULL, ...) standardGeneric("jobClient")) # ############################################################################## ## SampleClient -setGeneric("sampleClient", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) +setGeneric("sampleClient", function(OpencgaR, annotationSet, sample, samples, members, endpointName, params=NULL, ...) standardGeneric("sampleClient")) # ############################################################################## ## IndividualClient -setGeneric("individualClient", function(OpencgaR, members, annotationSet, individual, individuals, endpointName, params=NULL, ...) +setGeneric("individualClient", function(OpencgaR, individuals, individual, annotationSet, members, endpointName, params=NULL, ...) standardGeneric("individualClient")) # ############################################################################## ## FamilyClient -setGeneric("familyClient", function(OpencgaR, members, family, annotationSet, families, endpointName, params=NULL, ...) +setGeneric("familyClient", function(OpencgaR, annotationSet, family, families, members, endpointName, params=NULL, ...) standardGeneric("familyClient")) # ############################################################################## ## CohortClient -setGeneric("cohortClient", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) +setGeneric("cohortClient", function(OpencgaR, annotationSet, cohort, cohorts, members, endpointName, params=NULL, ...) standardGeneric("cohortClient")) # ############################################################################## ## PanelClient -setGeneric("panelClient", function(OpencgaR, members, panels, endpointName, params=NULL, ...) +setGeneric("panelClient", function(OpencgaR, panels, members, endpointName, params=NULL, ...) standardGeneric("panelClient")) # ############################################################################## @@ -65,7 +65,7 @@ setGeneric("variantClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## ClinicalClient -setGeneric("clinicalClient", function(OpencgaR, members, interpretations, annotationSet, interpretation, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) +setGeneric("clinicalClient", function(OpencgaR, clinicalAnalysis, interpretation, interpretations, members, annotationSet, clinicalAnalyses, endpointName, params=NULL, ...) standardGeneric("clinicalClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index 357d379bd5a..b2923145986 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -62,7 +62,7 @@ #' [*]: Required parameter #' @export -setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, interpretations, annotationSet, interpretation, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) { +setMethod("clinicalClient", "OpencgaR", function(OpencgaR, clinicalAnalysis, interpretation, interpretations, members, annotationSet, clinicalAnalyses, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/analysis/clinical/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Cohort-methods.R b/opencga-client/src/main/R/R/Cohort-methods.R index 1f3a876aaa9..62b021eec99 100644 --- a/opencga-client/src/main/R/R/Cohort-methods.R +++ b/opencga-client/src/main/R/R/Cohort-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("cohortClient", "OpencgaR", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) { +setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, cohort, cohorts, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/cohorts/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Family-methods.R b/opencga-client/src/main/R/R/Family-methods.R index f38a6e85189..c8cddece994 100644 --- a/opencga-client/src/main/R/R/Family-methods.R +++ b/opencga-client/src/main/R/R/Family-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +37,7 @@ #' [*]: Required parameter #' @export -setMethod("familyClient", "OpencgaR", function(OpencgaR, members, family, annotationSet, families, endpointName, params=NULL, ...) { +setMethod("familyClient", "OpencgaR", function(OpencgaR, annotationSet, family, families, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/families/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index f1bd5542eab..1621705b60b 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +54,7 @@ #' [*]: Required parameter #' @export -setMethod("fileClient", "OpencgaR", function(OpencgaR, members, files, file, annotationSet, folder, endpointName, params=NULL, ...) { +setMethod("fileClient", "OpencgaR", function(OpencgaR, members, annotationSet, file, files, folder, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/files/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/GA4GH-methods.R b/opencga-client/src/main/R/R/GA4GH-methods.R index 35dc15c1113..462ee8cd504 100644 --- a/opencga-client/src/main/R/R/GA4GH-methods.R +++ b/opencga-client/src/main/R/R/GA4GH-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Individual-methods.R b/opencga-client/src/main/R/R/Individual-methods.R index 065c23c60a1..ef43c130f5b 100644 --- a/opencga-client/src/main/R/R/Individual-methods.R +++ b/opencga-client/src/main/R/R/Individual-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("individualClient", "OpencgaR", function(OpencgaR, members, annotationSet, individual, individuals, endpointName, params=NULL, ...) { +setMethod("individualClient", "OpencgaR", function(OpencgaR, individuals, individual, annotationSet, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/individuals/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index 8ea4cb9e9d4..59d5b52b29f 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("jobClient", "OpencgaR", function(OpencgaR, members, jobs, job, endpointName, params=NULL, ...) { +setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/jobs/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Meta-methods.R b/opencga-client/src/main/R/R/Meta-methods.R index 7ce863ddb61..6e82b2b93cd 100644 --- a/opencga-client/src/main/R/R/Meta-methods.R +++ b/opencga-client/src/main/R/R/Meta-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 2793051d9de..371a9418d0e 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Organization-methods.R b/opencga-client/src/main/R/R/Organization-methods.R index 6c42ae9b06b..4fef5f13405 100644 --- a/opencga-client/src/main/R/R/Organization-methods.R +++ b/opencga-client/src/main/R/R/Organization-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -34,7 +34,7 @@ #' [*]: Required parameter #' @export -setMethod("organizationClient", "OpencgaR", function(OpencgaR, organization, id, endpointName, params=NULL, ...) { +setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/organizations/create: diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index 9e4720691d7..b354c2bd497 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -36,7 +36,7 @@ #' [*]: Required parameter #' @export -setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpointName, params=NULL, ...) { +setMethod("panelClient", "OpencgaR", function(OpencgaR, panels, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/panels/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Project-methods.R b/opencga-client/src/main/R/R/Project-methods.R index aca59165834..0a72fff1332 100644 --- a/opencga-client/src/main/R/R/Project-methods.R +++ b/opencga-client/src/main/R/R/Project-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Sample-methods.R b/opencga-client/src/main/R/R/Sample-methods.R index 65871dcb8bb..0ae9bf5c3a5 100644 --- a/opencga-client/src/main/R/R/Sample-methods.R +++ b/opencga-client/src/main/R/R/Sample-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("sampleClient", "OpencgaR", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) { +setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, sample, samples, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/samples/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index 22276f49f57..598d196e758 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -49,7 +49,7 @@ #' [*]: Required parameter #' @export -setMethod("studyClient", "OpencgaR", function(OpencgaR, members, templateId, group, studies, study, variableSet, id, endpointName, params=NULL, ...) { +setMethod("studyClient", "OpencgaR", function(OpencgaR, group, members, variableSet, study, id, templateId, studies, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/studies/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/User-methods.R b/opencga-client/src/main/R/R/User-methods.R index 76772b8a2d4..1c20e2108ef 100644 --- a/opencga-client/src/main/R/R/User-methods.R +++ b/opencga-client/src/main/R/R/User-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index 70c435fcc6c..f3c772a36d5 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-05-23 +# Autogenerated on: 2024-06-07 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index 3b43605603f..6805ee60e1a 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index 4d2c0d9fab4..df89c23091d 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -40,7 +40,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index 4bfe0704ebb..7b58be847a8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -55,7 +55,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java index 0163f130151..7153e499d15 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index 0ca3b876bcc..6e86d3ee3ec 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java index 4f53af041e6..46deec835bb 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index 687e43e6ea5..d91f720cee8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -43,7 +43,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java index 765d86384d2..73c638f2b7d 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java @@ -27,7 +27,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java index 939a53e5a86..8ef51668f7b 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index 8840fd8c7c8..75b5b33f8ae 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java index 2fb1c5df290..52424b8dca1 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java @@ -28,7 +28,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java index bbc2d910bbc..3af322403d9 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java @@ -33,7 +33,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java index 570af443ec9..bee02c2c0f7 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java @@ -31,7 +31,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java index 117401694cc..3398105fabb 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index 424a5cbc509..c4b1b730e3e 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -47,7 +47,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java index 468dffded80..57d314a52e1 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index 6dc2fdb9bd8..62844cdbc7b 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -62,7 +62,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index 52b0333e713..97365adc37c 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -50,7 +50,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-05-23 +* Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index 49ab10c86c2..c913713d534 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index 005457da0ab..95dcf16abae 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index 302b3084adc..0b305bbbf6f 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Cohort.js b/opencga-client/src/main/javascript/Cohort.js index e07f6c58dab..370a67215bf 100644 --- a/opencga-client/src/main/javascript/Cohort.js +++ b/opencga-client/src/main/javascript/Cohort.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index 1a04bd3c77f..85a9f584f7d 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Family.js b/opencga-client/src/main/javascript/Family.js index e12be40b17d..6a9865a8621 100644 --- a/opencga-client/src/main/javascript/Family.js +++ b/opencga-client/src/main/javascript/Family.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index ab7b241d600..e0f766a1b8f 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index eaff0099476..28f2f6f3109 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Individual.js b/opencga-client/src/main/javascript/Individual.js index e1304c41a58..823995b5866 100644 --- a/opencga-client/src/main/javascript/Individual.js +++ b/opencga-client/src/main/javascript/Individual.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index ad50f4b336b..b61ffc6aa97 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Meta.js b/opencga-client/src/main/javascript/Meta.js index 87fb7ba03ce..f778a22eb2d 100644 --- a/opencga-client/src/main/javascript/Meta.js +++ b/opencga-client/src/main/javascript/Meta.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Organization.js b/opencga-client/src/main/javascript/Organization.js index 2b94af1924f..fd5de8823d7 100644 --- a/opencga-client/src/main/javascript/Organization.js +++ b/opencga-client/src/main/javascript/Organization.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index 839907f1003..5e1dae4c60a 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Sample.js b/opencga-client/src/main/javascript/Sample.js index 1af15198986..725f4269b0f 100644 --- a/opencga-client/src/main/javascript/Sample.js +++ b/opencga-client/src/main/javascript/Sample.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 0145d803d6b..35c37618ac5 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index de654d8761e..b0b4b7ce576 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index 9bd0510f536..fcffab2c299 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index 02f03335138..86853659d29 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-05-23 + * Autogenerated on: 2024-06-07 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index 1b5c9fdd673..f4af9f3875f 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index abdd7e41f3c..cd9db387e74 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index edd4c870c13..db452e9365a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py index a93c8378fdb..c44071a4a6c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index 4da214c272c..12a26425317 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py index 935e447a4c7..b2543574e20 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index a370bb911cc..58601deda50 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py index c6d5a0bf458..53c0c9c0d2c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py index 32f7516aed3..d58e7490281 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 6def8e09fa7..b614737719c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py index 5090a04eb23..1d87605a03b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py index 7c71bbadfed..4b45d0b658e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py index da71c68e814..0e7ed8062d8 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py index df13fd02245..d79438ff556 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 6b804bc9fd0..4ee020670f9 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py index f7c61eb29be..0fe1f67c15f 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index 548eae4ff0a..2c20b3d9ebd 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index a4d4d61a095..e28a0e4b837 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-05-23 + Autogenerated on: 2024-06-07 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. From 983405d23fc7b8252e9b1bc664fb9cb3c6950fb3 Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 7 Jun 2024 14:19:45 +0200 Subject: [PATCH 042/128] catalog: apply data model changes, #TASK-5964 --- .../db/api/ClinicalAnalysisDBAdaptor.java | 2 +- .../db/api/InterpretationDBAdaptor.java | 1 + .../ClinicalAnalysisMongoDBAdaptor.java | 8 +- .../mongodb/InterpretationMongoDBAdaptor.java | 3 +- .../managers/ClinicalAnalysisManager.java | 103 ++++++++++----- .../opencga/catalog/managers/FileManager.java | 1 - .../managers/InterpretationManager.java | 71 +++++++--- .../src/main/resources/catalog-indexes.txt | 1 + .../managers/ClinicalAnalysisManagerTest.java | 125 ++++++++++-------- .../managers/InterpretationManagerTest.java | 7 +- .../catalog/managers/PanelManagerTest.java | 2 +- .../opencga/core/api/ParamConstants.java | 2 + .../models/clinical/ClinicalAnalysis.java | 34 +++-- .../ClinicalAnalysisCreateParams.java | 22 +-- .../clinical/ClinicalAnalysisInternal.java | 10 +- ...us.java => ClinicalAnalysisStatusOld.java} | 8 +- .../ClinicalAnalysisUpdateParams.java | 26 ++-- .../core/models/clinical/ClinicalStatus.java | 77 +++++++++++ .../models/clinical/ClinicalStatusValue.java | 12 +- .../core/models/clinical/Interpretation.java | 55 +++++--- .../clinical/InterpretationCreateParams.java | 25 +++- .../clinical/InterpretationUpdateParams.java | 16 ++- .../core/models/common/IndexStatus.java | 2 +- .../core/models/common/InternalStatus.java | 19 +-- .../core/models/common/StatusParam.java | 9 ++ .../core/models/file/VariantIndexStatus.java | 2 +- .../ClinicalAnalysisStudyConfiguration.java | 41 +++--- .../InterpretationStudyConfiguration.java | 13 +- .../rest/analysis/ClinicalWebService.java | 2 + 29 files changed, 452 insertions(+), 247 deletions(-) rename opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/{ClinicalAnalysisStatus.java => ClinicalAnalysisStatusOld.java} (91%) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 0a4c46bc1b6..73fe6ac7e20 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -81,7 +81,7 @@ enum QueryParams implements QueryParam { VERSION("version", INTEGER, ""), RELEASE("release", INTEGER, ""), SNAPSHOT("snapshot", INTEGER, ""), - PANEL_LOCK("panelLock", BOOLEAN, ""), + PANEL_LOCKED("panelLocked", BOOLEAN, ""), LOCKED("locked", BOOLEAN, ""), SAMPLE("sample", TEXT_ARRAY, ""), // Alias to search for samples within proband.samples or family.members.samples diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java index 8c95202e0de..13e2cb64dfa 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java @@ -41,6 +41,7 @@ enum QueryParams implements QueryParam { ID("id", TEXT, ""), UID("uid", INTEGER, ""), UUID("uuid", TEXT, ""), + NAME("name", TEXT, ""), CLINICAL_ANALYSIS_ID("clinicalAnalysisId", TEXT, ""), DESCRIPTION("description", TEXT, ""), INTERNAL_STATUS("internal.status", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index c5b67364079..5624bfb9058 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -418,7 +418,7 @@ UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, List create(String studyStr, ClinicalAnalysis List events = new LinkedList<>(); - clinicalAnalysis.setStatus(ParamUtils.defaultObject(clinicalAnalysis.getStatus(), Status::new)); + clinicalAnalysis.setStatus(ParamUtils.defaultObject(clinicalAnalysis.getStatus(), ClinicalStatus::new)); clinicalAnalysis.setInternal(ClinicalAnalysisInternal.init()); clinicalAnalysis.setDisorder(ParamUtils.defaultObject(clinicalAnalysis.getDisorder(), new Disorder("", "", "", Collections.emptyMap(), "", Collections.emptyList()))); @@ -591,10 +591,10 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis validateCustomPriorityParameters(clinicalAnalysis, clinicalConfiguration); validateCustomFlagParameters(clinicalAnalysis, clinicalConfiguration); validateCustomConsentParameters(clinicalAnalysis, clinicalConfiguration); - validateStatusParameter(clinicalAnalysis, clinicalConfiguration); + validateStatusParameter(clinicalAnalysis, clinicalConfiguration, userId, true); if (StringUtils.isNotEmpty(clinicalAnalysis.getStatus().getId())) { - List clinicalStatusValues = clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = clinicalConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (clinicalAnalysis.getStatus().getId().equals(clinicalStatusValue.getId())) { if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -802,17 +802,15 @@ private void fillResponsible(String organizationId, ClinicalResponsible responsi responsible.setEmail(ParamUtils.defaultString(responsible.getEmail(), result.first().getEmail())); } - private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, ClinicalAnalysisStudyConfiguration clinicalConfiguration) - throws CatalogException { + private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, ClinicalAnalysisStudyConfiguration clinicalConfiguration, + String userId, boolean initIfUndefined) throws CatalogException { // Status - if (clinicalConfiguration.getStatus() == null - || CollectionUtils.isEmpty(clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()))) { - throw new CatalogException("Missing status configuration in study for type '" + clinicalAnalysis.getType() - + "'. Please add a proper set of valid statuses."); + if (CollectionUtils.isEmpty(clinicalConfiguration.getStatus())) { + throw new CatalogException("Missing status configuration in study. Please add a proper set of valid statuses."); } if (StringUtils.isNotEmpty(clinicalAnalysis.getStatus().getId())) { Map statusMap = new HashMap<>(); - for (ClinicalStatusValue status : clinicalConfiguration.getStatus().get(clinicalAnalysis.getType())) { + for (ClinicalStatusValue status : clinicalConfiguration.getStatus()) { statusMap.put(status.getId(), status); } if (!statusMap.containsKey(clinicalAnalysis.getStatus().getId())) { @@ -821,10 +819,26 @@ private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, Clinical } ClinicalStatusValue clinicalStatusValue = statusMap.get(clinicalAnalysis.getStatus().getId()); clinicalAnalysis.getStatus().setDescription(clinicalStatusValue.getDescription()); - clinicalAnalysis.getStatus().setDate(TimeUtils.getTime()); + clinicalAnalysis.getStatus().setType(clinicalStatusValue.getType()); } else if (clinicalAnalysis.getStatus().getId() == null) { - clinicalAnalysis.getStatus().setId(""); + if (initIfUndefined) { + // Look for first status of type NOT_STARTED + for (ClinicalStatusValue status : clinicalConfiguration.getStatus()) { + if (status.getType() == ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) { + clinicalAnalysis.getStatus().setId(status.getId()); + clinicalAnalysis.getStatus().setDescription(status.getDescription()); + clinicalAnalysis.getStatus().setType(status.getType()); + break; + } + } + } else { + throw new CatalogException("Missing status id in clinical analysis"); + } } + clinicalAnalysis.getStatus().setDate(TimeUtils.getTime()); + clinicalAnalysis.getStatus().setVersion(GitRepositoryState.getInstance().getBuildVersion()); + clinicalAnalysis.getStatus().setCommit(GitRepositoryState.getInstance().getCommitId()); + clinicalAnalysis.getStatus().setAuthor(userId); } private void validateCustomConsentParameters(ClinicalAnalysis clinicalAnalysis, @@ -1395,7 +1409,7 @@ private OpenCGAResult update(String organizationId, Study stud // Get the clinical status that are CLOSED Set closedStatus = new HashSet<>(); - for (ClinicalStatusValue clinicalStatusValue : clinicalConfiguration.getStatus().get(clinicalAnalysis.getType())) { + for (ClinicalStatusValue clinicalStatusValue : clinicalConfiguration.getStatus()) { if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { closedStatus.add(clinicalStatusValue.getId()); } @@ -1406,11 +1420,22 @@ private OpenCGAResult update(String organizationId, Study stud // - the user wants to update the locked status // - the user wants to update the status to/from a closed status boolean adminPermissionsChecked = false; - if (clinicalAnalysis.isLocked() || closedStatus.contains(clinicalAnalysis.getInternal().getStatus().getId()) + if (clinicalAnalysis.isLocked() || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED || updateParamsClone.getLocked() != null || (updateParams.getStatus() != null && closedStatus.contains(updateParams.getStatus().getId()))) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.ADMIN); + + // Current status is of type CLOSED + if (clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + // The only allowed action is to remove the CLOSED status + if (updateParams.getStatus() == null || closedStatus.contains(updateParams.getStatus().getId())) { + throw new CatalogException("Cannot update a ClinicalAnalysis with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the ClinicalAnalysis."); + } + } + adminPermissionsChecked = true; } // Check permissions... @@ -1553,13 +1578,13 @@ private OpenCGAResult update(String organizationId, Study stud parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key(), clinicalAnalysis.getFiles()); } - if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels()) && updateParamsClone.getPanelLock() != null - && updateParamsClone.getPanelLock()) { + if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels()) && updateParamsClone.getPanelLocked() != null + && updateParamsClone.getPanelLocked()) { throw new CatalogException("Updating the list of panels and setting 'panelLock' to true at the same time is not allowed."); } if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels())) { - if (clinicalAnalysis.isPanelLock() && (updateParamsClone.getPanelLock() == null || updateParamsClone.getPanelLock())) { + if (clinicalAnalysis.isPanelLocked() && (updateParamsClone.getPanelLocked() == null || updateParamsClone.getPanelLocked())) { throw new CatalogException("Cannot update panels from ClinicalAnalysis '" + clinicalAnalysis.getId() + "'. " + "'panelLocked' field from ClinicalAnalysis is set to true."); } @@ -1576,7 +1601,7 @@ private OpenCGAResult update(String organizationId, Study stud parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), panelResult.getResults()); } - if (updateParamsClone.getPanelLock() != null && updateParamsClone.getPanelLock() && !clinicalAnalysis.isPanelLock()) { + if (updateParamsClone.getPanelLocked() != null && updateParamsClone.getPanelLocked() && !clinicalAnalysis.isPanelLocked()) { // if user wants to set panelLock to true // We need to check if the CA has interpretations. If so, the interpretations should contain at least one of the case panels // in order to set panelLock to true. Otherwise, that action is not allowed. @@ -1635,7 +1660,7 @@ private OpenCGAResult update(String organizationId, Study stud if (parameters.containsKey(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key())) { Map status = (Map) parameters.get(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key()); if (!(status instanceof Map) || StringUtils.isEmpty(String.valueOf(status.get("name"))) - || !ClinicalAnalysisStatus.isValid(String.valueOf(status.get("name")))) { + || !ClinicalAnalysisStatusOld.isValid(String.valueOf(status.get("name")))) { throw new CatalogException("Missing or invalid status"); } } @@ -1679,12 +1704,12 @@ private OpenCGAResult update(String organizationId, Study stud parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.CONSENT.key(), clinicalAnalysis.getConsent()); } if (parameters.containsKey(ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key())) { - clinicalAnalysis.setStatus(updateParamsClone.getStatus().toStatus()); - validateStatusParameter(clinicalAnalysis, clinicalConfiguration); + clinicalAnalysis.setStatus(updateParamsClone.getStatus().toClinicalStatus()); + validateStatusParameter(clinicalAnalysis, clinicalConfiguration, userId, false); parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key(), clinicalAnalysis.getStatus()); if (StringUtils.isNotEmpty(updateParamsClone.getStatus().getId())) { - List clinicalStatusValues = clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = clinicalConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (updateParamsClone.getStatus().getId().equals(clinicalStatusValue.getId())) { if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -2656,23 +2681,29 @@ public OpenCGAResult configureStudy(String studyStr, ClinicalAnalysisStudyConfig } } - private void validateClinicalStatus(Map> status, String field) + private void validateClinicalStatus(List status, String field) throws CatalogException { - if (status == null) { + if (CollectionUtils.isEmpty(status)) { throw CatalogParameterException.isNull(field); } - for (ClinicalAnalysis.Type value : ClinicalAnalysis.Type.values()) { - if (!status.containsKey(value)) { - throw new CatalogParameterException(field + ": Missing status values for ClinicalAnalysis type '" + value + "'"); + // Ensure there's at least one status id per status type + Map presentMap = new HashMap<>(); + for (ClinicalStatusValue.ClinicalStatusType value : ClinicalStatusValue.ClinicalStatusType.values()) { + // Init map + presentMap.put(value, false); + } + for (ClinicalStatusValue clinicalStatusValue : status) { + if (StringUtils.isEmpty(clinicalStatusValue.getId())) { + throw CatalogParameterException.isNull(field + ".id"); } - List statuses = status.get(value); - for (ClinicalStatusValue clinicalStatusValue : statuses) { - if (StringUtils.isEmpty(clinicalStatusValue.getId())) { - throw CatalogParameterException.isNull(field + ".{" + value + "}.id"); - } - if (clinicalStatusValue.getType() == null) { - throw CatalogParameterException.isNull(field + ".{" + value + "}.type"); - } + if (clinicalStatusValue.getType() == null) { + throw CatalogParameterException.isNull(field + ".type"); + } + presentMap.put(clinicalStatusValue.getType(), true); + } + for (Map.Entry entry : presentMap.entrySet()) { + if (!entry.getValue()) { + throw new CatalogException("Missing status values for ClinicalStatus type '" + entry.getKey() + "'"); } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index 48ee0d6adda..9c1af82e3f5 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -3249,7 +3249,6 @@ private void checkCanDeleteFile(String organizationId, Study study, String fileI // We need to remove the reference to the transformed files and change their status from TRANSFORMED to NONE next.getInternal().getVariant().getIndex().setTransform(null); next.getInternal().getVariant().getIndex().getStatus().setId(VariantIndexStatus.NONE); - next.getInternal().getVariant().getIndex().getStatus().setName(VariantIndexStatus.NONE); break; case VariantIndexStatus.NONE: case VariantIndexStatus.DELETED: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 95ae214a1d7..72c013ea3ee 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -27,7 +27,6 @@ import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; import org.opencb.biodata.models.clinical.interpretation.InterpretationStats; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -45,6 +44,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.JwtPayload; @@ -69,7 +69,7 @@ public class InterpretationManager extends ResourceManager { public static final QueryOptions INCLUDE_CLINICAL_ANALYSIS = keepFieldsInQueryOptions(ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, - Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key())); + Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key())); public static final QueryOptions INCLUDE_INTERPRETATION_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( InterpretationDBAdaptor.QueryParams.ID.key(), InterpretationDBAdaptor.QueryParams.UID.key(), InterpretationDBAdaptor.QueryParams.UUID.key(), InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), @@ -219,7 +219,7 @@ public OpenCGAResult create(String studyStr, String clinicalAnal try { QueryOptions clinicalOptions = keepFieldsInQueryOptions(ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), - ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key(), + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.AUDIT.key(), ClinicalAnalysisDBAdaptor.QueryParams.INTERPRETATION_ID.key(), ClinicalAnalysisDBAdaptor.QueryParams.SECONDARY_INTERPRETATIONS_ID.key())); @@ -277,6 +277,7 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio } interpretation.setId(clinicalAnalysis.getId() + "." + count); + interpretation.setName(ParamUtils.defaultString(interpretation.getName(), interpretation.getId())); interpretation.setClinicalAnalysisId(clinicalAnalysis.getId()); @@ -290,7 +291,7 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio interpretation.setPrimaryFindings(ParamUtils.defaultObject(interpretation.getPrimaryFindings(), Collections.emptyList())); interpretation.setSecondaryFindings(ParamUtils.defaultObject(interpretation.getSecondaryFindings(), Collections.emptyList())); interpretation.setComments(ParamUtils.defaultObject(interpretation.getComments(), Collections.emptyList())); - interpretation.setStatus(ParamUtils.defaultObject(interpretation.getStatus(), Status::new)); + interpretation.setStatus(ParamUtils.defaultObject(interpretation.getStatus(), ClinicalStatus::new)); interpretation.setRelease(studyManager.getCurrentRelease(study)); interpretation.setVersion(1); interpretation.setAttributes(ParamUtils.defaultObject(interpretation.getAttributes(), Collections.emptyMap())); @@ -300,7 +301,7 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio if (CollectionUtils.isEmpty(interpretation.getPanels())) { interpretation.setPanels(clinicalAnalysis.getPanels()); } else { - if (clinicalAnalysis.isPanelLock()) { + if (clinicalAnalysis.isPanelLocked()) { // Check the panels are the same provided in the Clinical Analysis Map clinicalPanelIds = clinicalAnalysis.getPanels().stream() .collect(Collectors.toMap(DiseasePanel::getId, panel -> panel)); @@ -331,9 +332,9 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio } // Validate status - validateStatusParameter(interpretation, clinicalAnalysis.getType(), interpretationConfiguration); + validateStatusParameter(interpretation, interpretationConfiguration, userId, true); if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { - List clinicalStatusValues = interpretationConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = interpretationConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (interpretation.getStatus().getId().equals(clinicalStatusValue.getId()) && clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -413,6 +414,16 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); auditManager.initAuditBatch(operationId); + InterpretationStudyConfiguration interpretationConfiguration = + study.getInternal().getConfiguration().getClinical().getInterpretation(); + ClinicalStatusValue initStatus = null; + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { + if (status.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { + initStatus = status; + break; + } + } + OpenCGAResult result = OpenCGAResult.empty(); for (String interpretationStr : interpretationList) { String interpretationId = interpretationStr; @@ -457,13 +468,13 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly actionMap.put(InterpretationDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - InterpretationUpdateParams params = new InterpretationUpdateParams("", new ClinicalAnalystParam(), + InterpretationUpdateParams params = new InterpretationUpdateParams("", "", new ClinicalAnalystParam(), InterpretationMethod.init(), null, null, Collections.emptyList(), Collections.emptyList(), clinicalAnalysis.getPanels() != null ? clinicalAnalysis.getPanels().stream() .map(p -> new PanelReferenceParam().setId(p.getId())).collect(Collectors.toList()) : null, - Collections.emptyList(), new StatusParam(), false, new ObjectMap()); + Collections.emptyList(), new StatusParam(initStatus.getId()), false, new ObjectMap()); ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.CLEAR_INTERPRETATION, "Clear interpretation '" + interpretationId + "'", TimeUtils.getTime()); @@ -982,7 +993,7 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation } if (updateParams != null && CollectionUtils.isNotEmpty(updateParams.getPanels())) { - if (clinicalAnalysis.isPanelLock()) { + if (clinicalAnalysis.isPanelLocked()) { throw new CatalogException("Updating panels from Interpretation is not allowed. 'panelLock' from ClinicalAnalysis is set " + "to True."); } @@ -1068,12 +1079,12 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation } if (parameters.containsKey(InterpretationDBAdaptor.QueryParams.STATUS.key())) { - interpretation.setStatus(updateParams.getStatus().toStatus()); - validateStatusParameter(interpretation, clinicalAnalysis.getType(), interpretationConfiguration); + interpretation.setStatus(updateParams.getStatus().toClinicalStatus()); + validateStatusParameter(interpretation, interpretationConfiguration, userId, false); parameters.put(InterpretationDBAdaptor.QueryParams.STATUS.key(), interpretation.getStatus()); if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { - List clinicalStatusValues = interpretationConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = interpretationConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (interpretation.getStatus().getId().equals(clinicalStatusValue.getId()) && clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -1128,7 +1139,7 @@ public OpenCGAResult revert(String studyStr, String clinicalAnal throw new CatalogException("Could not revert the Interpretation. Case is locked so no further modifications can be made to" + " the Interpretation."); } - if (clinicalAnalysis.isPanelLock()) { + if (clinicalAnalysis.isPanelLocked()) { throw new CatalogException("Could not revert the Interpretation. 'panelLock' is set to True, so no further modifications" + " can be made to the Interpretation."); } @@ -1515,17 +1526,15 @@ protected void fixQueryObject(String organizationId, Study study, Query query, S } } - private void validateStatusParameter(Interpretation interpretation, ClinicalAnalysis.Type type, - InterpretationStudyConfiguration interpretationConfiguration) throws CatalogException { + private void validateStatusParameter(Interpretation interpretation, InterpretationStudyConfiguration interpretationConfiguration, + String userId, boolean initIfUndefined) throws CatalogException { // Status - if (interpretationConfiguration.getStatus() == null - || CollectionUtils.isEmpty(interpretationConfiguration.getStatus().get(type))) { - throw new CatalogException("Missing status configuration in study for type '" + type - + "'. Please add a proper set of valid statuses."); + if (CollectionUtils.isEmpty(interpretationConfiguration.getStatus())) { + throw new CatalogException("Missing status configuration in study. Please add a proper set of valid statuses."); } if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { Map statusMap = new HashMap<>(); - for (ClinicalStatusValue status : interpretationConfiguration.getStatus().get(type)) { + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { statusMap.put(status.getId(), status); } if (!statusMap.containsKey(interpretation.getStatus().getId())) { @@ -1534,7 +1543,25 @@ private void validateStatusParameter(Interpretation interpretation, ClinicalAnal } ClinicalStatusValue clinicalStatusValue = statusMap.get(interpretation.getStatus().getId()); interpretation.getStatus().setDescription(clinicalStatusValue.getDescription()); - interpretation.getStatus().setDate(TimeUtils.getTime()); + interpretation.getStatus().setType(clinicalStatusValue.getType()); + } else { + if (initIfUndefined) { + // Look for first status of type NOT_STARTED + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { + if (status.getType() == ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) { + interpretation.getStatus().setId(status.getId()); + interpretation.getStatus().setDescription(status.getDescription()); + interpretation.getStatus().setType(status.getType()); + break; + } + } + } else { + throw new CatalogException("Missing status id in Interpretation"); + } } + interpretation.getStatus().setDate(TimeUtils.getTime()); + interpretation.getStatus().setVersion(GitRepositoryState.getInstance().getBuildVersion()); + interpretation.getStatus().setCommit(GitRepositoryState.getInstance().getCommitId()); + interpretation.getStatus().setAuthor(userId); } } diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index dab0064fbb5..87f134a85c9 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -236,6 +236,7 @@ {"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"id": 1, "version": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"clinicalAnalysisId": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"method.name": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index f2ad85c2740..ae0ffc77613 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -25,7 +25,6 @@ import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariantEvidence; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; -import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.variant.avro.VariantAvro; import org.opencb.biodata.models.variant.avro.VariantType; @@ -158,7 +157,7 @@ private DataResult createDummyFamily() throws CatalogException { private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new Status().setId(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION)) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); @@ -204,25 +203,21 @@ public void createAndTestStatusIdIsNotNull() throws CatalogException { OpenCGAResult result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, false, INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, result.first().getStatus().getType()); clinicalAnalysis = new ClinicalAnalysis() .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) - .setStatus(new Status(null, null, null, null)) + .setStatus(new ClinicalStatus()) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))) .setFamily(new Family().setId("family") .setMembers(Collections.singletonList(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))))); result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, false, INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, result.first().getStatus().getType()); - result = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(null)), INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); - - result = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("")), INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); + String clinicalId = clinicalAnalysis.getId(); + assertThrows(CatalogException.class, () -> catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalId, + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(null)), INCLUDE_RESULT, ownerToken)); } @Test @@ -643,7 +638,7 @@ public void automaticallyLockCaseTest() throws CatalogException { .setProband(individual); OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); - assertTrue(StringUtils.isEmpty(clinical.first().getStatus().getId())); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, clinical.first().getStatus().getType()); assertFalse(clinical.first().isLocked()); clinical = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), @@ -654,10 +649,11 @@ public void automaticallyLockCaseTest() throws CatalogException { clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical2") .setType(ClinicalAnalysis.Type.SINGLE) - .setStatus(new ClinicalAnalysisStatus().setId("CLOSED")) + .setStatus(new ClinicalStatus().setId("CLOSED")) .setProband(individual); clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals("CLOSED", clinical.first().getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, clinical.first().getStatus().getType()); assertTrue(clinical.first().isLocked()); } @@ -1679,10 +1675,25 @@ public void adminPermissionTest() throws CatalogException { new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")), INCLUDE_RESULT, normalToken1) ); - // Unset status from CLOSED to other with user2 - WORKS + // Edit CLOSED ClinicalAnalysis from user with ADMIN permission + assertThrows(CatalogException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("new description"), INCLUDE_RESULT, normalToken2) + ); + + // Send CLOSED status and edit description from CLOSED ClinicalAnalysis from user with ADMIN permission + assertThrows(CatalogException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")).setDescription("new description"), + INCLUDE_RESULT, normalToken2) + ); + + // Remove CLOSED status and edit description from CLOSED ClinicalAnalysis from user with ADMIN permission clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")), INCLUDE_RESULT, normalToken2).first(); + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")).setDescription("new description"), + INCLUDE_RESULT, normalToken2).first(); assertEquals("READY_FOR_INTERPRETATION", clinicalAnalysis.getStatus().getId()); + assertEquals("new description", clinicalAnalysis.getDescription()); // Set status to CLOSED with study admin - WORKS clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), @@ -1708,7 +1719,7 @@ public void updateCustomStatusTest() throws CatalogException { DataResult dummyEnvironment = createDummyEnvironment(true, false); - ClinicalStatusValue status = configuration.getStatus().get(dummyEnvironment.first().getType()).get(0); + ClinicalStatusValue status = configuration.getStatus().get(0); ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setStatus(new StatusParam(status.getId())); @@ -1876,7 +1887,7 @@ public void updateInterpretationCustomStatusTest() throws CatalogException { InterpretationStudyConfiguration configuration = study.getInternal().getConfiguration().getClinical().getInterpretation(); DataResult dummyEnvironment = createDummyEnvironment(true, true); - ClinicalStatusValue status = configuration.getStatus().get(dummyEnvironment.first().getType()).get(0); + ClinicalStatusValue status = configuration.getStatus().get(1); InterpretationUpdateParams updateParams = new InterpretationUpdateParams() .setStatus(new StatusParam(status.getId())); @@ -2689,35 +2700,35 @@ public void searchClinicalAnalysisByStatus() throws CatalogException { createDummyEnvironment(false, false); OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.DONE), + new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.DONE), new QueryOptions(QueryOptions.INCLUDE, ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_ID.key()), ownerToken); assertEquals(0, search.getNumResults()); search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.READY_FOR_INTERPRETATION), + new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION), new QueryOptions(), ownerToken); assertEquals(2, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION, result.getStatus().getId()); } catalogManager.getClinicalAnalysisManager().update(studyFqn, search.first().getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(ClinicalAnalysisStatus.REJECTED)), QueryOptions.empty(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(ClinicalAnalysisStatusOld.REJECTED)), QueryOptions.empty(), ownerToken); search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.READY_FOR_INTERPRETATION), + new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION), new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION, result.getStatus().getId()); } search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.REJECTED), + new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.REJECTED), new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.REJECTED, result.getStatus().getId()); + assertEquals(ClinicalAnalysisStatusOld.REJECTED, result.getStatus().getId()); } } @@ -3239,7 +3250,7 @@ public void createClinicalAnalysisWithPanelsTest() throws CatalogException { assertNotNull(panel.getId()); assertNotNull(panel.getName()); } - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); } @Test @@ -3430,7 +3441,7 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogExc catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, true, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Map actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); @@ -3439,7 +3450,7 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogExc try { catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())) - .setPanelLock(true), + .setPanelLocked(true), updateOptions, ownerToken); fail("Updating panels and setting panellock to true in one call should not be accepted"); } catch (CatalogException e) { @@ -3450,24 +3461,24 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogExc .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), updateOptions, ownerToken); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), updateOptions, ownerToken); result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertTrue(result.first().isPanelLock()); + assertTrue(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))) - .setPanelLock(false), + .setPanelLocked(false), updateOptions, ownerToken); result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(1, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), updateOptions, ownerToken); thrown.expect(CatalogException.class); thrown.expectMessage("panelLock"); @@ -3494,13 +3505,13 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisWithInterpretationTest() catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); thrown.expect(CatalogException.class); thrown.expectMessage("not allowed"); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())) - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); } @@ -3522,7 +3533,7 @@ public void setPanelLockWithInterpretationWithNoPanelsTest() throws CatalogExcep catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), @@ -3534,7 +3545,7 @@ public void setPanelLockWithInterpretationWithNoPanelsTest() throws CatalogExcep thrown.expect(CatalogException.class); thrown.expectMessage("any of the case panels"); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); } @@ -3556,7 +3567,7 @@ public void setPanelLockWithInterpretationWithPanelSubsetTest() throws CatalogEx catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), @@ -3570,15 +3581,15 @@ public void setPanelLockWithInterpretationWithPanelSubsetTest() throws CatalogEx .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))), null, QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertFalse(clinicalAnalysis.isPanelLock()); + assertFalse(clinicalAnalysis.isPanelLocked()); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(1, clinicalAnalysis.getInterpretation().getPanels().size()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(1, clinicalAnalysis.getInterpretation().getPanels().size()); } @@ -3601,7 +3612,7 @@ public void setPanelLockWithInterpretationWithDifferentPanelsTest() throws Catal catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))), @@ -3621,7 +3632,7 @@ public void setPanelLockWithInterpretationWithDifferentPanelsTest() throws Catal thrown.expect(CatalogException.class); thrown.expectMessage("not defined by the case"); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); } @@ -3644,7 +3655,7 @@ public void updatePanelsFromClinicalAnalysisWithPanelLockTest() throws CatalogEx catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); @@ -3652,9 +3663,9 @@ public void updatePanelsFromClinicalAnalysisWithPanelLockTest() throws CatalogEx // Set panelLock to true catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), ownerToken); + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); thrown.expect(CatalogException.class); thrown.expectMessage("panelLock"); @@ -3682,7 +3693,7 @@ public void updatePanelLockWithDifferentPanels() throws CatalogException { catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); @@ -3690,15 +3701,15 @@ public void updatePanelLockWithDifferentPanels() throws CatalogException { // Set panelLock to true catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), ownerToken); + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); // Set panelLock to false catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(false), QueryOptions.empty(), ownerToken); + .setPanelLocked(false), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertFalse(clinicalAnalysis.isPanelLock()); + assertFalse(clinicalAnalysis.isPanelLocked()); Map actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); @@ -3714,9 +3725,9 @@ public void updatePanelLockWithDifferentPanels() throws CatalogException { thrown.expectMessage("panels"); // Set panelLock to true catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), ownerToken); + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); } @Test @@ -3732,14 +3743,14 @@ public void updatePanelsFromInterpretationWithLockedCATest() throws CatalogExcep .setId("analysis") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(proband) - .setPanelLock(true) + .setPanelLocked(true) .setPanels(panels.subList(0, 2)); OpenCGAResult result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertTrue(result.first().isPanelLock()); + assertTrue(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); @@ -3771,7 +3782,7 @@ public void updatePanelsFromInterpretationWithUnlockedCATest() throws CatalogExc catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java index c31ba8e9301..185b6e8c7bd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java @@ -5,7 +5,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -45,7 +44,7 @@ private DataResult createDummyFamily() throws CatalogException { private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new Status().setId(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION)) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); @@ -128,7 +127,7 @@ public void automaticallyLockInterpretationTest() throws CatalogException { interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation() - .setStatus(new Status("REJECTED", "", "", "")), + .setStatus(new ClinicalStatus("REJECTED", "", null, "", "", "", "")), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); assertEquals("REJECTED", interpretation.getStatus().getId()); assertTrue(interpretation.isLocked()); @@ -247,7 +246,7 @@ public void createInterpretationWithSubsetOfPanels() throws CatalogException { catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), updateParams, QueryOptions.empty(), ownerToken); updateParams = new ClinicalAnalysisUpdateParams() - .setPanelLock(true); + .setPanelLocked(true); catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), updateParams, QueryOptions.empty(), ownerToken); // Create interpretation with just panel1 diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java index 10b54926e6c..43191dc6838 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java @@ -244,7 +244,7 @@ public void panelIncrementVersionWithCasesAndInterpretations() throws CatalogExc ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getInterpretationManager().create(studyFqn, case2.getId(), interpretation5, ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); - catalogManager.getClinicalAnalysisManager().update(studyFqn, case2.getId(), new ClinicalAnalysisUpdateParams().setPanelLock(true), + catalogManager.getClinicalAnalysisManager().update(studyFqn, case2.getId(), new ClinicalAnalysisUpdateParams().setPanelLocked(true), QueryOptions.empty(), ownerToken); // case 3 diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 871ff79fff4..719382d28da 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -468,6 +468,7 @@ public class ParamConstants { + "Interpretation object is passed."; public static final String INTERPRETATION_ID_PARAM = "id"; public static final String INTERPRETATION_UUID_PARAM = "uuid"; + public static final String INTERPRETATION_NAME_PARAM = "name"; public static final String INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM = "clinicalAnalysisId"; public static final String INTERPRETATION_ANALYST_ID_PARAM = "analystId"; public static final String INTERPRETATION_METHOD_NAME_PARAM = "methodName"; @@ -1513,6 +1514,7 @@ public class ParamConstants { public static final String CLINICAL_ANALYSES_DESCRIPTION = "Comma separated list of clinical analysis IDs or names" + UP_TO_100; public static final String INTERPRETATION_ID_DESCRIPTION = "Comma separated list of Interpretation IDs" + UP_TO_100 + REGEX_SUPPORT; public static final String INTERPRETATION_UUID_DESCRIPTION = "Comma separated list of Interpretation UUIDs" + UP_TO_100; + public static final String INTERPRETATION_NAME_DESCRIPTION = "Comma separated list of Interpretation names" + UP_TO_100; public static final String INTERPRETATION_DESCRIPTION = "Comma separated list of clinical interpretation IDs " + UP_TO_100; public static final String PANEL_ID_DESCRIPTION = "Comma separated list of panel IDs " + UP_TO_100 + REGEX_SUPPORT; public static final String PANEL_UUID_DESCRIPTION = "Comma separated list of panel UUIDs " + UP_TO_100; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java index 060e27696c6..d2d48863d31 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java @@ -22,7 +22,6 @@ import org.opencb.biodata.models.clinical.ClinicalAudit; import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; @@ -100,9 +99,9 @@ public class ClinicalAnalysis extends Annotable { description = FieldConstants.CLINICAL_ANALYSIS_PANELS) private List panels; - @DataField(id = "panelLock", indexed = true, + @DataField(id = "panelLocked", indexed = true, description = FieldConstants.CLINICAL_ANALYSIS_PANEL_LOCK) - private boolean panelLock; + private boolean panelLocked; @DataField(id = "locked", indexed = true, description = FieldConstants.CLINICAL_ANALYSIS_LOCKED) @@ -225,19 +224,19 @@ public class ClinicalAnalysis extends Annotable { */ @DataField(id = "status", indexed = true, uncommentedClasses = {"Status"}, description = FieldConstants.GENERIC_STATUS_DESCRIPTION) - private Status status; + private ClinicalStatus status; public ClinicalAnalysis() { } public ClinicalAnalysis(String id, String description, Type type, Disorder disorder, List files, Individual proband, - Family family, List panels, boolean panelLock, boolean locked, Interpretation interpretation, + Family family, List panels, boolean panelLocked, boolean locked, Interpretation interpretation, List secondaryInterpretations, ClinicalConsentAnnotation consent, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, ClinicalPriorityAnnotation priority, List flags, String creationDate, String modificationDate, String dueDate, int release, int version, List comments, ClinicalAnalysisQualityControl qualityControl, List audit, ClinicalAnalysisInternal internal, - List annotationSets, Map attributes, Status status) { + List annotationSets, Map attributes, ClinicalStatus status) { this.id = id; this.description = description; this.type = type; @@ -246,7 +245,7 @@ public ClinicalAnalysis(String id, String description, Type type, Disorder disor this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.locked = locked; this.interpretation = interpretation; this.secondaryInterpretations = secondaryInterpretations; @@ -283,7 +282,7 @@ public String toString() { sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", locked=").append(locked); sb.append(", interpretation=").append(interpretation); sb.append(", secondaryInterpretations=").append(secondaryInterpretations); @@ -393,12 +392,23 @@ public ClinicalAnalysis setPanels(List panels) { return this; } + @Deprecated + @JsonIgnore public boolean isPanelLock() { - return panelLock; + return isPanelLocked(); } + @Deprecated public ClinicalAnalysis setPanelLock(boolean panelLock) { - this.panelLock = panelLock; + return setPanelLocked(panelLock); + } + + public boolean isPanelLocked() { + return panelLocked; + } + + public ClinicalAnalysis setPanelLocked(boolean panelLocked) { + this.panelLocked = panelLocked; return this; } @@ -602,11 +612,11 @@ public ClinicalAnalysis setAttributes(Map attributes) { return this; } - public Status getStatus() { + public ClinicalStatus getStatus() { return status; } - public ClinicalAnalysis setStatus(Status status) { + public ClinicalAnalysis setStatus(ClinicalStatus status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java index 1809e375628..157b9f0b12e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java @@ -46,7 +46,7 @@ public class ClinicalAnalysisCreateParams { private FamilyParam family; private List panels; - private Boolean panelLock; + private Boolean panelLocked; @Deprecated private ClinicalAnalystParam analyst; @@ -75,7 +75,7 @@ public ClinicalAnalysisCreateParams() { public ClinicalAnalysisCreateParams(String id, String description, ClinicalAnalysis.Type type, DisorderReferenceParam disorder, List files, ProbandParam proband, FamilyParam family, - List panels, Boolean panelLock, List analysts, + List panels, Boolean panelLocked, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, InterpretationCreateParams interpretation, ClinicalConsentAnnotationParam consent, String creationDate, String modificationDate, String dueDate, List comments, @@ -90,7 +90,7 @@ public ClinicalAnalysisCreateParams(String id, String description, ClinicalAnaly this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.report = report; this.request = request; this.responsible = responsible; @@ -120,7 +120,7 @@ public static ClinicalAnalysisCreateParams of(ClinicalAnalysis clinicalAnalysis) clinicalAnalysis.getPanels() != null ? clinicalAnalysis.getPanels().stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList()) : null, - clinicalAnalysis.isPanelLock(), + clinicalAnalysis.isPanelLocked(), clinicalAnalysis.getAnalysts() != null ? clinicalAnalysis.getAnalysts().stream().map(ClinicalAnalystParam::of).collect(Collectors.toList()) : null, @@ -155,7 +155,7 @@ public String toString() { sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", analysts=").append(analysts); sb.append(", report=").append(report); sb.append(", request=").append(request); @@ -231,14 +231,14 @@ public ClinicalAnalysis toClinicalAnalysis() { } return new ClinicalAnalysis(id, description, type, disorder != null ? disorder.toDisorder() : null, caFiles, individual, f, - diseasePanelList, panelLock != null ? panelLock : false, false, primaryInterpretation, new LinkedList<>(), + diseasePanelList, panelLocked != null ? panelLocked : false, false, primaryInterpretation, new LinkedList<>(), consent != null ? consent.toClinicalConsentAnnotation() : null, clinicalAnalystList, report, request, responsible, priority != null ? priority.toClinicalPriorityAnnotation() : null, flags != null ? flags.stream().map(FlagValueParam::toFlagAnnotation).collect(Collectors.toList()) : null, creationDate, modificationDate, dueDate, 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, new LinkedList<>(), null, - annotationSets, attributes, status != null ? status.toStatus() : null); + annotationSets, attributes, status != null ? status.toClinicalStatus() : null); } public String getId() { @@ -313,12 +313,12 @@ public ClinicalAnalysisCreateParams setPanels(List panels) return this; } - public Boolean getPanelLock() { - return panelLock; + public Boolean getPanelLocked() { + return panelLocked; } - public ClinicalAnalysisCreateParams setPanelLock(Boolean panelLock) { - this.panelLock = panelLock; + public ClinicalAnalysisCreateParams setPanelLocked(Boolean panelLocked) { + this.panelLocked = panelLocked; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java index a529d6d7d27..0cc243f78a3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java @@ -21,18 +21,18 @@ public class ClinicalAnalysisInternal extends Internal { - private ClinicalAnalysisStatus status; + private ClinicalAnalysisStatusOld status; public ClinicalAnalysisInternal() { } - public ClinicalAnalysisInternal(String registrationDate, String modificationDate, ClinicalAnalysisStatus status) { + public ClinicalAnalysisInternal(String registrationDate, String modificationDate, ClinicalAnalysisStatusOld status) { super(null, registrationDate, modificationDate); this.status = status; } public static ClinicalAnalysisInternal init() { - return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new ClinicalAnalysisStatus()); + return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new ClinicalAnalysisStatusOld()); } @Override @@ -45,11 +45,11 @@ public String toString() { return sb.toString(); } - public ClinicalAnalysisStatus getStatus() { + public ClinicalAnalysisStatusOld getStatus() { return status; } - public ClinicalAnalysisInternal setStatus(ClinicalAnalysisStatus status) { + public ClinicalAnalysisInternal setStatus(ClinicalAnalysisStatusOld status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatusOld.java similarity index 91% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatusOld.java index b898a7c21d5..5e1ec106c57 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatusOld.java @@ -21,7 +21,7 @@ import java.util.Arrays; import java.util.List; -public class ClinicalAnalysisStatus extends InternalStatus { +public class ClinicalAnalysisStatusOld extends InternalStatus { public static final String INCOMPLETE = "INCOMPLETE"; public static final String READY_FOR_VALIDATION = "READY_FOR_VALIDATION"; @@ -41,7 +41,7 @@ public class ClinicalAnalysisStatus extends InternalStatus { READY_FOR_INTERPRETATION, INTERPRETATION_IN_PROGRESS, READY_FOR_INTEPRETATION_REVIEW, INTERPRETATION_REVIEW_IN_PROGRESS, READY_FOR_REPORT, REPORT_IN_PROGRESS, DONE, REVIEW_IN_PROGRESS, CLOSED, REJECTED); - public ClinicalAnalysisStatus(String status, String message) { + public ClinicalAnalysisStatusOld(String status, String message) { if (isValid(status)) { init(status, message); } else { @@ -49,11 +49,11 @@ public ClinicalAnalysisStatus(String status, String message) { } } - public ClinicalAnalysisStatus(String status) { + public ClinicalAnalysisStatusOld(String status) { this(status, ""); } - public ClinicalAnalysisStatus() { + public ClinicalAnalysisStatusOld() { this(READY_FOR_INTERPRETATION, ""); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java index 8f6b5e1cef3..e22feef43ae 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java @@ -44,7 +44,7 @@ public class ClinicalAnalysisUpdateParams { private List files; private List panels; - private Boolean panelLock; + private Boolean panelLocked; private ProbandParam proband; private FamilyParam family; @@ -77,7 +77,7 @@ public ClinicalAnalysisUpdateParams() { public ClinicalAnalysisUpdateParams(String id, String description, ClinicalAnalysis.Type type, DisorderReferenceParam disorder, List files, ProbandParam proband, FamilyParam family, - List panels, Boolean panelLock, Boolean locked, + List panels, Boolean panelLocked, Boolean locked, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, ClinicalAnalysisQualityControlUpdateParam qualityControl, ClinicalConsentAnnotationParam consent, String creationDate, String modificationDate, @@ -92,7 +92,7 @@ public ClinicalAnalysisUpdateParams(String id, String description, ClinicalAnaly this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.locked = locked; this.analysts = analysts; this.report = report; @@ -123,7 +123,7 @@ public ClinicalAnalysis toClinicalAnalysis() { proband != null ? proband.toIndividual() : null, family != null ? family.toFamily() : null, panels != null ? panels.stream().map(p -> new Panel().setId(p.getId())).collect(Collectors.toList()) : null, - panelLock != null ? panelLock : false, + panelLocked != null ? panelLocked : false, locked != null && locked, null, null, consent != null ? consent.toClinicalConsentAnnotation() : null, @@ -136,7 +136,7 @@ public ClinicalAnalysis toClinicalAnalysis() { 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, null, null, annotationSets, attributes, - status != null ? status.toStatus() : null); + status != null ? status.toClinicalStatus() : null); } @Override @@ -148,7 +148,7 @@ public String toString() { sb.append(", disorder=").append(disorder); sb.append(", files=").append(files); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", locked=").append(locked); @@ -243,12 +243,22 @@ public ClinicalAnalysisUpdateParams setFamily(FamilyParam family) { return this; } + @Deprecated public Boolean getPanelLock() { - return panelLock; + return getPanelLocked(); } + @Deprecated public ClinicalAnalysisUpdateParams setPanelLock(Boolean panelLock) { - this.panelLock = panelLock; + return setPanelLocked(panelLock); + } + + public Boolean getPanelLocked() { + return panelLocked; + } + + public ClinicalAnalysisUpdateParams setPanelLocked(Boolean panelLocked) { + this.panelLocked = panelLocked; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java new file mode 100644 index 00000000000..baa92b6f22a --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java @@ -0,0 +1,77 @@ +package org.opencb.opencga.core.models.clinical; + +public class ClinicalStatus extends ClinicalStatusValue { + + private String author; + private String version; + private String commit; + private String date; + + public ClinicalStatus() { + } + + public ClinicalStatus(String id, String description, ClinicalStatusType type, String author, String version, String commit, + String date) { + super(id, description, type); + this.author = author; + this.version = version; + this.commit = commit; + this.date = date; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ClinicalStatus{"); + sb.append("id='").append(id).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", type=").append(type); + sb.append(", author='").append(author).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append(", commit='").append(commit).append('\''); + sb.append(", date='").append(date).append('\''); + sb.append('}'); + return sb.toString(); + } + + @Override + public ClinicalStatus setId(String id) { + super.setId(id); + return this; + } + + public String getVersion() { + return version; + } + + public ClinicalStatus setVersion(String version) { + this.version = version; + return this; + } + + public String getCommit() { + return commit; + } + + public ClinicalStatus setCommit(String commit) { + this.commit = commit; + return this; + } + + public String getAuthor() { + return author; + } + + public ClinicalStatus setAuthor(String author) { + this.author = author; + return this; + } + + public String getDate() { + return date; + } + + public ClinicalStatus setDate(String date) { + this.date = date; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java index 9efa33d4a33..68816055dde 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java @@ -2,15 +2,15 @@ public class ClinicalStatusValue { - private String id; - private String description; - private ClinicalStatusType type; + protected String id; + protected String description; + protected ClinicalStatusType type; public enum ClinicalStatusType { NOT_STARTED, - IN_PROGRESS, - CLOSED, - UNKNOWN + ACTIVE, + DONE, + CLOSED } public ClinicalStatusValue() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java index 273ce53a4c9..a6774d4f751 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java @@ -20,7 +20,6 @@ import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.models.IPrivateStudyUid; @@ -54,35 +53,56 @@ public class Interpretation extends org.opencb.biodata.models.clinical.interpret description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; + @DataField(id = "status", indexed = true, + description = FieldConstants.GENERIC_STATUS_DESCRIPTION) + private ClinicalStatus status; + public Interpretation() { super(); } - public Interpretation(String id, String description, String clinicalAnalysisId, ClinicalAnalyst analyst, + public Interpretation(String id, String name, String description, String clinicalAnalysisId, ClinicalAnalyst analyst, InterpretationMethod method, String creationDate, String modificationDate, boolean locked, List primaryFindings, List secondaryFindings, List panels, - List comments, Status status, Map attributes) { - super(id, "", description, clinicalAnalysisId, analyst, method, primaryFindings, secondaryFindings, comments, null, status, - creationDate, modificationDate, locked, 0, attributes); - + List comments, ClinicalStatus status, Map attributes) { + super(id, "", name, description, clinicalAnalysisId, analyst, method, primaryFindings, secondaryFindings, comments, null, locked, + creationDate, modificationDate, 0, attributes); + this.status = status; this.panels = panels; } public Interpretation(org.opencb.biodata.models.clinical.interpretation.Interpretation interpretation) { - this(interpretation.getId(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), interpretation.getAnalyst(), - interpretation.getMethod(), interpretation.getCreationDate(), interpretation.getModificationDate(), - interpretation.isLocked(), interpretation.getPrimaryFindings(), interpretation.getSecondaryFindings(), - Collections.emptyList(), interpretation.getComments(), interpretation.getStatus(), interpretation.getAttributes()); + this(interpretation.getId(), interpretation.getName(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), + interpretation.getAnalyst(), interpretation.getMethod(), interpretation.getCreationDate(), + interpretation.getModificationDate(), interpretation.isLocked(), interpretation.getPrimaryFindings(), + interpretation.getSecondaryFindings(), Collections.emptyList(), interpretation.getComments(), null, + interpretation.getAttributes()); } @Override public String toString() { final StringBuilder sb = new StringBuilder("Interpretation{"); - sb.append("studyUid=").append(studyUid); + sb.append("id='").append(id).append('\''); + sb.append(", uuid='").append(uuid).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", studyUid=").append(studyUid); sb.append(", uid=").append(uid); sb.append(", panels=").append(panels); sb.append(", internal=").append(internal); sb.append(", release=").append(release); + sb.append(", status=").append(status); + sb.append(", clinicalAnalysisId='").append(clinicalAnalysisId).append('\''); + sb.append(", analyst=").append(analyst); + sb.append(", method=").append(method); + sb.append(", primaryFindings=").append(primaryFindings); + sb.append(", secondaryFindings=").append(secondaryFindings); + sb.append(", comments=").append(comments); + sb.append(", stats=").append(stats); + sb.append(", locked=").append(locked); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", version=").append(version); sb.append('}'); return sb.toString(); } @@ -185,14 +205,17 @@ public Interpretation setComments(List comments) { } @Override - public Interpretation setStatus(Status status) { - super.setStatus(status); + public Interpretation setLocked(boolean locked) { + super.setLocked(locked); return this; } - @Override - public Interpretation setLocked(boolean locked) { - super.setLocked(locked); + public ClinicalStatus getStatus() { + return status; + } + + public Interpretation setStatus(ClinicalStatus status) { + this.status = status; return this; } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java index e35ccec46f8..9c710fa0493 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java @@ -26,13 +26,13 @@ import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; public class InterpretationCreateParams { + private String name; private String description; private String clinicalAnalysisId; private String creationDate; @@ -50,11 +50,12 @@ public class InterpretationCreateParams { public InterpretationCreateParams() { } - public InterpretationCreateParams(String description, String clinicalAnalysisId, String creationDate, String modificationDate, + public InterpretationCreateParams(String name, String description, String clinicalAnalysisId, String creationDate, String modificationDate, ClinicalAnalystParam analyst, InterpretationMethod method, List primaryFindings, List secondaryFindings, List panels, List comments, StatusParam status, Boolean locked, Map attributes) { + this.name = name; this.description = description; this.clinicalAnalysisId = clinicalAnalysisId; this.creationDate = creationDate; @@ -71,7 +72,7 @@ public InterpretationCreateParams(String description, String clinicalAnalysisId, } public static InterpretationCreateParams of(Interpretation interpretation) { - return new InterpretationCreateParams(interpretation.getDescription(), + return new InterpretationCreateParams(interpretation.getName(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), interpretation.getCreationDate(), interpretation.getModificationDate(), ClinicalAnalystParam.of(interpretation.getAnalyst()), interpretation.getMethod(), interpretation.getPrimaryFindings(), interpretation.getSecondaryFindings(), @@ -89,7 +90,8 @@ public static InterpretationCreateParams of(Interpretation interpretation) { @Override public String toString() { final StringBuilder sb = new StringBuilder("InterpretationCreateParams{"); - sb.append("description='").append(description).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); sb.append(", clinicalAnalysisId='").append(clinicalAnalysisId).append('\''); sb.append(", creationDate='").append(creationDate).append('\''); sb.append(", modificationDate='").append(modificationDate).append('\''); @@ -99,25 +101,34 @@ public String toString() { sb.append(", secondaryFindings=").append(secondaryFindings); sb.append(", panels=").append(panels); sb.append(", comments=").append(comments); - sb.append(", status=").append(status); sb.append(", locked=").append(locked); + sb.append(", status=").append(status); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); } public Interpretation toClinicalInterpretation() { - return new Interpretation(null, description, clinicalAnalysisId, analyst != null ? analyst.toClinicalAnalyst() : null, method, + return new Interpretation("", name, description, clinicalAnalysisId, analyst != null ? analyst.toClinicalAnalyst() : null, method, creationDate, modificationDate, locked != null ? locked : false, primaryFindings, secondaryFindings, panels != null ? panels.stream().map(p -> new Panel().setId(p.getId())).collect(Collectors.toList()) : null, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, - status != null ? status.toStatus() : null, attributes); + status != null ? status.toClinicalStatus() : null, attributes); } public ObjectMap toInterpretationObjectMap() throws JsonProcessingException { return new ObjectMap(getUpdateObjectMapper().writeValueAsString(this.toClinicalInterpretation())); } + public String getName() { + return name; + } + + public InterpretationCreateParams setName(String name) { + this.name = name; + return this; + } + public String getDescription() { return description; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java index c0bee8b89ce..3da416d1b28 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java @@ -31,6 +31,7 @@ public class InterpretationUpdateParams { + private String name; private String description; private ClinicalAnalystParam analyst; private InterpretationMethod method; @@ -47,11 +48,12 @@ public class InterpretationUpdateParams { public InterpretationUpdateParams() { } - public InterpretationUpdateParams(String description, ClinicalAnalystParam analyst, InterpretationMethod method, + public InterpretationUpdateParams(String name, String description, ClinicalAnalystParam analyst, InterpretationMethod method, String creationDate, String modificationDate, List primaryFindings, List secondaryFindings, List panels, List comments, StatusParam status, Boolean locked, Map attributes) { + this.name = name; this.description = description; this.analyst = analyst; this.method = method; @@ -74,7 +76,8 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { @Override public String toString() { final StringBuilder sb = new StringBuilder("InterpretationUpdateParams{"); - sb.append("description='").append(description).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); sb.append(", analyst=").append(analyst); sb.append(", method=").append(method); sb.append(", creationDate='").append(creationDate).append('\''); @@ -90,6 +93,15 @@ public String toString() { return sb.toString(); } + public String getName() { + return name; + } + + public InterpretationUpdateParams setName(String name) { + this.name = name; + return this; + } + public String getDescription() { return description; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java index 5a1518440ab..59827d9655d 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java @@ -18,7 +18,7 @@ public class IndexStatus extends InternalStatus { public IndexStatus(String status, String message) { if (isValid(status)) { - init(status, status, message); + init(status, message); } else { throw new IllegalArgumentException("Unknown status " + status); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java index a934efdc3ee..a1d28c6914b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java @@ -57,8 +57,8 @@ public InternalStatus(String id, String description) { } } - public InternalStatus(String id, String name, String description, String date, String version, String commit) { - super(id, name, description, date); + public InternalStatus(String id, String description, String date, String version, String commit) { + super(id, description, date); if (!isValid(id)) { throw new IllegalArgumentException("Unknown status id '" + id + "'"); } @@ -112,12 +112,7 @@ private static void fillPositiveNegativeList(String[] statusList, List p } protected void init(String statusId, String description) { - init(statusId, statusId, description); - } - - protected void init(String statusId, String statusName, String description) { super.id = statusId; - super.name = statusName; super.description = description; super.date = TimeUtils.getTime(); this.version = GitRepositoryState.getInstance().getBuildVersion(); @@ -150,7 +145,6 @@ public String toString() { sb.append("version='").append(version).append('\''); sb.append(", commit='").append(commit).append('\''); sb.append(", id='").append(id).append('\''); - sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); sb.append(", date='").append(date).append('\''); sb.append('}'); @@ -184,15 +178,6 @@ public InternalStatus setId(String id) { return this; } - public String getName() { - return StringUtils.isNotEmpty(name) ? name : id; - } - - public InternalStatus setName(String name) { - this.name = name; - return this; - } - public String getDate() { return date; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java index 806ef09be4d..d826354369e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java @@ -2,6 +2,7 @@ import org.opencb.biodata.models.common.Status; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.clinical.ClinicalStatus; public class StatusParam { @@ -18,10 +19,18 @@ public static StatusParam of(Status status) { return status != null ? new StatusParam(status.getId()) : null; } + public static StatusParam of(ClinicalStatus status) { + return status != null ? new StatusParam(status.getId()) : null; + } + public Status toStatus() { return new Status(id, "", "", TimeUtils.getTime()); } + public ClinicalStatus toClinicalStatus() { + return new ClinicalStatus(id, "", null, "", "", "", ""); + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("StatusParam{"); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java index 3dbe5f21f7d..2edb133227b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java @@ -23,7 +23,7 @@ public class VariantIndexStatus extends IndexStatus { public VariantIndexStatus(String status, String message) { if (isValid(status)) { - init(status, status, message); + init(status, message); } else { throw new IllegalArgumentException("Unknown status " + status); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java index 97d09661407..cd8e3e5d36c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java @@ -8,7 +8,7 @@ public class ClinicalAnalysisStudyConfiguration { - private Map> status; + private List status; private InterpretationStudyConfiguration interpretation; private List priorities; private Map> flags; @@ -18,9 +18,9 @@ public class ClinicalAnalysisStudyConfiguration { public ClinicalAnalysisStudyConfiguration() { } - public ClinicalAnalysisStudyConfiguration(Map> status, - InterpretationStudyConfiguration interpretation, List priorities, - Map> flags, ClinicalConsentConfiguration consent) { + public ClinicalAnalysisStudyConfiguration(List status, InterpretationStudyConfiguration interpretation, + List priorities, Map> flags, + ClinicalConsentConfiguration consent) { this.status = status; this.interpretation = interpretation; this.priorities = priorities; @@ -29,20 +29,22 @@ public ClinicalAnalysisStudyConfiguration(Map> status = new HashMap<>(); - Map> interpretationStatus = new HashMap<>(); + List clinicalStatusValueList = new ArrayList<>(4); + List interpretationStatusList = new ArrayList<>(3); List priorities = new ArrayList<>(5); Map> flags = new HashMap<>(); List clinicalConsentList = new ArrayList<>(); - List clinicalStatusValueList = new ArrayList<>(4); clinicalStatusValueList.add( new ClinicalStatusValue("READY_FOR_INTERPRETATION", "The Clinical Analysis is ready for interpretations", ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) ); clinicalStatusValueList.add( new ClinicalStatusValue("READY_FOR_REPORT", "The Interpretation is finished and it is to create the report", - ClinicalStatusValue.ClinicalStatusType.IN_PROGRESS) + ClinicalStatusValue.ClinicalStatusType.ACTIVE) + ); + clinicalStatusValueList.add( + new ClinicalStatusValue("DONE", "The Clinical Analysis is done", ClinicalStatusValue.ClinicalStatusType.DONE) ); clinicalStatusValueList.add( new ClinicalStatusValue("CLOSED", "The Clinical Analysis is closed", ClinicalStatusValue.ClinicalStatusType.CLOSED) @@ -50,21 +52,12 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { clinicalStatusValueList.add( new ClinicalStatusValue("REJECTED", "The Clinical Analysis is rejected", ClinicalStatusValue.ClinicalStatusType.CLOSED) ); - status.put(ClinicalAnalysis.Type.FAMILY, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.CANCER, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.COHORT, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.SINGLE, clinicalStatusValueList); - List interpretationStatusList = new ArrayList<>(3); - interpretationStatusList.add(new ClinicalStatusValue("IN_PROGRESS", "Interpretation in progress", ClinicalStatusValue.ClinicalStatusType.IN_PROGRESS)); + interpretationStatusList.add(new ClinicalStatusValue("NOT_STARTED", "Interpretation not started", ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)); + interpretationStatusList.add(new ClinicalStatusValue("IN_PROGRESS", "Interpretation in progress", ClinicalStatusValue.ClinicalStatusType.ACTIVE)); + interpretationStatusList.add(new ClinicalStatusValue("DONE", "Interpretation done", ClinicalStatusValue.ClinicalStatusType.DONE)); interpretationStatusList.add(new ClinicalStatusValue("READY", "Interpretation ready", ClinicalStatusValue.ClinicalStatusType.CLOSED)); interpretationStatusList.add(new ClinicalStatusValue("REJECTED", "Interpretation rejected", ClinicalStatusValue.ClinicalStatusType.CLOSED)); - interpretationStatus.put(ClinicalAnalysis.Type.FAMILY, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.CANCER, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.COHORT, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.SINGLE, interpretationStatusList); priorities.add(new ClinicalPriorityValue("URGENT", "Highest priority of all", 1, false)); priorities.add(new ClinicalPriorityValue("HIGH", "Second highest priority of all", 2, false)); @@ -91,8 +84,8 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { clinicalConsentList.add(new ClinicalConsent("CARRIER_FINDINGS", "Carrier findings", "")); clinicalConsentList.add(new ClinicalConsent("RESEARCH_FINDINGS", "Research findings", "")); - return new ClinicalAnalysisStudyConfiguration(status, - new InterpretationStudyConfiguration(interpretationStatus, Collections.emptyList(), Collections.emptyMap(), + return new ClinicalAnalysisStudyConfiguration(clinicalStatusValueList, + new InterpretationStudyConfiguration(interpretationStatusList, Collections.emptyList(), Collections.emptyMap(), Collections.emptyList()), priorities, flags, new ClinicalConsentConfiguration(clinicalConsentList)); } @@ -108,11 +101,11 @@ public String toString() { return sb.toString(); } - public Map> getStatus() { + public List getStatus() { return status; } - public ClinicalAnalysisStudyConfiguration setStatus(Map> status) { + public ClinicalAnalysisStudyConfiguration setStatus(List status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java index b038fd5674d..0b9c98a780a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java @@ -1,6 +1,5 @@ package org.opencb.opencga.core.models.study.configuration; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; import java.util.List; @@ -8,16 +7,16 @@ public class InterpretationStudyConfiguration { - private Map> status; + private List status; private List variantCallers; + @Deprecated private Map defaultFilter; private List inclusion; public InterpretationStudyConfiguration() { } - public InterpretationStudyConfiguration(Map> status, - List variantCallers, + public InterpretationStudyConfiguration(List status, List variantCallers, Map defaultFilter, List inclusion) { this.status = status; this.variantCallers = variantCallers; @@ -36,11 +35,11 @@ public String toString() { return sb.toString(); } - public Map> getStatus() { + public List getStatus() { return status; } - public InterpretationStudyConfiguration setStatus(Map> status) { + public InterpretationStudyConfiguration setStatus(List status) { this.status = status; return this; } @@ -54,10 +53,12 @@ public InterpretationStudyConfiguration setVariantCallers(List getDefaultFilter() { return defaultFilter; } + @Deprecated public InterpretationStudyConfiguration setDefaultFilter(Map defaultFilter) { this.defaultFilter = defaultFilter; return this; diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java index 11bb051b37f..4e110e8b4b0 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java @@ -767,6 +767,7 @@ public Response interpretationSearch( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INTERPRETATION_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ID_PARAM) String id, @ApiParam(value = ParamConstants.INTERPRETATION_UUID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.INTERPRETATION_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_NAME_PARAM) String name, @ApiParam(value = ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM) String clinicalAnalysisId, @ApiParam(value = ParamConstants.INTERPRETATION_ANALYST_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ANALYST_ID_PARAM) String analystId, @ApiParam(value = ParamConstants.INTERPRETATION_METHOD_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_METHOD_NAME_PARAM) String method, @@ -800,6 +801,7 @@ public Response interpretationDistinct( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INTERPRETATION_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ID_PARAM) String id, @ApiParam(value = ParamConstants.INTERPRETATION_UUID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.INTERPRETATION_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_NAME_PARAM) String name, @ApiParam(value = ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM) String clinicalAnalysisId, @ApiParam(value = ParamConstants.INTERPRETATION_ANALYST_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ANALYST_ID_PARAM) String analystId, @ApiParam(value = ParamConstants.INTERPRETATION_METHOD_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_METHOD_NAME_PARAM) String methodsName, From d45c720d697b9abf5f1d2061c05d5b9179f8e056 Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 10 Jun 2024 13:22:55 +0200 Subject: [PATCH 043/128] app: add some migrations, #TASK-5964 --- .../v3_2_0/AddInterpretationName.java | 35 +++++++++++ .../UpdateClinicalStudyConfiguration.java | 58 +++++++++++++++++++ .../managers/ClinicalAnalysisManager.java | 11 ++-- .../managers/ClinicalAnalysisManagerTest.java | 10 ++-- .../managers/InterpretationManagerTest.java | 3 +- .../ClinicalAnalysisStudyConfiguration.java | 15 ++--- 6 files changed, 109 insertions(+), 23 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddInterpretationName.java create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/UpdateClinicalStudyConfiguration.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddInterpretationName.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddInterpretationName.java new file mode 100644 index 00000000000..d464e3b559b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddInterpretationName.java @@ -0,0 +1,35 @@ +package org.opencb.opencga.app.migrations.v3_2_0; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "add_interpretation_name", description = "Add Interpretation name #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) +public class AddInterpretationName extends MigrationTool { + + @Override + protected void run() throws Exception { + // Add new Interpretation name field + Bson nameDoesNotExistQuery = Filters.exists("name", false); + Bson projection = Projections.include("id"); + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + migrateCollection(collection, nameDoesNotExistQuery, projection, + (document, bulk) -> { + Document updateDocument = new Document() + .append("name", document.get("id")); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); + }); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/UpdateClinicalStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/UpdateClinicalStudyConfiguration.java new file mode 100644 index 00000000000..7618586c2a1 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/UpdateClinicalStudyConfiguration.java @@ -0,0 +1,58 @@ +package org.opencb.opencga.app.migrations.v3_2_0; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; + +import java.util.Arrays; +import java.util.List; + +@Migration(id = "update_clinical_study_configuration", + description = "Setting new default Clinical Study Configuration status values, #TASK-5964", + version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) +public class UpdateClinicalStudyConfiguration extends MigrationTool { + + @Override + protected void run() throws Exception { + ClinicalAnalysisStudyConfiguration clinicalAnalysisStudyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + Document clinicalConfigurationDocument = convertToDocument(clinicalAnalysisStudyConfiguration); + + Bson query = new Document(); + Bson projection = Projections.include("configuration", "fqn"); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + Document configuration = document.get("configuration", Document.class); + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + if (configuration == null) { + logger.warn("Found empty study configuration in study '{}'. Creating a new one...", document.get("fqn")); + updateDocument.getSet().put("configuration", clinicalConfigurationDocument); + } else { + Object statusObject = configuration.get("status"); + if (statusObject instanceof List) { + // The study seems to be already migrated. Skipping... + logger.warn("Study '{}' seems to be already migrated. Skipping...", document.get("fqn")); + return; + } + // Study needs to be migrated + logger.info("Migrating study '{}'", document.get("fqn")); + updateDocument.getSet().put("configuration.status", clinicalConfigurationDocument.get("status")); + updateDocument.getSet().put("configuration.flags", clinicalConfigurationDocument.get("flags")); + updateDocument.getSet().put("configuration.interpretation.status", clinicalConfigurationDocument.get("interpretation", Document.class).get("status")); + } + logger.debug("Updating study '{}': {}", document.get("fqn"), updateDocument.toFinalUpdateDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 73f0051e02d..34ba5b01821 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -893,12 +893,11 @@ private void validateCustomFlagParameters(ClinicalAnalysis clinicalAnalysis, Cli throws CatalogException { // Flag definition if (CollectionUtils.isNotEmpty(clinicalAnalysis.getFlags())) { - if (CollectionUtils.isEmpty(clinicalConfiguration.getFlags().get(clinicalAnalysis.getType()))) { - throw new CatalogException("Missing flags configuration in study for type '" + clinicalAnalysis.getType() - + "'. Please add a proper set of valid priorities."); + if (CollectionUtils.isEmpty(clinicalConfiguration.getFlags())) { + throw new CatalogException("Missing flags configuration. Please add a proper set of valid flags."); } Map supportedFlags = new HashMap<>(); - for (FlagValue flagValue : clinicalConfiguration.getFlags().get(clinicalAnalysis.getType())) { + for (FlagValue flagValue : clinicalConfiguration.getFlags()) { supportedFlags.put(flagValue.getId(), flagValue); } @@ -912,8 +911,8 @@ private void validateCustomFlagParameters(ClinicalAnalysis clinicalAnalysis, Cli flag.setDescription(supportedFlags.get(flag.getId()).getDescription()); flag.setDate(TimeUtils.getTime()); } else { - throw new CatalogException("Flag '" + flag.getId() + "' not supported. Supported flags for Clinical Analyses of " - + "type '" + clinicalAnalysis.getType() + "' are: '" + String.join(", ", supportedFlags.keySet()) + "'."); + throw new CatalogException("Flag '" + flag.getId() + "' not supported. Supported flags for Clinical Analyses are: '" + + String.join(", ", supportedFlags.keySet()) + "'."); } } clinicalAnalysis.setFlags(new ArrayList<>(flagMap.values())); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index ae0ffc77613..71eb78c3112 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -1764,9 +1764,9 @@ public void updateCustomFlagTest() throws CatalogException { DataResult dummyEnvironment = createDummyEnvironment(true, false); - FlagValue flag1 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(1); - FlagValue flag2 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(3); - FlagValue flag3 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(4); + FlagValue flag1 = configuration.getFlags().get(1); + FlagValue flag2 = configuration.getFlags().get(3); + FlagValue flag3 = configuration.getFlags().get(4); ObjectMap actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FLAGS.key(), ParamUtils.BasicUpdateAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); @@ -1803,8 +1803,8 @@ public void updateCustomFlagTest() throws CatalogException { } // Set other flags - flag1 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(0); - flag2 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(2); + flag1 = configuration.getFlags().get(0); + flag2 = configuration.getFlags().get(2); actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FLAGS.key(), ParamUtils.BasicUpdateAction.SET); options = new QueryOptions(Constants.ACTIONS, actionMap); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java index 185b6e8c7bd..2ba297c49f3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java @@ -1,7 +1,6 @@ package org.opencb.opencga.catalog.managers; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -117,7 +116,7 @@ public void automaticallyLockInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); - assertTrue(StringUtils.isEmpty(interpretation.getStatus().getId())); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, interpretation.getStatus().getType()); assertFalse(interpretation.isLocked()); interpretation = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java index cd8e3e5d36c..900bfd07d78 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java @@ -11,7 +11,7 @@ public class ClinicalAnalysisStudyConfiguration { private List status; private InterpretationStudyConfiguration interpretation; private List priorities; - private Map> flags; + private List flags; private ClinicalConsentConfiguration consent; @@ -19,7 +19,7 @@ public ClinicalAnalysisStudyConfiguration() { } public ClinicalAnalysisStudyConfiguration(List status, InterpretationStudyConfiguration interpretation, - List priorities, Map> flags, + List priorities, List flags, ClinicalConsentConfiguration consent) { this.status = status; this.interpretation = interpretation; @@ -73,11 +73,6 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { flagValueList.add(new FlagValue("UNUSUAL_KARYOTYPE", "")); flagValueList.add(new FlagValue("SUSPECTED_MOSAICISM", "")); flagValueList.add(new FlagValue("LOW_QUALITY_SAMPLE", "")); - flags.put(ClinicalAnalysis.Type.FAMILY, flagValueList); - flags.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, flagValueList); - flags.put(ClinicalAnalysis.Type.CANCER, flagValueList); - flags.put(ClinicalAnalysis.Type.COHORT, flagValueList); - flags.put(ClinicalAnalysis.Type.SINGLE, flagValueList); clinicalConsentList.add(new ClinicalConsent("PRIMARY_FINDINGS", "Primary findings", "")); clinicalConsentList.add(new ClinicalConsent("SECONDARY_FINDINGS", "Secondary findings", "")); @@ -86,7 +81,7 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { return new ClinicalAnalysisStudyConfiguration(clinicalStatusValueList, new InterpretationStudyConfiguration(interpretationStatusList, Collections.emptyList(), Collections.emptyMap(), - Collections.emptyList()), priorities, flags, new ClinicalConsentConfiguration(clinicalConsentList)); + Collections.emptyList()), priorities, flagValueList, new ClinicalConsentConfiguration(clinicalConsentList)); } @Override @@ -128,11 +123,11 @@ public ClinicalAnalysisStudyConfiguration setPriorities(List> getFlags() { + public List getFlags() { return flags; } - public ClinicalAnalysisStudyConfiguration setFlags(Map> flags) { + public ClinicalAnalysisStudyConfiguration setFlags(List flags) { this.flags = flags; return this; } From 2c1608eb5813d44162f4ce0bf60e385a6d070e57 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 11 Jun 2024 10:45:22 +0200 Subject: [PATCH 044/128] app: add new ClinicalStatus migration, #TASK-5964 --- .../AddInterpretationName.java | 2 +- .../TASK_5964/AddNewClinicalStatusValues.java | 105 ++++++++++++++++++ ...AddVersionToClinicalAnalysisMigration.java | 2 +- .../UpdateClinicalStudyConfiguration.java | 2 +- .../ClinicalAnalysisMongoDBAdaptor.java | 2 +- .../managers/ClinicalAnalysisManager.java | 9 +- .../managers/ClinicalAnalysisManagerTest.java | 18 +-- .../managers/InterpretationManagerTest.java | 2 +- .../clinical/ClinicalAnalysisInternal.java | 14 +-- .../clinical/ClinicalAnalysisStatusOld.java | 70 ------------ 10 files changed, 128 insertions(+), 98 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/{ => TASK_5964}/AddInterpretationName.java (96%) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/{ => TASK_5964}/AddVersionToClinicalAnalysisMigration.java (98%) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/{ => TASK_5964}/UpdateClinicalStudyConfiguration.java (98%) delete mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatusOld.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddInterpretationName.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddInterpretationName.java similarity index 96% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddInterpretationName.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddInterpretationName.java index d464e3b559b..960424a3953 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddInterpretationName.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddInterpretationName.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0; +package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java new file mode 100644 index 00000000000..e2942f3fcf9 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java @@ -0,0 +1,105 @@ +package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.clinical.ClinicalStatus; +import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; + +import java.util.Arrays; +import java.util.List; + +@Migration(id = "add_new_clinical_status", + description = "Add new ClinicalStatus to ClinicalAnalysis and Interpretation, #TASK-5964", + version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240611) +public class AddNewClinicalStatusValues extends MigrationTool { + + @Override + protected void run() throws Exception { + ClinicalAnalysisStudyConfiguration defaultConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + + Bson query = Filters.exists("status.author", false); + Bson projection = Projections.include("status", "id"); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + String clinicalId = document.getString("id"); + Document status = document.get("status", Document.class); + + Document newStatus = fillStatusValues("Clinical Analysis", clinicalId, status, defaultConfiguration.getStatus()); + updateDocument.getSet().put("status", newStatus); + + Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); + + logger.debug("Updating clinical analysis '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); + }); + } + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + String interpretationId = document.getString("id"); + Document status = document.get("status", Document.class); + + Document newStatus = fillStatusValues("Interpretation", interpretationId, status, + defaultConfiguration.getInterpretation().getStatus()); + updateDocument.getSet().put("status", newStatus); + + Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); + + logger.debug("Updating interpretation '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); + }); + } + } + + private Document fillStatusValues(String entity, String id, Document status, List statusValueList) { + ClinicalStatus clinicalStatus = new ClinicalStatus(); + String clinicalId = status != null ? status.getString("id") : null; + if (status == null || StringUtils.isEmpty(clinicalId)) { + logger.warn("Status is empty or does not contain 'id' field. Setting default status value for {} '{}'", entity, id); + + } else { + for (ClinicalStatusValue clinicalStatusValue : statusValueList) { + if (clinicalStatusValue.getId().equals(clinicalId)) { + clinicalStatus.setId(clinicalStatusValue.getId()); + clinicalStatus.setDescription(clinicalStatusValue.getDescription()); + clinicalStatus.setType(clinicalStatusValue.getType()); + break; + } + } + if (clinicalStatus.getType() == null) { + logger.warn("Status '{}' not found in the list of available status values. Status type cannot be set for {} '{}'", + clinicalId, entity, id); + + } + } + clinicalStatus.setDate(TimeUtils.getTime()); + clinicalStatus.setVersion(GitRepositoryState.getInstance().getBuildVersion()); + clinicalStatus.setCommit(GitRepositoryState.getInstance().getCommitId()); + clinicalStatus.setAuthor("opencga"); + + Document document = convertToDocument(clinicalStatus); + return document; + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java similarity index 98% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java index 34cb362e25c..9bfafc457a5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/AddVersionToClinicalAnalysisMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0; +package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/UpdateClinicalStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java similarity index 98% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/UpdateClinicalStudyConfiguration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java index 7618586c2a1..915a9def616 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/UpdateClinicalStudyConfiguration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0; +package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index 5624bfb9058..5d12067aca6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -1339,7 +1339,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case INTERNAL_STATUS: case INTERNAL_STATUS_ID: // Convert the status to a positive status - queryCopy.put(queryParam.key(), InternalStatus.getPositiveStatus(ClinicalAnalysisStatusOld.STATUS_LIST, + queryCopy.put(queryParam.key(), InternalStatus.getPositiveStatus(InternalStatus.STATUS_LIST, queryCopy.getString(queryParam.key()))); addAutoOrQuery(INTERNAL_STATUS_ID.key(), queryParam.key(), queryCopy, INTERNAL_STATUS_ID.type(), andBsonList); break; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 34ba5b01821..13f29ae3602 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -51,10 +51,7 @@ import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.clinical.*; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.common.FlagAnnotation; -import org.opencb.opencga.core.models.common.FlagValue; +import org.opencb.opencga.core.models.common.*; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.family.FamilyCreateParams; import org.opencb.opencga.core.models.file.File; @@ -1658,8 +1655,8 @@ private OpenCGAResult update(String organizationId, Study stud if (parameters.containsKey(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key())) { Map status = (Map) parameters.get(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key()); - if (!(status instanceof Map) || StringUtils.isEmpty(String.valueOf(status.get("name"))) - || !ClinicalAnalysisStatusOld.isValid(String.valueOf(status.get("name")))) { + if (!(status instanceof Map) || StringUtils.isEmpty(String.valueOf(status.get("id"))) + || !InternalStatus.isValid(String.valueOf(status.get("id")))) { throw new CatalogException("Missing or invalid status"); } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index 71eb78c3112..8243077223b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -157,7 +157,7 @@ private DataResult createDummyFamily() throws CatalogException { private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new ClinicalStatus().setId(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId("READY_FOR_INTERPRETATION")) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); @@ -2700,35 +2700,35 @@ public void searchClinicalAnalysisByStatus() throws CatalogException { createDummyEnvironment(false, false); OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.DONE), + new Query(ParamConstants.STATUS_PARAM, "DONE"), new QueryOptions(QueryOptions.INCLUDE, ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_ID.key()), ownerToken); assertEquals(0, search.getNumResults()); search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION), + new Query(ParamConstants.STATUS_PARAM, "READY_FOR_INTERPRETATION"), new QueryOptions(), ownerToken); assertEquals(2, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals("READY_FOR_INTERPRETATION", result.getStatus().getId()); } catalogManager.getClinicalAnalysisManager().update(studyFqn, search.first().getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(ClinicalAnalysisStatusOld.REJECTED)), QueryOptions.empty(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("REJECTED")), QueryOptions.empty(), ownerToken); search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION), + new Query(ParamConstants.STATUS_PARAM, "READY_FOR_INTERPRETATION"), new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals("READY_FOR_INTERPRETATION", result.getStatus().getId()); } search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatusOld.REJECTED), + new Query(ParamConstants.STATUS_PARAM, "REJECTED"), new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatusOld.REJECTED, result.getStatus().getId()); + assertEquals("REJECTED", result.getStatus().getId()); } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java index 2ba297c49f3..99232c2e11d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java @@ -43,7 +43,7 @@ private DataResult createDummyFamily() throws CatalogException { private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new ClinicalStatus().setId(ClinicalAnalysisStatusOld.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId("READY_FOR_INTERPRETATION")) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java index 0cc243f78a3..425654bcc55 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java @@ -18,21 +18,19 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; +import org.opencb.opencga.core.models.common.InternalStatus; public class ClinicalAnalysisInternal extends Internal { - private ClinicalAnalysisStatusOld status; - public ClinicalAnalysisInternal() { } - public ClinicalAnalysisInternal(String registrationDate, String modificationDate, ClinicalAnalysisStatusOld status) { - super(null, registrationDate, modificationDate); - this.status = status; + public ClinicalAnalysisInternal(String registrationDate, String modificationDate, InternalStatus status) { + super(status, registrationDate, modificationDate); } public static ClinicalAnalysisInternal init() { - return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new ClinicalAnalysisStatusOld()); + return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new InternalStatus()); } @Override @@ -45,11 +43,11 @@ public String toString() { return sb.toString(); } - public ClinicalAnalysisStatusOld getStatus() { + public InternalStatus getStatus() { return status; } - public ClinicalAnalysisInternal setStatus(ClinicalAnalysisStatusOld status) { + public ClinicalAnalysisInternal setStatus(InternalStatus status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatusOld.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatusOld.java deleted file mode 100644 index 5e1ec106c57..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatusOld.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.core.models.clinical; - -import org.opencb.opencga.core.models.common.InternalStatus; - -import java.util.Arrays; -import java.util.List; - -public class ClinicalAnalysisStatusOld extends InternalStatus { - - public static final String INCOMPLETE = "INCOMPLETE"; - public static final String READY_FOR_VALIDATION = "READY_FOR_VALIDATION"; - public static final String READY_FOR_INTERPRETATION = "READY_FOR_INTERPRETATION"; - public static final String INTERPRETATION_IN_PROGRESS = "INTERPRETATION_IN_PROGRESS"; - // public static final String INTERPRETED = "INTERPRETED"; - public static final String READY_FOR_INTEPRETATION_REVIEW = "READY_FOR_INTEPRETATION_REVIEW"; - public static final String INTERPRETATION_REVIEW_IN_PROGRESS = "INTERPRETATION_REVIEW_IN_PROGRESS"; - public static final String READY_FOR_REPORT = "READY_FOR_REPORT"; - public static final String REPORT_IN_PROGRESS = "REPORT_IN_PROGRESS"; - public static final String DONE = "DONE"; - public static final String REVIEW_IN_PROGRESS = "REVIEW_IN_PROGRESS"; - public static final String CLOSED = "CLOSED"; - public static final String REJECTED = "REJECTED"; - - public static final List STATUS_LIST = Arrays.asList(INCOMPLETE, READY, DELETED, READY_FOR_VALIDATION, - READY_FOR_INTERPRETATION, INTERPRETATION_IN_PROGRESS, READY_FOR_INTEPRETATION_REVIEW, INTERPRETATION_REVIEW_IN_PROGRESS, - READY_FOR_REPORT, REPORT_IN_PROGRESS, DONE, REVIEW_IN_PROGRESS, CLOSED, REJECTED); - - public ClinicalAnalysisStatusOld(String status, String message) { - if (isValid(status)) { - init(status, message); - } else { - throw new IllegalArgumentException("Unknown status " + status); - } - } - - public ClinicalAnalysisStatusOld(String status) { - this(status, ""); - } - - public ClinicalAnalysisStatusOld() { - this(READY_FOR_INTERPRETATION, ""); - } - - public static boolean isValid(String status) { - if (InternalStatus.isValid(status)) { - return true; - } - - if (STATUS_LIST.contains(status)) { - return true; - } - return false; - } -} From 1ee2068ca65997a70461fa9e012294daf517a460 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 11 Jun 2024 14:38:52 +0200 Subject: [PATCH 045/128] app: fix migration script, #TASK-5964 --- .../UpdateClinicalStudyConfiguration.java | 28 +++++++++++++------ .../migration/MigrationRuntimeException.java | 13 +++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java index 915a9def616..5a7d2da963e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java @@ -8,6 +8,7 @@ import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationRuntimeException; import org.opencb.opencga.catalog.migration.MigrationTool; import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; @@ -26,18 +27,27 @@ protected void run() throws Exception { Document clinicalConfigurationDocument = convertToDocument(clinicalAnalysisStudyConfiguration); Bson query = new Document(); - Bson projection = Projections.include("configuration", "fqn"); + Bson projection = Projections.include("internal.configuration.clinical", "fqn"); for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { migrateCollection(collection, query, projection, (document, bulk) -> { - Document configuration = document.get("configuration", Document.class); - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + Document internal = document.get("internal", Document.class); + if (internal == null) { + throw new MigrationRuntimeException("'internal' field not found in study '" + document.get("fqn") + "'"); + } + Document configuration = internal.get("configuration", Document.class); if (configuration == null) { - logger.warn("Found empty study configuration in study '{}'. Creating a new one...", document.get("fqn")); - updateDocument.getSet().put("configuration", clinicalConfigurationDocument); + throw new MigrationRuntimeException("'internal.configuration' field not found in study '" + document.get("fqn") + "'"); + } + Document clinicalDocument = configuration.get("clinical", Document.class); + + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + if (clinicalDocument == null) { + logger.warn("Found empty 'internal.configuration.clinical' field in study '{}'. Creating a new one...", document.get("fqn")); + updateDocument.getSet().put("internal.configuration.clinical", clinicalConfigurationDocument); } else { - Object statusObject = configuration.get("status"); + Object statusObject = clinicalDocument.get("status"); if (statusObject instanceof List) { // The study seems to be already migrated. Skipping... logger.warn("Study '{}' seems to be already migrated. Skipping...", document.get("fqn")); @@ -45,9 +55,9 @@ protected void run() throws Exception { } // Study needs to be migrated logger.info("Migrating study '{}'", document.get("fqn")); - updateDocument.getSet().put("configuration.status", clinicalConfigurationDocument.get("status")); - updateDocument.getSet().put("configuration.flags", clinicalConfigurationDocument.get("flags")); - updateDocument.getSet().put("configuration.interpretation.status", clinicalConfigurationDocument.get("interpretation", Document.class).get("status")); + updateDocument.getSet().put("internal.configuration.clinical.status", clinicalConfigurationDocument.get("status")); + updateDocument.getSet().put("internal.configuration.clinical.flags", clinicalConfigurationDocument.get("flags")); + updateDocument.getSet().put("internal.configuration.clinical.interpretation.status", clinicalConfigurationDocument.get("interpretation", Document.class).get("status")); } logger.debug("Updating study '{}': {}", document.get("fqn"), updateDocument.toFinalUpdateDocument()); bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java new file mode 100644 index 00000000000..01d1d59d4a2 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.catalog.migration; + +public class MigrationRuntimeException extends IllegalArgumentException { + + public MigrationRuntimeException(String message) { + super(message); + } + + public MigrationRuntimeException(String message, Throwable cause) { + super(message, cause); + } + +} From 57b6b8d82102c15e986e04ee6e9ff3a08ec0cd37 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 11 Jun 2024 15:06:04 +0200 Subject: [PATCH 046/128] app: improve scenario where status.id is null, #TASK-5964 --- .../v3_2_0/TASK_5964/AddNewClinicalStatusValues.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java index e2942f3fcf9..10439dfb9f7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java @@ -77,7 +77,13 @@ private Document fillStatusValues(String entity, String id, Document status, Lis String clinicalId = status != null ? status.getString("id") : null; if (status == null || StringUtils.isEmpty(clinicalId)) { logger.warn("Status is empty or does not contain 'id' field. Setting default status value for {} '{}'", entity, id); - + for (ClinicalStatusValue clinicalStatusValue : statusValueList) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { + clinicalStatus.setId(clinicalStatusValue.getId()); + clinicalStatus.setDescription(clinicalStatusValue.getDescription()); + clinicalStatus.setType(clinicalStatusValue.getType()); + } + } } else { for (ClinicalStatusValue clinicalStatusValue : statusValueList) { if (clinicalStatusValue.getId().equals(clinicalId)) { @@ -90,7 +96,6 @@ private Document fillStatusValues(String entity, String id, Document status, Lis if (clinicalStatus.getType() == null) { logger.warn("Status '{}' not found in the list of available status values. Status type cannot be set for {} '{}'", clinicalId, entity, id); - } } clinicalStatus.setDate(TimeUtils.getTime()); From 4b41d6510bdf76708292833e69849c3d5aafa1d8 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 11 Jun 2024 15:22:28 +0200 Subject: [PATCH 047/128] catalog: fix status indexes, #TASK-5964 --- .../src/main/resources/catalog-indexes.txt | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 87f134a85c9..e196782a565 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -37,8 +37,8 @@ {"collections": ["job"], "fields": {"outDir.uid": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"visited": 1, "studyUid": 1}, "options": {}} -{"collections": ["job"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["job"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"_priority": 1, "_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"_priority": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} @@ -65,8 +65,8 @@ {"collections": ["file"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"jobId": 1, "studyUid": 1}, "options": {}} -{"collections": ["file"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["file"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"internal.variant.index.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} {"collections": ["file"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} @@ -104,8 +104,8 @@ {"collections": ["sample", "sample_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["sample", "sample_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["sample", "sample_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"internal.rga.status": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} @@ -138,8 +138,8 @@ {"collections": ["individual", "individual_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["individual", "individual_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["individual", "individual_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["cohort"], "fields": {"uuid": 1}, "options": {"unique": true}} {"collections": ["cohort"], "fields": {"uid": 1}, "options": {"unique": true}} @@ -158,8 +158,8 @@ {"collections": ["cohort"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["cohort"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["cohort"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["cohort"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["cohort"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["family", "family_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -182,8 +182,8 @@ {"collections": ["family", "family_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["family", "family_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["family", "family_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["panel", "panel_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -201,11 +201,11 @@ {"collections": ["panel", "panel_archive"], "fields": {"categories.name": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["panel", "panel_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} -{"collections": ["panel", "panel_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["clinical", "clinical_archive"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} {"collections": ["clinical", "clinical_archive"], "fields": {"uuid": 1}, "options": {"unique": true}} @@ -231,7 +231,7 @@ {"collections": ["clinical", "clinical_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["clinical", "clinical_archive"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} {"collections": ["clinical", "clinical_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical", "clinical_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -247,7 +247,7 @@ {"collections": ["interpretation", "interpretation_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "release": 1}, "options": {}} From f53cb74c68dd5058e6844c715930845d3e41df3a Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 12 Jun 2024 15:12:11 +0200 Subject: [PATCH 048/128] app: remove status name migration script, #TASK-5964 --- .../TASK_5964/RemoveStatusNameMigration.java | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/RemoveStatusNameMigration.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/RemoveStatusNameMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/RemoveStatusNameMigration.java new file mode 100644 index 00000000000..151c29c320e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/RemoveStatusNameMigration.java @@ -0,0 +1,114 @@ +package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Migration(id = "remove_status_name", description = "Remove status name #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240612) +public class RemoveStatusNameMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // Remove from user + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from project + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_PROJECT_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from study + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from sample + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", + "internal.variant.sampleGenotypeIndex.status.name", "internal.variant.sampleGenotypeIndex.familyStatus.name", + "internal.variant.secondarySampleIndex.status.name", "internal.variant.secondarySampleIndex.familyStatus.name", + "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from file + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_FILE_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", + "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name", + "internal.alignment.index.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from individual + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INDIVIDUAL_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from family + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_FAMILY_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from cohort + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_COHORT_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from panel + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, + OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_PANEL_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from job + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from clinical + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from interpretation + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + } + + private void removeStatusField(String collection, List statusList) throws CatalogDBException { + logger.info("Removing status.name fields from collection {}...", collection); + List exists = statusList.stream().map(Filters::exists).collect(Collectors.toList()); + List unsetList = statusList.stream().map(Updates::unset).collect(Collectors.toList()); + getMongoCollection(collection).updateMany(Filters.or(exists), Updates.combine(unsetList)); + } + +} From b6e6e868e4a59d5df2ef53f86555bf60b39c8349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 20 Jun 2024 13:03:41 +0100 Subject: [PATCH 049/128] storage: Fix compoundHet query in single-study projects. #TASK-6311 --- .../VariantStorageMetadataRepairTool.java | 1 - .../VariantStorageMetadataManager.java | 13 +++ .../core/variant/VariantStorageEngine.java | 2 +- .../variant/query/ParsedVariantQuery.java | 21 +++-- .../variant/query/VariantQueryParser.java | 80 ++++++++++++------- .../BreakendVariantQueryExecutor.java | 4 +- .../CompoundHeterozygousQueryExecutor.java | 6 +- .../DBAdaptorVariantQueryExecutor.java | 4 +- .../executors/NoOpVariantQueryExecutor.java | 4 +- .../query/executors/VariantQueryExecutor.java | 5 +- ...amplesSearchIndexVariantQueryExecutor.java | 4 +- .../SearchIndexVariantQueryExecutor.java | 8 +- .../variant/search/VariantSearchUtils.java | 4 +- .../executors/VariantQueryExecutorTest.java | 57 ++++++++++--- .../variant/HadoopVariantQueryParser.java | 5 +- ...seColumnIntersectVariantQueryExecutor.java | 5 +- ...ampleIndexMendelianErrorQueryExecutor.java | 9 +-- .../SampleIndexOnlyVariantQueryExecutor.java | 8 +- .../SampleIndexVariantQueryExecutor.java | 4 +- .../index/family/FamilyIndexDriver.java | 5 +- .../sample/SampleIndexSchemaFactory.java | 3 +- .../variant/index/sample/SampleIndexTest.java | 12 ++- 22 files changed, 177 insertions(+), 87 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java index a8e9ba5b528..29ed6efb996 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java @@ -134,7 +134,6 @@ private void rebuildSampleFileIds(VariantStorageMetadataManager metadataManager, for (Map.Entry> entry : batch.entrySet()) { Integer sampleId = entry.getKey(); List fileIds = entry.getValue(); - List actualFiles = metadataManager.getSampleMetadata(studyId, sampleId).getFiles(); if (actualFiles.size() != fileIds.size() || !actualFiles.containsAll(fileIds)) { fixedSamples++; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index a27f908173b..6c539e4b077 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -840,6 +840,19 @@ public Iterator fileMetadataIterator(int studyId) { return fileDBAdaptor.fileIterator(studyId); } + public SampleMetadata getSampleMetadata(Integer studyId, Integer sampleId) { + return getSampleMetadata(studyId.intValue(), sampleId.intValue()); + } + + public SampleMetadata getSampleMetadata(int studyId, Integer sampleId) { + return getSampleMetadata(studyId, sampleId.intValue()); + } + + public SampleMetadata getSampleMetadata(int studyId, Object sample) { + int sampleId = getSampleIdOrFail(studyId, sample); + return getSampleMetadata(studyId, sampleId); + } + public SampleMetadata getSampleMetadata(int studyId, int sampleId) { return sampleDBAdaptor.getSampleMetadata(studyId, sampleId, null); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index dd24ee1334d..0f51fbe39eb 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -1348,7 +1348,7 @@ public VariantQueryExecutor getVariantQueryExecutor(Query query, QueryOptions op public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQuery) { try { for (VariantQueryExecutor executor : getVariantQueryExecutors()) { - if (executor.canUseThisExecutor(variantQuery.getQuery(), variantQuery.getInputOptions())) { + if (executor.canUseThisExecutor(variantQuery, variantQuery.getInputOptions())) { logger.info("Using VariantQueryExecutor : " + executor.getClass().getName()); logger.info(" Query : " + VariantQueryUtils.printQuery(variantQuery.getInputQuery())); logger.info(" Options : " + variantQuery.getInputOptions().toJson()); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java index 8468ab34317..300f2e6b48a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java @@ -267,14 +267,6 @@ public VariantStudyQuery setStudies(ParsedQuery studies) { return this; } - public String getStudyOrFail() { - if (studies == null || studies.size() != 1) { - throw new VariantQueryException("Require exactly one study"); - } else { - return studies.get(0); - } - } - public ParsedQuery>> getGenotypes() { return genotypes; } @@ -311,6 +303,19 @@ public void setDefaultStudy(StudyMetadata defaultStudy) { public StudyMetadata getDefaultStudy() { return defaultStudy; } + + public StudyMetadata getDefaultStudyOrFail() { + if (defaultStudy == null) { + if (studies.size() != 1) { + throw new VariantQueryException("Only one study is allowed. Found " + studies.size() + " studies"); + } else { + throw new VariantQueryException("One study required. None provided"); + } + } else { + return defaultStudy; + } + } + } public static class VariantQueryXref { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java index 641e365a51d..bd02a6b4303 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java @@ -17,10 +17,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.core.models.variant.VariantAnnotationConstants; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; -import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -170,15 +167,14 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.defaultValue())); - variantQuery.setProjection(projectionParser.parseVariantQueryProjection(inputQuery, options)); - VariantQuery query; if (!skipPreProcess) { - query = new VariantQuery(preProcessQuery(inputQuery, options, variantQuery.getProjection())); + query = new VariantQuery(preProcessQuery(inputQuery, options)); } else { query = new VariantQuery(inputQuery); } variantQuery.setQuery(query); + variantQuery.setProjection(projectionParser.parseVariantQueryProjection(query, options)); List geneRegions = Region.parseRegions(query.getString(ANNOT_GENE_REGIONS.key())); variantQuery.setGeneRegions(geneRegions == null ? Collections.emptyList() : geneRegions); @@ -221,10 +217,7 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo = new ParsedQuery<>(sampleDataQuery.getKey(), sampleDataQuery.getOperation(), new ArrayList<>(sampleDataQuery.size())); for (KeyValues> keyValues : sampleDataQuery) { sampleDataQueryWithMetadata.getValues().add( - keyValues.mapKey(sample -> { - int sampleId = metadataManager.getSampleIdOrFail(defaultStudy.getId(), sample); - return metadataManager.getSampleMetadata(defaultStudy.getId(), sampleId); - })); + keyValues.mapKey(sample -> metadataManager.getSampleMetadata(defaultStudy.getId(), sample))); } studyQuery.setSampleDataQuery(sampleDataQueryWithMetadata); } @@ -232,17 +225,13 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo return variantQuery; } - public final Query preProcessQuery(Query originalQuery, QueryOptions options) { - return preProcessQuery(originalQuery, options, null); - } - - protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { + public Query preProcessQuery(Query originalQuery, QueryOptions options) { // Copy input query! Do not modify original query! Query query = VariantQueryUtils.copy(originalQuery); preProcessAnnotationParams(query); - preProcessStudyParams(query, options, projection); + preProcessStudyParams(query, options); if (options != null && options.getLong(QueryOptions.LIMIT) < 0) { throw VariantQueryException.malformedParam(QueryOptions.LIMIT, options.getString(QueryOptions.LIMIT), @@ -388,7 +377,7 @@ private VariantType parseVariantType(String type) { } } - protected void preProcessStudyParams(Query query, QueryOptions options, VariantQueryProjection projection) { + protected void preProcessStudyParams(Query query, QueryOptions options) { StudyMetadata defaultStudy = getDefaultStudy(query); QueryOperation formatOperator = null; if (isValidParam(query, SAMPLE_DATA)) { @@ -613,10 +602,13 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ if (isValidParam(query, SAMPLE_MENDELIAN_ERROR) || isValidParam(query, SAMPLE_DE_NOVO) - || isValidParam(query, SAMPLE_DE_NOVO_STRICT)) { + || isValidParam(query, SAMPLE_DE_NOVO_STRICT) + || isValidParam(query, SAMPLE_COMPOUND_HETEROZYGOUS)) { + boolean requireMendelianReady = false; QueryParam param = null; if (isValidParam(query, SAMPLE_MENDELIAN_ERROR)) { param = SAMPLE_MENDELIAN_ERROR; + requireMendelianReady = true; } if (isValidParam(query, SAMPLE_DE_NOVO)) { if (param != null) { @@ -624,6 +616,7 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ param, query.getString(param.key()), SAMPLE_DE_NOVO, query.getString(SAMPLE_DE_NOVO.key())); } + requireMendelianReady = true; param = SAMPLE_DE_NOVO; } if (isValidParam(query, SAMPLE_DE_NOVO_STRICT)) { @@ -632,8 +625,21 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ param, query.getString(param.key()), SAMPLE_DE_NOVO_STRICT, query.getString(SAMPLE_DE_NOVO_STRICT.key())); } + requireMendelianReady = true; param = SAMPLE_DE_NOVO_STRICT; } + if (isValidParam(query, SAMPLE_COMPOUND_HETEROZYGOUS)) { + if (param != null) { + throw VariantQueryException.unsupportedParamsCombination( + param, query.getString(param.key()), + SAMPLE_COMPOUND_HETEROZYGOUS, query.getString(SAMPLE_COMPOUND_HETEROZYGOUS.key())); + } + requireMendelianReady = false; + param = SAMPLE_COMPOUND_HETEROZYGOUS; + } + if (param == null) { + throw new IllegalStateException("Unknown param"); + } if (defaultStudy == null) { throw VariantQueryException.missingStudyForSamples(query.getAsStringList(param.key()), metadataManager.getStudyNames()); @@ -645,15 +651,18 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ genotypeParam, query.getString(genotypeParam.key()) ); } - List samples = query.getAsStringList(param.key()); + Object value = query.get(param.key()); + List samples; + if (value instanceof Trio) { + samples = Collections.singletonList(((Trio) value).getChild()); + } else { + samples = query.getAsStringList(param.key()); + } Set samplesAndParents = new LinkedHashSet<>(samples); for (String sample : samples) { - Integer sampleId = metadataManager.getSampleId(defaultStudy.getId(), sample); - if (sampleId == null) { - throw VariantQueryException.sampleNotFound(sample, defaultStudy.getName()); - } - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(defaultStudy.getId(), sampleId); - if (TaskMetadata.Status.READY != sampleMetadata.getMendelianErrorStatus()) { + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(defaultStudy.getId(), sample); + if (requireMendelianReady + && TaskMetadata.Status.READY != sampleMetadata.getMendelianErrorStatus()) { throw VariantQueryException.malformedParam(param, "Sample \"" + sampleMetadata.getName() + "\" does not have the Mendelian Errors precomputed yet"); } @@ -674,6 +683,21 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ } else { query.put(INCLUDE_SAMPLE.key(), new ArrayList<>(samplesAndParents)); } + if (param == SAMPLE_COMPOUND_HETEROZYGOUS) { + int studyId = defaultStudy.getId(); + if (!(value instanceof Trio)) { + if (samples.size() > 1) { + throw VariantQueryException.malformedParam(SAMPLE, value.toString(), + "More than one sample provided for compound heterozygous filter."); + } + SampleMetadata sm = metadataManager.getSampleMetadata(studyId, samples.get(0)); + Trio trio = new Trio(null, + metadataManager.getSampleName(studyId, sm.getFather()), + metadataManager.getSampleName(studyId, sm.getMother()), + sm.getName()); + query.put(SAMPLE_COMPOUND_HETEROZYGOUS.key(), trio); + } + } } if (isValidParam(query, SCORE)) { @@ -704,9 +728,7 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ || isValidParam(query, SAMPLE_SKIP) || isValidParam(query, SAMPLE_LIMIT) ) { - if (projection == null) { - projection = projectionParser.parseVariantQueryProjection(query, options); - } + VariantQueryProjection projection = projectionParser.parseVariantQueryProjection(query, options); // Apply the sample pagination. // Remove the sampleLimit and sampleSkip to avoid applying the pagination twice query.remove(SAMPLE_SKIP.key()); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java index 6eb237ea4b3..bc40c5b6418 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java @@ -14,6 +14,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; @@ -40,7 +41,8 @@ public BreakendVariantQueryExecutor(String storageEngineId, ObjectMap options, } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); return query.getString(VariantQueryParam.TYPE.key()).equals(VariantType.BREAKEND.name()) && VariantQueryUtils.isValidParam(query, VariantQueryParam.GENOTYPE); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java index c6f4b87a5f0..bcecac57b5a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java @@ -64,14 +64,14 @@ public CompoundHeterozygousQueryExecutor(VariantStorageMetadataManager metadataM } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { - return isValidParam(query, VariantQueryUtils.SAMPLE_COMPOUND_HETEROZYGOUS); + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + return isValidParam(variantQuery.getQuery(), VariantQueryUtils.SAMPLE_COMPOUND_HETEROZYGOUS); } @Override protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { Trio trio = getCompHetTrio(variantQuery.getQuery()); - String study = variantQuery.getStudyQuery().getStudyOrFail(); + String study = variantQuery.getStudyQuery().getDefaultStudyOrFail().getName(); return getOrIterator(study, trio.getChild(), trio.getFather(), trio.getMother(), variantQuery, iterator); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java index 474cbc3fa9f..e667e2a1485 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java @@ -52,9 +52,9 @@ protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { for (QueryParam unsupportedParam : UNSUPPORTED_PARAMS) { - if (VariantQueryUtils.isValidParam(query, unsupportedParam)) { + if (VariantQueryUtils.isValidParam(variantQuery.getQuery(), unsupportedParam)) { logger.warn("Unsupported variant query param {} in {}", unsupportedParam.key(), DBAdaptorVariantQueryExecutor.class.getSimpleName()); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java index e286b4a07ce..5f2da2c2056 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java @@ -8,6 +8,7 @@ import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.query.*; @@ -34,7 +35,8 @@ public NoOpVariantQueryExecutor(VariantStorageMetadataManager metadataManager, S } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); boolean sampleQuery = false; String sample = null; if (VariantQueryUtils.isValidParam(query, VariantQueryParam.GENOTYPE)) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java index 26d53e89e5c..49ddc339658 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java @@ -2,7 +2,6 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -65,12 +64,12 @@ public static void setDefaultTimeout(QueryOptions queryOptions, ObjectMap config /** * Determine if this VariantQueryExecutor can run the given query. - * @param query Query to execute + * @param variantQuery Query to execute * @param options Options for the query * @return True if this variant query executor is valid for the query * @throws StorageEngineException if there is an error */ - public abstract boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException; + public abstract boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException; protected abstract Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) throws StorageEngineException; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java index 200b2eb463d..cfa794cbe10 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java @@ -7,6 +7,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; @@ -28,7 +29,8 @@ public SamplesSearchIndexVariantQueryExecutor(VariantDBAdaptor dbAdaptor, Varian } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); String samplesCollection = inferSpecificSearchIndexSamplesCollection(query, options, getMetadataManager(), dbName); return samplesCollection != null && searchActiveAndAlive(samplesCollection); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index 12c86cc4e2b..bd6b8e6437f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -11,10 +11,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; -import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; -import org.opencb.opencga.storage.core.variant.adaptors.VariantField; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.adaptors.*; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; @@ -76,7 +73,8 @@ public SearchIndexVariantQueryExecutor setIntersectParamsThreshold(int intersect } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); return doQuerySearchManager(query, options) || doIntersectWithSearch(query, options); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java index e46c6379dc6..5365643b1fa 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java @@ -280,7 +280,9 @@ public static String inferSpecificSearchIndexSamplesCollection( Set samples = new HashSet<>(); if (isValidParam(query, VariantQueryParam.SAMPLE)) { String value = query.getString(VariantQueryParam.SAMPLE.key()); - samples.addAll(splitValue(value).getValue()); + for (String sample : splitValue(value).getValue()) { + samples.add(sample.split(IS)[0]); + } } if (isValidParam(query, VariantQueryParam.GENOTYPE)) { HashMap> map = new HashMap<>(); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java index 778257d5999..3d0b2524681 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java @@ -12,6 +12,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -72,6 +73,9 @@ public void setUp() throws Exception { fileIndexed = true; Integer indexedFileId = metadataManager.getIndexedFiles(studyMetadata.getId()).iterator().next(); + Trio trio = new Trio("NA19660", "NA19661", "NA19685"); + variantStorageEngine.familyIndex(studyMetadata.getName(), Collections.singletonList(trio), new ObjectMap()); + //Calculate stats QueryOptions options = new QueryOptions(VariantStorageOptions.STUDY.key(), STUDY_NAME) .append(VariantStorageOptions.LOAD_BATCH_SIZE.key(), 100) @@ -188,23 +192,53 @@ public void testXRefRs() throws StorageEngineException { } } + @Test + public void testCompHetQuery() throws StorageEngineException { +// Matcher matcher = allOf( +// anyOf( +// samePosition(new Variant("1:2441358:T:C")), +// samePosition(new Variant("1:2458010:G:C")), +// samePosition(new Variant("19:501725:G:A")), +// samePosition(new Variant("19:501900:C:A"))), +// withStudy(STUDY_NAME, withSampleGt("NA19685"))); + Matcher matcher = null; + testQuery(new VariantQuery().sample("NA19685:compoundheterozygous") + .study(STUDY_NAME) + .biotype("protein_coding"), + new QueryOptions(), + matcher, + false); + } + public VariantQueryResult testQuery(Query query, QueryOptions options, Matcher matcher) throws StorageEngineException { + return testQuery(query, options, matcher, true); + } + + public VariantQueryResult testQuery(Query query, QueryOptions options, Matcher matcher, boolean expectDBAdaptorExecutor) + throws StorageEngineException { logger.info(""); logger.info(""); logger.info("####################################################"); logger.info("########## TEST QUERY :" + query.toJson()); logger.info("####################################################"); logger.info("## Allowed VariantQueryExecutors:"); + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { - if (variantQueryExecutor.canUseThisExecutor(query, options)) { + if (variantQueryExecutor.canUseThisExecutor(variantQuery, options)) { logger.info("## - " + variantQueryExecutor.getClass().getSimpleName()); } } - logger.info("## Using DBAdaptorVariantQueryExecutor for expected results"); - Assert.assertTrue(dbQueryExecutor.canUseThisExecutor(query, options)); + VariantQueryResult expected; + if (expectDBAdaptorExecutor) { + logger.info("## Using DBAdaptorVariantQueryExecutor for expected results"); + Assert.assertTrue(dbQueryExecutor.canUseThisExecutor(variantQuery, options)); - ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); - VariantQueryResult expected = dbQueryExecutor.get(variantQuery); + expected = dbQueryExecutor.get(variantQuery); + } else { + logger.info("## DBAdaptorVariantQueryExecutor can not be used for expected results"); + Assert.assertFalse(dbQueryExecutor.canUseThisExecutor(variantQuery, options)); + expected = null; + } VariantQueryResult unfilteredResult = null; VariantQueryResult result = null; @@ -243,21 +277,24 @@ public VariantQueryResult testQuery(Query query, QueryOptions options, QueryOptions emptyOptions = new QueryOptions(); emptyOptions.putIfNotEmpty(QueryOptions.INCLUDE, options.getString(QueryOptions.INCLUDE)); emptyOptions.putIfNotEmpty(QueryOptions.EXCLUDE, options.getString(QueryOptions.EXCLUDE)); + logger.info("## unfiltered query " + VariantQueryUtils.printQuery(emptyQuery)); + logger.info("## unfiltered options " + emptyOptions.toJson()); unfilteredResult = dbQueryExecutor.get(variantStorageEngine.parseQuery(emptyQuery, emptyOptions)); } for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { - if (variantQueryExecutor.canUseThisExecutor(query, options)) { + if (variantQueryExecutor.canUseThisExecutor(variantQuery, options)) { logger.info(""); logger.info("###################"); logger.info("### Testing " + variantQueryExecutor.getClass().getSimpleName()); result = variantQueryExecutor.get(variantQuery); logger.info("### Num results : " + result.getNumResults()); logger.info("###################"); - expected.getResults().sort(Comparator.comparing(Variant::toString)); - result.getResults().sort(Comparator.comparing(Variant::toString)); - Assert.assertEquals(expected.getResults(), result.getResults()); - + if (expected != null) { + expected.getResults().sort(Comparator.comparing(Variant::toString)); + result.getResults().sort(Comparator.comparing(Variant::toString)); + Assert.assertEquals(expected.getResults(), result.getResults()); + } assertThat(result, numResults(gt(0))); if (matcher != null) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java index 6aa472ee735..45682df31f4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java @@ -5,7 +5,6 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; -import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; import java.util.List; @@ -18,8 +17,8 @@ public HadoopVariantQueryParser(CellBaseUtils cellBaseUtils, VariantStorageMetad } @Override - protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { - Query query = super.preProcessQuery(originalQuery, options, projection); + public Query preProcessQuery(Query originalQuery, QueryOptions options) { + Query query = super.preProcessQuery(originalQuery, options); List studyNames = metadataManager.getStudyNames(); if (isValidParam(query, STUDY) && studyNames.size() == 1) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java index c672e94fb58..ceb52e5eeaa 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java @@ -7,6 +7,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.query.ParsedQuery; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; @@ -43,8 +44,8 @@ public HBaseColumnIntersectVariantQueryExecutor(VariantDBAdaptor dbAdaptor, Stri } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { - + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { + VariantQuery query = variantQuery.getQuery(); if (!options.getBoolean(HBASE_COLUMN_INTERSECT, ACTIVE_BY_DEFAULT)) { // HBase column intersect not active return false; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java index 4dd50c9eacf..5063ca1fe4b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java @@ -9,7 +9,6 @@ import org.opencb.biodata.tools.pedigree.MendelianError; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.Trio; @@ -33,10 +32,10 @@ public SampleIndexMendelianErrorQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { - if (VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_MENDELIAN_ERROR) - || VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_DE_NOVO) - || VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_DE_NOVO_STRICT)) { + public boolean canUseThisExecutor(ParsedVariantQuery query, QueryOptions options) { + if (VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_MENDELIAN_ERROR) + || VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_DE_NOVO) + || VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_DE_NOVO_STRICT)) { return super.canUseThisExecutor(query, options); } else { return false; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java index 371d57ac4da..3e9e0b9b75f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java @@ -86,7 +86,8 @@ public SampleIndexOnlyVariantQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, Sam } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { + VariantQuery query = variantQuery.getQuery(); if (SampleIndexQueryParser.validSampleIndexQuery(query)) { if (isFullyCoveredQuery(query, options)) { @@ -179,10 +180,13 @@ protected boolean shouldGetCount(QueryOptions options, boolean iterator) { private boolean isFullyCoveredQuery(Query inputQuery, QueryOptions options) { Query query = new Query(inputQuery); + if (!isQueryCovered(query)) { + return false; + } // ParsedVariantQuery parsedVariantQuery = variantQueryProjectionParser.parseQuery(query, options, true); SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); - return isQueryCovered(query) && isIncludeCovered(sampleIndexQuery, inputQuery, options); + return isIncludeCovered(sampleIndexQuery, inputQuery, options); } private boolean isQueryCovered(Query query) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java index 6175128018d..03e8a5fe880 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java @@ -58,9 +58,9 @@ public SampleIndexVariantQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, SampleI } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery query, QueryOptions options) { if (options.getBoolean(SAMPLE_INDEX_INTERSECT, true)) { - return SampleIndexQueryParser.validSampleIndexQuery(query); + return SampleIndexQueryParser.validSampleIndexQuery(query.getQuery()); } return false; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java index af632d69436..ad09896e627 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java @@ -112,10 +112,9 @@ protected void parseAndValidateParameters() throws IOException { } else { trioList.add(metadataManager.getSampleIdOrFail(getStudyId(), trio.getMother())); } - int childId = metadataManager.getSampleIdOrFail(getStudyId(), trio.getChild()); - trioList.add(childId); - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), childId); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), trio.getChild()); + trioList.add(sampleMetadata.getId()); if (!overwrite && sampleMetadata.getFamilyIndexStatus(sampleIndexVersion) == TaskMetadata.Status.READY) { LOGGER.info("Skip sample " + sampleMetadata.getName() + ". Already precomputed!"); } else { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java index ef2aecb803b..a087420f425 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java @@ -78,8 +78,7 @@ public Collection getSampleIndexConfigurationVersions(int studyId, Coll private Collection getSampleIndexConfigurationVersions(int studyId, Object sample, boolean withAnnotation, boolean withFamilyIndex) { - int sampleId = metadataManager.getSampleIdOrFail(studyId, sample); - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sample); Collection versions = sampleMetadata.getSampleIndexVersions(); if (withAnnotation) { versions = CollectionUtils.intersection( diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index 9ee362f8724..144083e5bd0 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -1068,7 +1068,11 @@ public void testFamilyIndexQueryCount() { .append(QueryOptions.LIMIT, 10) .append(QueryOptions.COUNT, true)); - System.out.println(result.getResults().stream().map(Variant::getAnnotation).flatMap(v -> v.getConsequenceTypes().stream()).map(ConsequenceType::getGeneName).collect(Collectors.toSet())); + System.out.println(result.getResults().stream() + .map(Variant::getAnnotation) + .flatMap(v -> v.getConsequenceTypes().stream()) + .map(ConsequenceType::getGeneName) + .collect(Collectors.toSet())); result = variantStorageEngine.get( new Query() @@ -1082,7 +1086,11 @@ public void testFamilyIndexQueryCount() { .append(QueryOptions.LIMIT, 10) .append(QueryOptions.COUNT, true)); - System.out.println(result.getResults().stream().map(Variant::getAnnotation).flatMap(v -> v.getConsequenceTypes().stream()).map(ConsequenceType::getGeneName).collect(Collectors.toSet())); + System.out.println(result.getResults().stream() + .map(Variant::getAnnotation) + .flatMap(v -> v.getConsequenceTypes().stream()) + .map(ConsequenceType::getGeneName) + .collect(Collectors.toSet())); } @Test From b4319d08b55388f85525715f0e29d2cdadc2b963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Wed, 26 Jun 2024 17:32:36 +0200 Subject: [PATCH 050/128] analysis: add config files for Exomiser versions 13.1 and 14.0 using different default values, #TASK-6297, #TASK-6255 On branch TASK-6255 Changes to be committed: modified: opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java modified: opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java renamed: opencga-app/app/analysis/exomiser/application.properties -> opencga-app/app/analysis/exomiser/13.1/application.properties renamed: opencga-app/app/analysis/exomiser/exomiser-analysis.yml -> opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml renamed: opencga-app/app/analysis/exomiser/output.yml -> opencga-app/app/analysis/exomiser/13.1/output.yml new file: opencga-app/app/analysis/exomiser/14.0/application.properties new file: opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml new file: opencga-app/app/analysis/exomiser/14.0/output.yml --- .../ExomiserWrapperAnalysisExecutor.java | 30 ++----- .../variant/OpenCGATestExternalResource.java | 11 ++- .../{ => 13.1}/application.properties | 6 +- .../exomiser/{ => 13.1}/exomiser-analysis.yml | 0 .../analysis/exomiser/{ => 13.1}/output.yml | 0 .../exomiser/14.0/application.properties | 47 ++++++++++ .../exomiser/14.0/exomiser-analysis.yml | 85 +++++++++++++++++++ .../app/analysis/exomiser/14.0/output.yml | 11 +++ 8 files changed, 158 insertions(+), 32 deletions(-) rename opencga-app/app/analysis/exomiser/{ => 13.1}/application.properties (92%) rename opencga-app/app/analysis/exomiser/{ => 13.1}/exomiser-analysis.yml (100%) rename opencga-app/app/analysis/exomiser/{ => 13.1}/output.yml (100%) create mode 100644 opencga-app/app/analysis/exomiser/14.0/application.properties create mode 100644 opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml create mode 100644 opencga-app/app/analysis/exomiser/14.0/output.yml diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 7887106f868..b5bf468330c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -49,11 +49,6 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private final static String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; private static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; - // These constants must match in the file application.properties to be replaced - private static final String HG38_DATA_VERSION_MARK = "put_here_hg38_data_version"; - private static final String PHENOTYPE_DATA_VERSION_MARK = "put_here_phenotype_data_version"; - private static final String CLINVAR_WHITELIST_MARK = "put_here_clinvar_whitelist"; - private String studyId; private String sampleId; private String exomiserVersion; @@ -166,8 +161,8 @@ public void run() throws ToolException, IOException, CatalogException { // Copy the analysis try { - copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME) + .toFile(), getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser analysis file", e); } @@ -175,30 +170,15 @@ public void run() throws ToolException, IOException, CatalogException { // Copy the application.properties and update data according to Exomiser version try { Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); - copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), target.toFile()); - // Update hg38 data version - Command cmd = new Command("sed -i \"s/" + HG38_DATA_VERSION_MARK + "/" + getHg38DataVersion() + "/g\" " + target); - cmd.run(); - // Update phenotype data version - cmd = new Command("sed -i \"s/" + PHENOTYPE_DATA_VERSION_MARK + "/" + getPhenotypeDataVersion() + "/g\" " + target); - cmd.run(); - // Update clinvar whitelist - String whitelist; - String clinvarWhitelistFilename = getHg38DataVersion() + "_hg38_clinvar_whitelist.tsv.gz"; - if (Files.exists(exomiserDataPath.resolve(getHg38DataVersion() + "_" + assembly).resolve(clinvarWhitelistFilename))) { - whitelist = "exomiser.hg38.variant-white-list-path=" + clinvarWhitelistFilename; - } else { - whitelist = "#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz"; - } - cmd = new Command("sed -i \"s/" + CLINVAR_WHITELIST_MARK + "/" + whitelist + "/g\" " + target); - cmd.run(); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME) + .toFile(), target.toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser properties file", e); } // Copy the output options try { - copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser output options file", e); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 1f05dfc5dd3..02962b3e3c4 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -263,11 +263,14 @@ public Path isolateOpenCGA() throws IOException { Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); // Exomiser analysis files - analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser")).toAbsolutePath(); + List exomiserVersions = Arrays.asList("13.1", "14.0"); List exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml"); - for (String exomiserFile : exomiserFiles) { - inputStream = new FileInputStream("../opencga-app/app/analysis/exomiser/" + exomiserFile); - Files.copy(inputStream, analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + for (String exomiserVersion : exomiserVersions) { + analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).toAbsolutePath()); + for (String exomiserFile : exomiserFiles) { + inputStream = new FileInputStream("../opencga-app/app/analysis/exomiser/" + exomiserVersion + "/" + exomiserFile); + Files.copy(inputStream, analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + } } return opencgaHome; diff --git a/opencga-app/app/analysis/exomiser/application.properties b/opencga-app/app/analysis/exomiser/13.1/application.properties similarity index 92% rename from opencga-app/app/analysis/exomiser/application.properties rename to opencga-app/app/analysis/exomiser/13.1/application.properties index ee9a2b7d7ed..1775e32d14c 100644 --- a/opencga-app/app/analysis/exomiser/application.properties +++ b/opencga-app/app/analysis/exomiser/13.1/application.properties @@ -37,9 +37,9 @@ exomiser.data-directory=/data #remm.version=0.3.1.post1 #cadd.version=1.4 #exomiser.hg19.data-version=1811 -exomiser.hg38.data-version=put_here_hg38_data_version +exomiser.hg38.data-version=2109 #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -put_here_clinvar_whitelist -exomiser.phenotype.data-version=put_here_phenotype_data_version +exomiser.hg38.variant-white-list-path=2109_hg38_clinvar_whitelist.tsv.gz +exomiser.phenotype.data-version=2109 logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/exomiser-analysis.yml rename to opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml diff --git a/opencga-app/app/analysis/exomiser/output.yml b/opencga-app/app/analysis/exomiser/13.1/output.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/output.yml rename to opencga-app/app/analysis/exomiser/13.1/output.yml diff --git a/opencga-app/app/analysis/exomiser/14.0/application.properties b/opencga-app/app/analysis/exomiser/14.0/application.properties new file mode 100644 index 00000000000..d1658370370 --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/application.properties @@ -0,0 +1,47 @@ +# +# The Exomiser - A tool to annotate and prioritize genomic variants +# +# Copyright (c) 2016-2021 Queen Mary University of London. +# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +## exomiser root data directory ## +# root path where data is to be downloaded and worked on it is assumed that all the files required by exomiser listed +# in this properties file will be found in the data directory, unless specifically overridden here. +#exomiser.data-directory=/exomiser-data +exomiser.data-directory=/data + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg38.data-version=1811 +#exomiser.hg38.data-version=2109 +#exomiser.hg38.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg38.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +#exomiser.phenotype.data-version=2109 +#logging.file.name=/exomiser-data/logs/exomiser.log + + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg19.data-version=1811 +exomiser.hg38.data-version=2402 +#exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz +#exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.phenotype.data-version=2402 +exomiser.hg38.clin-var-data-version=2402 +exomiser.hg38.use-clinvar-white-list=true +logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml new file mode 100644 index 00000000000..e3d064ba07b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml @@ -0,0 +1,85 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + X_DOMINANT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + X_RECESSIVE_HOM_ALT: 0.1, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + UK10K, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + # GNOMAD_E_FIN, + GNOMAD_E_NFE, + # GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + # GNOMAD_G_FIN, + GNOMAD_G_NFE, + # GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +#all steps are optional +steps: [ + #hiPhivePrioritiser: {}, + #priorityScoreFilter: {priorityType: HIPHIVE_PRIORITY, minPriorityScore: 0.500}, + #intervalFilter: {interval: 'chr10:123256200-123256300'}, + # or for multiple intervals: + #intervalFilter: {intervals: ['chr10:123256200-123256300', 'chr10:123256290-123256350']}, + # or using a BED file - NOTE this should be 0-based, Exomiser otherwise uses 1-based coordinates in line with VCF + #intervalFilter: {bed: /full/path/to/bed_file.bed}, + #genePanelFilter: {geneSymbols: ['FGFR1','FGFR2']}, + #geneBlacklistFilter: { }, + failedVariantFilter: { }, + #qualityFilter: {minQuality: 50.0}, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + UPSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + DOWNSTREAM_GENE_VARIANT + ] + }, + # removes variants represented in the database + #knownVariantFilter: {}, + frequencyFilter: {maxFrequency: 2.0}, + pathogenicityFilter: {keepNonPathogenic: true}, + # inheritanceFilter and omimPrioritiser should always run AFTER all other filters have completed + # they will analyse genes according to the specified modeOfInheritance above- UNDEFINED will not be analysed. + inheritanceFilter: {}, + # omimPrioritiser isn't mandatory. + omimPrioritiser: {}, + #priorityScoreFilter: {minPriorityScore: 0.4}, + # Other prioritisers: Only combine omimPrioritiser with one of these. + # Don't include any if you only want to filter the variants. + hiPhivePrioritiser: {}, + # or run hiPhive in benchmarking mode: + #hiPhivePrioritiser: {runParams: 'mouse'}, + #phivePrioritiser: {} + #phenixPrioritiser: {} + #exomeWalkerPrioritiser: {seedGeneIds: [11111, 22222, 33333]} +] diff --git a/opencga-app/app/analysis/exomiser/14.0/output.yml b/opencga-app/app/analysis/exomiser/14.0/output.yml new file mode 100644 index 00000000000..4fc4a4faa6b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/output.yml @@ -0,0 +1,11 @@ +--- +outputContributingVariantsOnly: true +#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results +numGenes: 20 +minExomiserGeneScore: 0.7 +# Path to the desired output directory. Will default to the 'results' subdirectory of the exomiser install directory +outputDirectory: /jobdir/ +# Filename for the output files. Will default to {input-vcf-filename}-exomiser +outputFileName: exomiser_output +#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) +outputFormats: [TSV_VARIANT, JSON, HTML] From 2c2f21bd1cc44b31a0a55d814a1ff93e2efc5fe7 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 26 Jun 2024 20:11:35 +0200 Subject: [PATCH 051/128] cicd: Modify pull request approve #TASK-6399 --- .github/workflows/pull-request-approved.yml | 38 +++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 753466dd67a..8a60928fc87 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -1,19 +1,37 @@ name: Pull request approve workflow +run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' on: pull_request_review: - types: [submitted] + types: [ submitted ] jobs: - build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop - with: - maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip + calculate-xetabase-branch: + name: Calculate Xetabase branch + runs-on: ubuntu-22.04 + outputs: + xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} + steps: + - name: Clone java-common-libs + uses: actions/checkout@v4 + with: + fetch-depth: '10' + - id: get_xetabase_branch + name: "Get current branch for Xetabase from target branch" + run: | + chmod +x ./.github/workflows/scripts/xetabase-branch.sh + ls ./.github/workflows/scripts/ + ls ./.github/workflows/ + bash --version + xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} + echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT test: - name: "Run all tests before merging, ie. short, medium and long tests." - uses: ./.github/workflows/test-analysis.yml - needs: build + name: "Run all tests before merging" + needs: calculate-xetabase-branch + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 with: - test_profile: runShortTests,runMediumTests,runLongTests - secrets: inherit + branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} + task: ${{ github.event.pull_request.head.ref }} + secrets: inherit \ No newline at end of file From 5a87a6eb8b69c3779f108483605cb243f02d73d9 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 26 Jun 2024 20:12:28 +0200 Subject: [PATCH 052/128] cicd: Added xetabase script #TASK-6399 --- .github/workflows/scripts/xetabase-branch.sh | 42 ++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/scripts/xetabase-branch.sh diff --git a/.github/workflows/scripts/xetabase-branch.sh b/.github/workflows/scripts/xetabase-branch.sh new file mode 100644 index 00000000000..af17f7f1265 --- /dev/null +++ b/.github/workflows/scripts/xetabase-branch.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Function to calculate the corresponding branch of Xetabase project +get_xetabase_branch() { + # Input parameter (branch name) + input_branch="$1" + + # Check if the branch name is "develop" in that case return the same branch name + if [[ "$input_branch" == "develop" ]]; then + echo "develop" + return 0 + fi + + # Check if the branch name starts with "release-" and follows the patterns "release-a.b.x" or "release-a.b.c.x" + if [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.([0-9]+)\.x$ ]]; then + # Extract the MAJOR part of the branch name + MAJOR=${BASH_REMATCH[1]} + # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR + XETABASE_MAJOR=$((MAJOR - 3)) + # Check if the XETABASE_MAJOR is negative + if (( XETABASE_MAJOR < 0 )); then + echo "Error: 'MAJOR' digit after subtraction results in a negative number." + return 1 + fi + # Construct and echo the new branch name + echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}" + return 0 + fi + + # If the branch name does not match any of the expected patterns + echo "Error: The branch name is not correct." + return 1 +} + +# Check if the script receives exactly one argument +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Call the function with the input branch name +get_xetabase_branch "$1" From 0fec64a00f66ef9cafba343570ad3c552df9b0b8 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 27 Jun 2024 12:00:47 +0200 Subject: [PATCH 053/128] catalog: check ADMIN permissions for DONE statuses, #TASK-5964 --- .../managers/ClinicalAnalysisManager.java | 19 +++++++++++++------ .../managers/InterpretationManager.java | 11 ++++++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 13f29ae3602..1cf414b2490 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -1403,22 +1403,29 @@ private OpenCGAResult update(String organizationId, Study stud } ClinicalAnalysisStudyConfiguration clinicalConfiguration = study.getInternal().getConfiguration().getClinical(); - // Get the clinical status that are CLOSED + // Get the clinical status that are CLOSED and DONE Set closedStatus = new HashSet<>(); + Set doneStatus = new HashSet<>(); for (ClinicalStatusValue clinicalStatusValue : clinicalConfiguration.getStatus()) { if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { closedStatus.add(clinicalStatusValue.getId()); + } else if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.DONE)) { + doneStatus.add(clinicalStatusValue.getId()); } } // If the current clinical analysis: - // - is locked - // - the user wants to update the locked status - // - the user wants to update the status to/from a closed status + // - is locked or panelLocked + // - the user wants to update the locked or panelLocked status + // - the user wants to update the status to/from a done|closed status boolean adminPermissionsChecked = false; - if (clinicalAnalysis.isLocked() || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED + if (clinicalAnalysis.isLocked() || clinicalAnalysis.isPanelLocked() + || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED + || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.DONE || updateParamsClone.getLocked() != null - || (updateParams.getStatus() != null && closedStatus.contains(updateParams.getStatus().getId()))) { + || updateParams.getPanelLocked() != null + || (updateParams.getStatus() != null && (closedStatus.contains(updateParams.getStatus().getId()) + || doneStatus.contains(updateParams.getStatus().getId())))) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.ADMIN); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 72c013ea3ee..2ba3f19350c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -926,14 +926,19 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation // Check if user has permissions to write clinical analysis ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().internalGet(organizationId, study.getUid(), interpretation.getClinicalAnalysisId(), INCLUDE_CLINICAL_ANALYSIS, userId).first(); - authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, - ClinicalAnalysisPermissions.WRITE); - if (clinicalAnalysis.isLocked()) { throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" + " the Interpretation."); } + if (interpretation.isLocked() || updateParams.getLocked() != null) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.ADMIN); + } else { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.WRITE); + } + List events = new ArrayList<>(); if (updateParams != null && StringUtils.isNotEmpty(updateParams.getCreationDate())) { From 9c2e72df02e9286c62e014dd58137d0147f1c8b3 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 27 Jun 2024 12:05:29 +0200 Subject: [PATCH 054/128] app: move migration folder, #TASK-5964 --- .../{ => v3}/v3_2_0/TASK_5964/AddInterpretationName.java | 2 +- .../{ => v3}/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java | 2 +- .../v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java | 2 +- .../{ => v3}/v3_2_0/TASK_5964/RemoveStatusNameMigration.java | 2 +- .../v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v3}/v3_2_0/TASK_5964/AddInterpretationName.java (96%) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v3}/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java (98%) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v3}/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java (98%) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v3}/v3_2_0/TASK_5964/RemoveStatusNameMigration.java (99%) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v3}/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java (98%) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddInterpretationName.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java similarity index 96% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddInterpretationName.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java index 960424a3953..372130c0a5e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddInterpretationName.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java similarity index 98% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java index 10439dfb9f7..39e1447774d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java similarity index 98% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java index 9bfafc457a5..302637cf968 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/RemoveStatusNameMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java similarity index 99% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/RemoveStatusNameMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java index 151c29c320e..80692ae910c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/RemoveStatusNameMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java similarity index 98% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java index 5a7d2da963e..35960978d4c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0.TASK_5964; +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; From f3ac930aced6ff2e4de8992a1c0b3a3693bca3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 28 Jun 2024 14:53:30 +0100 Subject: [PATCH 055/128] storage: Fix numTotalSamples variant result value. #TASK-6436 --- .../VariantQueryProjectionParser.java | 9 ++++++++- .../VariantDBAdaptorMultiFileTest.java | 18 +++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java index bdcd501ad88..95add22ec8e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java @@ -71,7 +71,14 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti } Map> sampleIdsMap = getIncludeSampleIds(query, options, includeStudies, metadataManager); - int numTotalSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); + int numTotalSamples; + if (isValidParam(query, NUM_TOTAL_SAMPLES)) { + // NUM_TOTAL_SAMPLES might have been defined in the PreProcess step. + // This implies that the current query has the samples already paginated. + numTotalSamples = query.getInt(NUM_TOTAL_SAMPLES.key()); + } else { + numTotalSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); + } skipAndLimitSamples(query, sampleIdsMap); int numSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java index 9548338e3d8..903ba51f3a2 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java @@ -381,18 +381,22 @@ public void testSampleLimitSkip() throws Exception { VariantQueryResult result = query(new Query(SAMPLE_METADATA.key(), true).append(VariantQueryParam.INCLUDE_SAMPLE.key(), ALL), options); System.out.println("samples(ALL) = " + result.getSamples()); - for (int i : new int[]{1, 3, 6, 8, 10}) { + int numSamples = metadataManager.getStudyIds().stream().mapToInt(id -> metadataManager.getIndexedSamples(id).size()).sum(); + assertEquals(8, numSamples); + for (int i : new int[]{1, 3, 6, numSamples, 10}) { result = query(new VariantQuery().sampleSkip(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(SKIP=" + i + ") = " + result.getSamples()); - assertEquals(Math.max(0, 8 - i), result.getSamples().values().stream().mapToInt(List::size).sum()); - assertEquals(Math.max(0, 8 - i), result.getNumSamples().intValue()); - assertEquals(8, result.getNumTotalSamples().intValue()); + int expected = Math.max(0, numSamples - i); + assertEquals("Skip = " + i + " , expected " + expected + " out of 8 samples", expected, result.getSamples().values().stream().mapToInt(List::size).sum()); + assertEquals("Skip = " + i + " , expected " + expected + " out of 8 samples", expected, result.getNumSamples().intValue()); + assertEquals(numSamples, result.getNumTotalSamples().intValue()); result = query(new VariantQuery().sampleLimit(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(LIMIT=" + i + ") = " + result.getSamples()); - assertEquals(Math.min(8, i), result.getSamples().values().stream().mapToInt(List::size).sum()); - assertEquals(Math.min(8, i), result.getNumSamples().intValue()); - assertEquals(8, result.getNumTotalSamples().intValue()); + expected = Math.min(numSamples, i); + assertEquals("Limit = " + i + " , expected " + expected + " out of 8 samples", expected, result.getSamples().values().stream().mapToInt(List::size).sum()); + assertEquals("Limit = " + i + " , expected " + expected + " out of 8 samples", expected, result.getNumSamples().intValue()); + assertEquals(numSamples, result.getNumTotalSamples().intValue()); } } From e95bae415936c8718ac5f0396dc342aadf477585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 28 Jun 2024 17:08:50 +0100 Subject: [PATCH 056/128] storage: Fix canUseThisExecutor on SampleIndexOnlyVariantQueryExecutor #TASK-6436 --- .../variant/index/SampleIndexOnlyVariantQueryExecutor.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java index 3e9e0b9b75f..7b1cdc5dceb 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java @@ -180,13 +180,11 @@ protected boolean shouldGetCount(QueryOptions options, boolean iterator) { private boolean isFullyCoveredQuery(Query inputQuery, QueryOptions options) { Query query = new Query(inputQuery); - if (!isQueryCovered(query)) { - return false; - } // ParsedVariantQuery parsedVariantQuery = variantQueryProjectionParser.parseQuery(query, options, true); SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); - return isIncludeCovered(sampleIndexQuery, inputQuery, options); + return isQueryCovered(sampleIndexQuery.getUncoveredQuery()) + && isIncludeCovered(sampleIndexQuery, inputQuery, options); } private boolean isQueryCovered(Query query) { From 76a5a43c5bf3995daa7388e1289e35f76378c884 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 17:39:27 +0200 Subject: [PATCH 057/128] cicd: Added xetabase script #TASK-6399 --- .github/workflows/pull-request-approved.yml | 9 +++------ .../{xetabase-branch.sh => get-xetabase-branch.sh} | 10 +++++++++- 2 files changed, 12 insertions(+), 7 deletions(-) rename .github/workflows/scripts/{xetabase-branch.sh => get-xetabase-branch.sh} (80%) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 8a60928fc87..7961eb047f4 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -12,18 +12,15 @@ jobs: outputs: xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} steps: - - name: Clone java-common-libs + - name: Clone project uses: actions/checkout@v4 with: fetch-depth: '10' - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | - chmod +x ./.github/workflows/scripts/xetabase-branch.sh - ls ./.github/workflows/scripts/ - ls ./.github/workflows/ - bash --version - xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh similarity index 80% rename from .github/workflows/scripts/xetabase-branch.sh rename to .github/workflows/scripts/get-xetabase-branch.sh index af17f7f1265..e971f990c2e 100644 --- a/.github/workflows/scripts/xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -5,6 +5,14 @@ get_xetabase_branch() { # Input parameter (branch name) input_branch="$1" + # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it + if [[ $input_branch == TASK* ]]; then + if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + echo "$GIT_BRANCH"; + exit 0; + fi + fi + # Check if the branch name is "develop" in that case return the same branch name if [[ "$input_branch" == "develop" ]]; then echo "develop" @@ -16,7 +24,7 @@ get_xetabase_branch() { # Extract the MAJOR part of the branch name MAJOR=${BASH_REMATCH[1]} # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR - XETABASE_MAJOR=$((MAJOR - 3)) + XETABASE_MAJOR=$((MAJOR - 1)) # Check if the XETABASE_MAJOR is negative if (( XETABASE_MAJOR < 0 )); then echo "Error: 'MAJOR' digit after subtraction results in a negative number." From 54b667661428dd9ac4c9179f4c12e9f4e739dd05 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 5 Jul 2024 09:47:56 +0200 Subject: [PATCH 058/128] Prepare release 2.12.6 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 14 +++++++------- 21 files changed, 27 insertions(+), 27 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 69a9b33089c..e6cd5eabffb 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index 18071afb30b..c17445c0570 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 46964bfae1f..986afb42ef4 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index cfdf162a973..4ba23d9eb1f 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index ad02409329b..917c115a102 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index 00abcbf61d7..d1a843a369a 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 4e304b5e34c..a0b09511331 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 798bdd26d41..85ac42f2654 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index b5d0d0fad60..726d8456aa3 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index 50c8105886a..ac49b0167ac 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 130716cf970..290972cc572 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 29e4ac40576..ddcd73ee837 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index e4d5e3c8439..64269c04696 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index 7831c67ec49..575b5ef7b73 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index f3792747d50..7583d28aa4f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index 8b769184fa3..1198b5718ab 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 1d3b4e5f795..f6cba2e9980 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 6a725c2746e..31cf6d9531f 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 1f00aeda658..669b237ff25 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 953a68ee373..7e4d5f81c7f 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 ../pom.xml diff --git a/pom.xml b/pom.xml index aef3f1a0bc0..3dfc10baa5a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6-SNAPSHOT + 2.12.6 pom OpenCGA @@ -43,12 +43,12 @@ - 2.12.6_dev - 2.12.6_dev - 5.8.4-SNAPSHOT - 2.12.3-SNAPSHOT - 4.12.1-SNAPSHOT - 2.12.6-SNAPSHOT + 2.12.6 + 2.12.6 + 5.8.4 + 2.12.2 + 4.12.0 + 2.12.6 0.2.0 2.11.4 From fb2df63cf2e9f7d6cdcaf01d72f7216f3ed01f63 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 5 Jul 2024 09:50:36 +0200 Subject: [PATCH 059/128] Prepare next release 2.12.7-SNAPSHOT --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 14 +++++++------- 21 files changed, 27 insertions(+), 27 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index e6cd5eabffb..19e958d2f24 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index c17445c0570..6e3fbc1de9b 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 986afb42ef4..8bc8d823f06 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 4ba23d9eb1f..38a2e260e5d 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 917c115a102..b28dda9d198 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index d1a843a369a..264631c0715 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index a0b09511331..f7f42cdb79b 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 85ac42f2654..14b4301df21 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 726d8456aa3..c7bbc86990e 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index ac49b0167ac..a64f7df1ae6 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 290972cc572..d731ec91136 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index ddcd73ee837..00cef59a609 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index 64269c04696..d1701f8cc3f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index 575b5ef7b73..b78e0af52c8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 7583d28aa4f..5caa3dbe832 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index 1198b5718ab..de9e737a091 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index f6cba2e9980..53166582a8d 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 31cf6d9531f..b4a77d6c006 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 669b237ff25..0c79bf18478 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 7e4d5f81c7f..c2d6741df0b 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 3dfc10baa5a..3b066129ccc 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.7-SNAPSHOT pom OpenCGA @@ -43,12 +43,12 @@ - 2.12.6 - 2.12.6 - 5.8.4 - 2.12.2 - 4.12.0 - 2.12.6 + 2.12.7_dev + 2.12.7_dev + 5.8.5-SNAPSHOT + 2.12.3-SNAPSHOT + 4.12.1-SNAPSHOT + 2.12.7-SNAPSHOT 0.2.0 2.11.4 From 732f2ffee9e6c324e00c1bec2a888b4ed15245cd Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 5 Jul 2024 15:17:13 +0200 Subject: [PATCH 060/128] catalog: use panelLocked constant, #TASK-5964 --- .../catalog/managers/ClinicalAnalysisManager.java | 9 ++++++--- .../catalog/managers/InterpretationManager.java | 10 ++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 1cf414b2490..7ab4c80bb32 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -1583,7 +1583,8 @@ private OpenCGAResult update(String organizationId, Study stud if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels()) && updateParamsClone.getPanelLocked() != null && updateParamsClone.getPanelLocked()) { - throw new CatalogException("Updating the list of panels and setting 'panelLock' to true at the same time is not allowed."); + throw new CatalogException("Updating the list of panels and setting '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' to true at the same time is not allowed."); } if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels())) { @@ -1610,9 +1611,11 @@ private OpenCGAResult update(String organizationId, Study stud // in order to set panelLock to true. Otherwise, that action is not allowed. Set panelIds = clinicalAnalysis.getPanels().stream().map(Panel::getId).collect(Collectors.toSet()); String exceptionMsgPrefix = "The interpretation '"; - String exceptionMsgSuffix = "' does not contain any of the case panels. 'panelLock' can only be set to true if all" + String exceptionMsgSuffix = "' does not contain any of the case panels. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' can only be set to true if all" + " all Interpretations contains a non-empty subset of the panels used by the case."; - String alternativeExceptionMsgSuffix = "' is using a panel not defined by the case. 'panelLock' can only be set to true if all" + String alternativeExceptionMsgSuffix = "' is using a panel not defined by the case. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' can only be set to true if all" + " all Interpretations contains a non-empty subset of the panels used by the case."; if (clinicalAnalysis.getInterpretation() != null) { if (CollectionUtils.isEmpty(clinicalAnalysis.getInterpretation().getPanels())) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 2ba3f19350c..7f7be104d95 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -309,7 +309,8 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio List panelList = new ArrayList<>(clinicalPanelIds.size()); for (Panel panel : interpretation.getPanels()) { if (!clinicalPanelIds.containsKey(panel.getId())) { - throw new CatalogException("'panelLock' from ClinicalAnalysis is set to True. Please, leave list of panels empty" + throw new CatalogException("'" + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + + "' from ClinicalAnalysis is set to True. Please, leave list of panels empty" + " so they can be inherited or pass at least a subset of the panels defined in the Clinical Analysis."); } panelList.add(clinicalPanelIds.get(panel.getId())); @@ -999,8 +1000,8 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation if (updateParams != null && CollectionUtils.isNotEmpty(updateParams.getPanels())) { if (clinicalAnalysis.isPanelLocked()) { - throw new CatalogException("Updating panels from Interpretation is not allowed. 'panelLock' from ClinicalAnalysis is set " - + "to True."); + throw new CatalogException("Updating panels from Interpretation is not allowed. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' from ClinicalAnalysis is set to True."); } // Validate and get panels @@ -1145,7 +1146,8 @@ public OpenCGAResult revert(String studyStr, String clinicalAnal + " the Interpretation."); } if (clinicalAnalysis.isPanelLocked()) { - throw new CatalogException("Could not revert the Interpretation. 'panelLock' is set to True, so no further modifications" + throw new CatalogException("Could not revert the Interpretation. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' is set to True, so no further modifications" + " can be made to the Interpretation."); } From 6485ab4f987378b170f5d77301262f2956624f23 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 8 Jul 2024 12:29:38 +0200 Subject: [PATCH 061/128] pom: Added github packages to manage snapshots #TASK-6345 --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3b066129ccc..4f20ae98919 100644 --- a/pom.xml +++ b/pom.xml @@ -210,9 +210,10 @@ github - https://maven.pkg.github.com/opencb + https://maven.pkg.github.com/opencb/* true + daily From 35526348ec829283e901332cf9e3653f835b3736 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 8 Jul 2024 12:50:28 +0200 Subject: [PATCH 062/128] pom: Added sonatype also manage snapshots #TASK-6345 --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4f20ae98919..1d43dac6518 100644 --- a/pom.xml +++ b/pom.xml @@ -213,7 +213,6 @@ https://maven.pkg.github.com/opencb/* true - daily From 941c98b970c914f534632a4aeb22eb2cf8a48868 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 8 Jul 2024 12:57:33 +0200 Subject: [PATCH 063/128] pom: Comment sonatype to manage snapshots #TASK-6345 --- pom.xml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 1d43dac6518..32740c81d6c 100644 --- a/pom.xml +++ b/pom.xml @@ -200,19 +200,20 @@ - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - true - daily - - + + + + + + + + github https://maven.pkg.github.com/opencb/* true + daily From 59fb8dec60052f245500a6a68da427b33ef3f762 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 8 Jul 2024 13:37:51 +0200 Subject: [PATCH 064/128] pom: Comment repositories section #TASK-6345 --- pom.xml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 32740c81d6c..baa98a08b25 100644 --- a/pom.xml +++ b/pom.xml @@ -199,7 +199,7 @@ - + @@ -208,15 +208,15 @@ - - github - https://maven.pkg.github.com/opencb/* - - true - daily - - - + + + + + + + + + From 166300c6c4336be2cea8f21414b05a9b53e4ced8 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 8 Jul 2024 13:58:37 +0200 Subject: [PATCH 065/128] pom: Restore repositories section #TASK-6345 --- pom.xml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index baa98a08b25..3b066129ccc 100644 --- a/pom.xml +++ b/pom.xml @@ -199,24 +199,23 @@ - - - - - - - - - - - - - - - - - - + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + true + daily + + + + github + https://maven.pkg.github.com/opencb + + true + + + From e3de2b88e0c9075393832fe5286b39384ed68906 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 8 Jul 2024 16:25:20 +0200 Subject: [PATCH 066/128] storage-core pom: Exclusion distlib #TASK-6345 --- opencga-storage/opencga-storage-core/pom.xml | 6 ++++++ pom.xml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index d731ec91136..b2b6da5bea7 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -191,6 +191,12 @@ com.databricks SnpEff + + + distlib + distlib + + diff --git a/pom.xml b/pom.xml index 3b066129ccc..895167718f4 100644 --- a/pom.xml +++ b/pom.xml @@ -814,6 +814,12 @@ com.databricks SnpEff ${snpeff.version} + + + distlib + distlib + + org.apache.parquet From e8666f990bcb99cdabe3bacf4e6959b5a98de121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 10 Jul 2024 09:52:11 +0100 Subject: [PATCH 067/128] app: Fix merge issues. #TASK-5448 --- .../migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java | 2 +- .../storage/DetectIllegalConcurrentFileLoadingsMigration.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java index 4efa260d965..c71f0483427 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java @@ -11,7 +11,7 @@ public class AddAllelesColumnToPhoenix extends StorageMigrationTool { @Override protected void run() throws Exception { - for (String project : getVariantStorageProjects(organizationId)) { + for (String project : getVariantStorageProjects()) { VariantStorageEngine engine = getVariantStorageEngineByProject(project); if (engine.getStorageEngineId().equals("hadoop")) { logger.info("Adding missing columns (if any) for project " + project); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java index 132e1357f1b..1b916a8316a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java @@ -42,7 +42,7 @@ public class DetectIllegalConcurrentFileLoadingsMigration extends StorageMigrati @Override protected void run() throws Exception { - for (String project : getVariantStorageProjects(organizationId)) { + for (String project : getVariantStorageProjects()) { VariantStorageEngine engine = getVariantStorageEngineByProject(project); if (!engine.getStorageEngineId().equals("hadoop")) { continue; From af360fa79828604e985c7c14d43023272209628f Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 10 Jul 2024 11:43:51 +0200 Subject: [PATCH 068/128] app: add missing migration for panelLock field, #TASK-5964 --- .../TASK_5964/RenamePanelLockMigration.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java new file mode 100644 index 00000000000..32c10c3fab7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java @@ -0,0 +1,26 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "rename_panel_lock_from_clinical__task_5964", description = "Rename 'panelLock' to 'panelLocked' #TASK-5964", + version = "3.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240710) +public class RenamePanelLockMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = new Document(); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + getMongoCollection(collection).updateMany(query, new Document("$rename", new Document("panelLock", "panelLocked"))); + } + } + +} From 44062c1060cbb2c962a62da47c702a1ef6a13dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 10 Jul 2024 11:02:13 +0100 Subject: [PATCH 069/128] storage: Allow repeat variant-setup before indexing any file. #TASK-5448 --- .../VariantSetupOperationManager.java | 21 ++++++++++++------- ...perationsVariantStorageCommandOptions.java | 2 +- .../catalog/managers/StudyManager.java | 11 +++++++++- .../models/variant/VariantSetupParams.java | 2 +- .../variant/archive/ArchiveTableHelper.java | 2 +- .../index/sample/SampleIndexDBAdaptor.java | 2 +- 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java index 31e8191ec2c..0003ca105bf 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -12,11 +12,14 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class VariantSetupOperationManager extends OperationManager { public static final String ID = "variant-setup"; + private static Logger logger = LoggerFactory.getLogger(VariantSetupOperationManager.class); public VariantSetupOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine variantStorageEngine) { super(variantStorageManager, variantStorageEngine); @@ -127,8 +130,7 @@ private void check(String studyStr, VariantSetupParams params, String token) thr } } if (hasVariantSetup(study)) { - // TODO: Allow double setup if no files are loaded? - throw new IllegalArgumentException("Study " + studyStr + " is already setup"); + logger.info("Study {} was already setup. Re executing variant-setup", studyStr); } if (params.getExpectedFiles() == null || params.getExpectedFiles() <= 0) { @@ -145,15 +147,20 @@ private void check(String studyStr, VariantSetupParams params, String token) thr public static boolean hasVariantSetup(Study study) { boolean hasSetup = false; + VariantSetupResult setup = getVariantSetupResult(study); + if (setup != null && setup.getStatus() == VariantSetupResult.Status.READY) { + hasSetup = true; + } + return hasSetup; + } + + private static VariantSetupResult getVariantSetupResult(Study study) { if (study.getInternal() != null && study.getInternal().getConfiguration() != null && study.getInternal().getConfiguration().getVariantEngine() != null) { - VariantSetupResult setup = study.getInternal().getConfiguration().getVariantEngine().getSetup(); - if (setup != null && setup.getStatus() == VariantSetupResult.Status.READY) { - hasSetup = true; - } + return study.getInternal().getConfiguration().getVariantEngine().getSetup(); } - return hasSetup; + return null; } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index 08fa3439d49..4192a64400b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -1200,7 +1200,7 @@ public class SetupVariantCommandOptions { @Parameter(names = {"--average-file-size"}, description = "Average size of the files that will be loaded. This number is only used as a hint. If the real size of the files is different, the loader should be able to handle it. Accepts units. e.g. 435MB, 2GB, 86KB. If not provided, the value will be inferred from the file type.", required = false, arity = 1) public String averageFileSize; - @Parameter(names = {"--variants-per-sample"}, description = "Number of variants per sample. This number is only used as a hint. If the real number of variants per sample is different, the loader should be able to handle it. If not provided, the value will be inferred from the file type.", required = false, arity = 1) + @Parameter(names = {"--variants-per-sample"}, description = "Number of variants per sample (non hom_ref variants). This number is only used as a hint. If the real number of variants per sample is different, the loader should be able to handle it. If not provided, the value will be inferred from the file type.", required = false, arity = 1) public Integer variantsPerSample; @Parameter(names = {"--average-samples-per-file"}, description = "Average number of samples per file. This number is only used as a hint. If the real number of samples per file is different, the loader should be able to handle it. If not provided, the value will be inferred from the expectedSamples and expectedFiles and dataDistribution.", required = false, arity = 1) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 04eff764dcb..70f933f6430 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -1835,11 +1835,20 @@ public void setVariantEngineSetupOptions(String studyStr, VariantSetupResult var if (configuration == null) { configuration = new StudyVariantEngineConfiguration(); } - configuration.setSetup(variantSetupResult); if (configuration.getOptions() == null) { configuration.setOptions(new ObjectMap()); } + VariantSetupResult prevSetupValue = configuration.getSetup(); + if (prevSetupValue != null && prevSetupValue.getOptions() != null) { + // Variant setup was already executed. + // Remove the options from the previous execution before adding the new ones + // Check that both key/value matches, to avoid removing options that might have been modified manually + for (Map.Entry entry : prevSetupValue.getOptions().entrySet()) { + configuration.getOptions().remove(entry.getKey(), entry.getValue()); + } + } configuration.getOptions().putAll(variantSetupResult.getOptions()); + configuration.setSetup(variantSetupResult); ObjectMap parameters = new ObjectMap(StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key(), configuration); getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java index c027bbf3df7..1a97c0de539 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -27,7 +27,7 @@ public class VariantSetupParams extends ToolParams { + "If not provided, the value will be inferred from the file type.") private String averageFileSize; - @DataField(description = "Number of variants per sample. This number is only used as a hint. " + @DataField(description = "Number of variants per sample (non hom_ref variants). This number is only used as a hint. " + "If the real number of variants per sample is different, the loader should be able to handle it. " + "If not provided, the value will be inferred from the file type.") private Integer variantsPerSample; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java index 3a0e168d96e..59de0f4b8b8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java @@ -211,7 +211,7 @@ public static void expandTableIfNeeded(ObjectMap options, String archiveTable, i batch -> generateBatchSplitsHuman(rowKeyFactory, splitsPerBatch, batch), extraBatches, batch -> Bytes.toBytes(rowKeyFactory.generateBlockIdFromBatch(batch))); if (newRegions > 0) { - logger.info("Table '" + archiveTable + "' expanded with " + newRegions + " new regions for batch " + thisBatch); + logger.info("Archive table '" + archiveTable + "' expanded with " + newRegions + " new regions for batch " + thisBatch); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java index 72884815476..0d785856776 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java @@ -834,7 +834,7 @@ public void expandTableIfNeeded(int studyId, int version, List sampleId sampleIndexExtraBatches, batch -> SampleIndexSchema.toRowKey(batch * preSplitSize)); if (newRegions != 0) { // Log number of new regions - logger.info("Created {} new regions in table '{}'", newRegions, sampleIndexTable); + logger.info("Sample index table '" + sampleIndexTable + "' expanded with " + newRegions + " new regions"); } } catch (IOException e) { throw new UncheckedIOException(e); From 00b985d95c40dc2816730824d1376401ac2b6d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 10 Jul 2024 16:58:48 +0100 Subject: [PATCH 070/128] storage: Remove some default normalization extensions. #TASK-5448 --- .../storage/core/variant/VariantStorageOptions.java | 5 +++-- .../storage/core/variant/VariantStoragePipeline.java | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java index cf516491a4a..9ee5b3068e9 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java @@ -1,10 +1,11 @@ package org.opencb.opencga.storage.core.variant; import org.opencb.biodata.models.variant.metadata.Aggregation; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.config.ConfigurationOption; +import java.util.Arrays; + public enum VariantStorageOptions implements ConfigurationOption { STUDY("study"), @@ -26,7 +27,7 @@ public enum VariantStorageOptions implements ConfigurationOption { TRANSFORM_ISOLATE("transform.isolate", false), // Do not store file in metadata NORMALIZATION_SKIP("normalization.skip", false), // Do not run normalization NORMALIZATION_REFERENCE_GENOME("normalization.referenceGenome"), - NORMALIZATION_EXTENSIONS("normalization.extensions", ParamConstants.ALL), + NORMALIZATION_EXTENSIONS("normalization.extensions", Arrays.asList("VAF", "SV", "CUSTOM")), DEDUPLICATION_POLICY("deduplication.policy", "maxQual"), DEDUPLICATION_BUFFER_SIZE("deduplication.bufferSize", 100), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 3a36accae04..34bbd5cfe5a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -423,7 +423,7 @@ protected Task initNormalizer(VariantFileMetadata metadata) th .then(new VariantReferenceBlockCreatorTask(metadata.getHeader())); } if (CollectionUtils.isEmpty(enabledExtensions)) { - enabledExtensions = Collections.singleton(NORMALIZATION_EXTENSIONS.defaultValue()); + enabledExtensions = NORMALIZATION_EXTENSIONS.defaultValue(); } if ((enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.NONE))) { logger.info("Skip normalization extensions"); @@ -431,10 +431,9 @@ protected Task initNormalizer(VariantFileMetadata metadata) th logger.info("Enable normalization extensions: {}", enabledExtensions); VariantNormalizerExtensionFactory extensionFactory; if (enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.ALL)) { - extensionFactory = new VariantNormalizerExtensionFactory(); - } else { - extensionFactory = new VariantNormalizerExtensionFactory(new HashSet<>(enabledExtensions)); + enabledExtensions = NORMALIZATION_EXTENSIONS.defaultValue(); } + extensionFactory = new VariantNormalizerExtensionFactory(new HashSet<>(enabledExtensions)); Task extension = extensionFactory.buildExtensions(metadata); if (extension == null) { logger.info("No normalization extensions can be used."); From f7e8ccb921ee577c79e6f8943ac2994705a28069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 16 Jul 2024 16:05:33 +0100 Subject: [PATCH 071/128] storage: Do not use CIEND on INSERTIONS to build HBase RowKey. #TASK-6558 --- .../org/opencb/opencga/core/api/ParamConstants.java | 2 +- .../core/variant/VariantStorageEngineSVTest.java | 11 ++++++++--- .../src/test/resources/variant-test-sv.vcf | 2 +- .../src/test/resources/variant-test-sv_2.vcf | 2 +- .../adaptors/phoenix/VariantPhoenixKeyFactory.java | 13 +++++++------ .../index/sample/SampleIndexVariantBiConverter.java | 2 +- .../phoenix/VariantPhoenixKeyFactoryTest.java | 2 +- 7 files changed, 20 insertions(+), 14 deletions(-) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index bd177657c23..086f90132ec 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -78,7 +78,7 @@ public class ParamConstants { public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase"; public static final String CELLBASE_VERSION = "v5.8"; public static final String CELLBASE_DATA_RELEASE = "7"; - public static final String CELLBASE_APIKEY = ""; + public static final String CELLBASE_APIKEY = null; public static final String POP_FREQ_1000G_CB_V4 = "1kG_phase3"; public static final String POP_FREQ_1000G_CB_V5 = "1000G"; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 64fc14c4c6a..638b10023be 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -60,8 +60,8 @@ public void before() throws Exception { protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); + variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); variantStorageEngine.reloadCellbaseConfiguration(); @@ -179,6 +179,11 @@ protected Map readVariants(URI input) throws StorageEngineExcep variantReader.post(); variantReader.close(); - return variants.stream().collect(Collectors.toMap(Variant::toString, v -> v)); + return variants.stream().peek(v->{ + if (v.getEnd() < v.getStart()) { + v.getSv().setCiEndLeft(null); + v.getSv().setCiEndRight(null); + } + }).collect(Collectors.toMap(Variant::toString, v -> v)); } } diff --git a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf index 9fc4fe9c325..6c1d32cf79a 100644 --- a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf +++ b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf @@ -44,7 +44,7 @@ 1 600000 . T . PASS SVTYPE=DEL;END=600250;SVLEN=-205;CIPOS=-56,20;CIEND=-10,62 GT 0/1 0/1 1 650000 . C . PASS SVTYPE=DEL;END=650100;CIEND=0,48;CIPOS=-49,0 GT 0/1 0/1 1 700000 . C . PASS SVTYPE=DEL;END=700297;SVLEN=-297;CIPOS=-22,18;CIEND=-12,32 GT 0/1 0/1 -1 800000 . A . PASS SVTYPE=INS;END=800000;SVLEN=6027;CIPOS=-16,22;RIGHT_SVINSSEQ=ACCACACCCACACAACACACA;LEFT_SVINSSEQ=TGTGGTGTGTGTGGTGTG GT 0/1 0/1 +1 800000 . A . PASS SVTYPE=INS;END=800000;SVLEN=6027;CIPOS=-16,22;CIEND=-16,22;RIGHT_SVINSSEQ=ACCACACCCACACAACACACA;LEFT_SVINSSEQ=TGTGGTGTGTGTGGTGTG GT 0/1 0/1 1 850000 . A . PASS SVTYPE=INS;END=850000;SVINSSEQ=ACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACA GT 0/1 0/1 1 860000 . A . PASS SVTYPE=INVERSION;END=870000 GT 0/1 0/1 1 900000 . G . PASS SVTYPE=INS;END=900000;SVLEN=6027;CIPOS=-16,22 GT 0/1 0/1 diff --git a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf index 32ad6f4c4f0..0cfa097d85f 100644 --- a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf +++ b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf @@ -56,4 +56,4 @@ 1 1500000 . C ,, . PASS SVTYPE=CNV;CIEND=-150,150;CIPOS=-150,150;END=1504000 GT 0/1 2/3 1 1600000 . T G . PASS . GT 0/1 0/1 2 100001 BND_V T ]chr13:300000]T . PASS SVTYPE=BND GT 0/1 0/1 -13 300000 BND_U C C[chr2:100001[ . PASS SVTYPE=BND;CIPOS=-150,0 GT 0/1 0/1 \ No newline at end of file +13 300000 BND_U C C[chr2:100001[ . PASS SVTYPE=BND;CIPOS=-150,0;CIEND=-150,0 GT 0/1 0/1 \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java index 9e06ebc4b51..d65588d2baf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java @@ -146,7 +146,7 @@ public static boolean mightHashAlleles(Variant variant) { */ public static byte[] generateVariantRowKey(String chrom, int start, Integer end, String ref, String alt, StructuralVariation sv) { chrom = Region.normalizeChromosome(chrom); - alt = buildSymbolicAlternate(ref, alt, end, sv); + alt = buildSymbolicAlternate(ref, alt, start, end, sv); int size = getSize(chrom, ref, alt); if (size > HConstants.MAX_ROW_LENGTH) { @@ -201,11 +201,11 @@ public static String buildAlleles(Variant v) { } public static String buildSymbolicAlternate(Variant v) { - return buildSymbolicAlternate(v.getReference(), v.getAlternate(), v.getEnd(), v.getSv()); + return buildSymbolicAlternate(v.getReference(), v.getAlternate(), v.getStart(), v.getEnd(), v.getSv()); } // visible for test - public static String buildSymbolicAlternate(String reference, String alternate, Integer end, StructuralVariation sv) { + public static String buildSymbolicAlternate(String reference, String alternate, int start, Integer end, StructuralVariation sv) { if (sv != null) { byte[] alternateBytes = alternate.getBytes(); if (!Allele.wouldBeSymbolicAllele(alternateBytes) && emptyCiStartEnd(sv)) { @@ -216,14 +216,15 @@ public static String buildSymbolicAlternate(String reference, String alternate, if (StructuralVariantType.TANDEM_DUPLICATION.equals(sv.getType())) { alternate = VariantBuilder.DUP_TANDEM_ALT; } - boolean bnd = Allele.wouldBeBreakpoint(alternateBytes); + // Ignore CIEND on variants without an actual END. This includes Breakends and INSERTIONS + boolean ignoreCiend = Allele.wouldBeBreakpoint(alternateBytes) || end < start; alternate = alternate + SV_ALTERNATE_SEPARATOR + end + SV_ALTERNATE_SEPARATOR + (sv.getCiStartLeft() == null ? 0 : sv.getCiStartLeft()) + SV_ALTERNATE_SEPARATOR + (sv.getCiStartRight() == null ? 0 : sv.getCiStartRight()) - + SV_ALTERNATE_SEPARATOR + (bnd | sv.getCiEndLeft() == null ? 0 : sv.getCiEndLeft()) - + SV_ALTERNATE_SEPARATOR + (bnd | sv.getCiEndRight() == null ? 0 : sv.getCiEndRight()); + + SV_ALTERNATE_SEPARATOR + ((ignoreCiend || sv.getCiEndLeft() == null) ? 0 : sv.getCiEndLeft()) + + SV_ALTERNATE_SEPARATOR + ((ignoreCiend || sv.getCiEndRight() == null) ? 0 : sv.getCiEndRight()); if (StringUtils.isNotEmpty(sv.getLeftSvInsSeq()) || StringUtils.isNotEmpty(sv.getRightSvInsSeq())) { alternate = alternate diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java index 83e9dec3900..387a10f60ec 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java @@ -672,7 +672,7 @@ protected int getRelativeStart(Variant variant) { } protected String getAlternate(Variant v) { - return VariantPhoenixKeyFactory.buildSymbolicAlternate(v.getReference(), v.getAlternate(), v.getEnd(), v.getSv()); + return VariantPhoenixKeyFactory.buildSymbolicAlternate(v); } /** diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java index 578ce5df6ca..80692ba9a48 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java @@ -129,7 +129,7 @@ public byte[] generateVariantRowKeyPhoenix(Variant variant) { ImmutableBytesWritable key = new ImmutableBytesWritable(); String reference = variant.getReference(); - String alternate = VariantPhoenixKeyFactory.buildSymbolicAlternate(reference, variant.getAlternate(), variant.getEnd(), variant.getSv()); + String alternate = VariantPhoenixKeyFactory.buildSymbolicAlternate(variant); table.newKey(key, new byte[][]{ Bytes.toBytes(variant.getChromosome()), Bytes.toBytes(variant.getStart()), From 9fae390cb630d41d5ee7ad6e25f9c017c64eea9b Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 17 Jul 2024 11:07:25 +0200 Subject: [PATCH 072/128] Prepare new development branch release-3.2.x --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-compat-api/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.0/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.2/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.4/pom.xml | 2 +- .../opencga-storage-hadoop-compat/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.13/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdi5.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 14 +++++++------- 27 files changed, 33 insertions(+), 33 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index da118995989..1d127746157 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index f1056a01351..cac11d63828 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 2648a06d5ca..7457317446b 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index e18ab8949ad..043c1c5788d 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 379849dd1a5..02a59d91cf6 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index 741360749ad..c59da1f754d 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index babb08f4107..73192c29e58 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index bdfa34a64b8..eabe8a65d7c 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 9c27d061dff..df69dcc79dd 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index cd5cb2c0888..12400612512 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 80157147a92..0a392fe9020 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml index 20ab59d0965..740b4bb3101 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml index 7547d0d8404..d8dd4f61336 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml index 58174b253b4..c24d2e7072b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml index e4a355cf8f5..2bf4db839f0 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml index 7717a2fd09f..0dd166e460a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index ddc5d8e9792..876fb2c1054 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml index f1b656e0c2b..44bbbe4acbf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml index 00c4dddb2f2..674db7f319c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml index 9dfa1bb4023..312d6ce845d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml index d6c2eecf171..3529773711c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml index 6462be6cfa4..9f791862b43 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 7e2f40af775..e72f24d30d9 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index e4c020243bd..2a91d2753e4 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 2fdd79ef9f3..83f9de6ce82 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index d2a279f522d..88dbdeb7783 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 69bbd8f1151..84e170213b8 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.3.0-SNAPSHOT + 3.2.1-SNAPSHOT pom OpenCGA @@ -44,12 +44,12 @@ - 3.3.0_dev - 3.3.0_dev - 6.2.0-SNAPSHOT - 3.3.0-SNAPSHOT - 5.3.0-SNAPSHOT - 3.3.0-SNAPSHOT + 3.2.1_dev + 3.2.1_dev + 6.1.1-SNAPSHOT + 3.2.1-SNAPSHOT + 5.2.1-SNAPSHOT + 3.2.1-SNAPSHOT 0.2.0 2.14.3 From f601da6065716709e85f986d71c90a3f8ac1f103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 17 Jul 2024 11:14:26 +0100 Subject: [PATCH 073/128] storage: Remove uneeded assumptions in tests. #TASK-6558 --- .../core/variant/VariantStorageEngineSVTest.java | 15 ++------------- .../phoenix/VariantPhoenixKeyFactory.java | 1 + 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 638b10023be..eb9d54d608f 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -7,7 +7,6 @@ import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.SampleEntry; -import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.commons.datastore.core.Query; @@ -149,12 +148,7 @@ protected void checkCorrectness(URI file) throws StorageEngineException, NonStan actualStudyEntry.getFiles().get(0).setFileId(""); assertEquals(expectedStudyEntry.getFiles().get(0), actualStudyEntry.getFiles().get(0)); - - if (actual.getAlternate().equals("") || actual.getType().equals(VariantType.BREAKEND)) { - System.err.println("WARN: Variant " + actual + (actual.getAnnotation() == null ? " without annotation" : " with annotation")); - } else { - assertNotNull(actual.toString(), actual.getAnnotation()); - } + assertNotNull(actual.toString(), actual.getAnnotation()); } } @@ -179,11 +173,6 @@ protected Map readVariants(URI input) throws StorageEngineExcep variantReader.post(); variantReader.close(); - return variants.stream().peek(v->{ - if (v.getEnd() < v.getStart()) { - v.getSv().setCiEndLeft(null); - v.getSv().setCiEndRight(null); - } - }).collect(Collectors.toMap(Variant::toString, v -> v)); + return variants.stream().collect(Collectors.toMap(Variant::toString, v -> v)); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java index d65588d2baf..977e6fbbf93 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java @@ -217,6 +217,7 @@ public static String buildSymbolicAlternate(String reference, String alternate, alternate = VariantBuilder.DUP_TANDEM_ALT; } // Ignore CIEND on variants without an actual END. This includes Breakends and INSERTIONS + // These variants are not expected to have CIEND. This is a redundant check, as the CIEND should be empty after normalization. boolean ignoreCiend = Allele.wouldBeBreakpoint(alternateBytes) || end < start; alternate = alternate From 18da2cca3416d067b79d0baca98aa28efa4f910a Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 18 Jul 2024 10:31:30 +0200 Subject: [PATCH 074/128] server: add new tagsAction query parameter, #TASK-6568 --- .../db/mongodb/NoteMongoDBAdaptor.java | 2 +- .../opencga/catalog/managers/NoteManager.java | 3 +- .../catalog/managers/NoteManagerTest.java | 44 +++++++++++++++++++ .../server/rest/OrganizationWSServer.java | 9 ++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java index c32644fee46..820f15ed2f7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java @@ -234,7 +234,7 @@ private UpdateDocument parseAndValidateUpdateParams(Document note, ObjectMap par filterStringListParams(parameters, document.getSet(), tagsParams); break; case REMOVE: - filterStringListParams(parameters, document.getPull(), tagsParams); + filterStringListParams(parameters, document.getPullAll(), tagsParams); break; case ADD: filterStringListParams(parameters, document.getAddToSet(), tagsParams); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java index b3c7139bd30..e81112cd04f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java @@ -302,8 +302,7 @@ private OpenCGAResult update(long noteUid, NoteUpdateParams noteUpdatePara // Write who's performing the update updateMap.put(NoteDBAdaptor.QueryParams.USER_ID.key(), tokenPayload.getUserId()); - OpenCGAResult update = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, - QueryOptions.empty()); + OpenCGAResult update = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated note OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java index 11e5d29e37a..9d1afcadb2b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java @@ -7,6 +7,9 @@ import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; import org.opencb.opencga.core.models.notes.NoteUpdateParams; @@ -16,6 +19,8 @@ import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import static org.junit.Assert.*; @@ -190,6 +195,45 @@ public void updateStudyNoteTest() throws CatalogException { catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), noteUpdateParams, INCLUDE_RESULT, normalToken1); } + @Test + public void noteTagsUpdateTest() throws CatalogException { + NoteCreateParams noteCreateParams = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setTags(Arrays.asList("tag1", "tag2")) + .setValue("hello"); + Note note = catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(2, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag1", "tag2").toArray(), note.getTags().toArray()); + + QueryOptions queryOptions = new QueryOptions(); + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.ADD); + queryOptions.put(Constants.ACTIONS, actionMap); + queryOptions.put(ParamConstants.INCLUDE_RESULT_PARAM, true); + NoteUpdateParams updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag3", "tag1")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(3, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag1", "tag2", "tag3").toArray(), note.getTags().toArray()); + + // Remove tag1 and tag2 + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.REMOVE); + queryOptions.put(Constants.ACTIONS, actionMap); + updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag1", "tag2")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(1, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag3").toArray(), note.getTags().toArray()); + + // Set new list of tags + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.SET); + queryOptions.put(Constants.ACTIONS, actionMap); + updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag4", "tag5")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(2, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag4", "tag5").toArray(), note.getTags().toArray()); + } + @Test public void getStudyNoteTest() throws CatalogException { NoteCreateParams noteCreateParams1 = new NoteCreateParams() diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java index 15085581bca..73079cf2704 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java @@ -17,6 +17,7 @@ package org.opencb.opencga.server.rest; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -204,9 +205,17 @@ public Response createNote( }) public Response updateNote( @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = "Action to be performed if the array of tags is being updated.", allowableValues = "ADD,REMOVE,SET", defaultValue = "ADD") + @QueryParam("tagsAction") ParamUtils.BasicUpdateAction tagsAction, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(value = "JSON containing the Note fields to be updated.", required = true) NoteUpdateParams parameters) { try { + if (tagsAction == null) { + tagsAction = ParamUtils.BasicUpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), tagsAction); + queryOptions.put(Constants.ACTIONS, actionMap); OpenCGAResult result = catalogManager.getNotesManager().updateOrganizationNote(noteId, parameters, queryOptions, token); return createOkResponse(result); } catch (Exception e) { From bea231f685cbfcdf9062517a233986e2837638de Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 18 Jul 2024 14:32:18 +0200 Subject: [PATCH 075/128] server: add tagsAction to study notes update, #TASK-6568 --- .../org/opencb/opencga/server/rest/StudyWSServer.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java index f6cc3fe8882..8492d30c490 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java @@ -23,6 +23,7 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.templates.TemplateRunner; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.Constants; @@ -46,7 +47,9 @@ import javax.ws.rs.core.*; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; @@ -613,9 +616,17 @@ public Response createNote( public Response updateNote( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @PathParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = "Action to be performed if the array of tags is being updated.", allowableValues = "ADD,REMOVE,SET", defaultValue = "ADD") + @QueryParam("tagsAction") ParamUtils.BasicUpdateAction tagsAction, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(value = "JSON containing the Note fields to be updated.", required = true) NoteUpdateParams parameters) { try { + if (tagsAction == null) { + tagsAction = ParamUtils.BasicUpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), tagsAction); + queryOptions.put(Constants.ACTIONS, actionMap); OpenCGAResult result = catalogManager.getNotesManager().updateStudyNote(studyStr, noteId, parameters, queryOptions, token); return createOkResponse(result); } catch (Exception e) { From bdbc364ac2a58d14601a4863efac491b49a1cfdf Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 18 Jul 2024 15:30:45 +0200 Subject: [PATCH 076/128] catalog: ensure password change and reset use same policy, #TASK-6494 --- .../CatalogAuthenticationManager.java | 4 ++-- .../opencga/catalog/managers/UserManager.java | 5 ++++- .../opencga/core/common/PasswordUtils.java | 16 +++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java index 65280a1be61..61f9fef8079 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java @@ -17,7 +17,6 @@ package org.opencb.opencga.catalog.auth.authentication; import io.jsonwebtoken.SignatureAlgorithm; -import org.apache.commons.lang3.RandomStringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; @@ -26,6 +25,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.MailUtils; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Email; import org.opencb.opencga.core.models.user.AuthenticationResponse; @@ -125,7 +125,7 @@ public String createNonExpiringToken(String organizationId, String userId, Map user = dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index 4c9f03d12e1..35c8bd46b64 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -84,6 +84,9 @@ public void changePassword(String organizationId, String userId, String oldPassw if (oldPassword.equals(newPassword)) { throw new CatalogException("New password is the same as the old password."); } + if (!PasswordUtils.isStrongPassword(newPassword)) { + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); + } getUserDBAdaptor(organizationId).checkId(userId); String authOrigin = getAuthenticationOriginId(organizationId, userId); @@ -174,7 +177,7 @@ public OpenCGAResult create(User user, String password, String token) thro try { if (StringUtils.isNotEmpty(password) && !PasswordUtils.isStrongPassword(password)) { - throw new CatalogException("Invalid password. Check password strength for user " + user.getId()); + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); } if (user.getProjects() != null && !user.getProjects().isEmpty()) { throw new CatalogException("Creating user and projects in a single transaction is forbidden"); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java index 17255ae3255..d0e102cebed 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.common; +import org.passay.CharacterData; import org.passay.*; import java.util.ArrayList; @@ -7,6 +8,11 @@ public class PasswordUtils { + public static final int MIN_STRONG_PASSWORD_LENGTH = 8; + public static final int DEFAULT_PASSWORD_LENGTH = 10; + public static final String PASSWORD_REQUIREMENT = "Password must contain at least " + MIN_STRONG_PASSWORD_LENGTH + + " characters, including at least one uppercase letter, one lowercase letter, one digit and one special character."; + private final static CharacterRule SPECIAL_CHARACTER_RULE = new CharacterRule(new CharacterData() { @Override public String getErrorCode() { @@ -20,7 +26,7 @@ public String getCharacters() { }); public static String getStrongRandomPassword() { - return getStrongRandomPassword(10); + return getStrongRandomPassword(DEFAULT_PASSWORD_LENGTH); } public static String getStrongRandomPassword(int length) { @@ -33,14 +39,10 @@ public static String getStrongRandomPassword(int length) { } public static boolean isStrongPassword(String password) { - return isStrongPassword(password, 8); - } - - public static boolean isStrongPassword(String password, int minLength) { List rules = new ArrayList<>(); //Rule 1: Password length should be in between - //minLength and 100 characters - rules.add(new LengthRule(minLength, 100)); + //MIN_STRONG_PASSWORD_LENGTH and 100 characters + rules.add(new LengthRule(MIN_STRONG_PASSWORD_LENGTH, 100)); //Rule 2: No whitespace allowed rules.add(new WhitespaceRule()); //Rule 3.a: At least one Upper-case character From 790750268aa1a8a48fbaefe129bde4d18ba35da1 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 18 Jul 2024 16:05:24 +0200 Subject: [PATCH 077/128] client: regenerate clients, #TASK-6568 --- .../app/cli/main/executors/OrganizationsCommandExecutor.java | 2 ++ .../opencga/app/cli/main/executors/StudiesCommandExecutor.java | 1 + .../app/cli/main/options/OrganizationsCommandOptions.java | 3 +++ .../opencga/app/cli/main/options/StudiesCommandOptions.java | 3 +++ opencga-client/src/main/R/R/Organization-methods.R | 3 ++- opencga-client/src/main/R/R/Study-methods.R | 3 ++- .../opencb/opencga/client/rest/clients/OrganizationClient.java | 1 + .../org/opencb/opencga/client/rest/clients/StudyClient.java | 1 + opencga-client/src/main/javascript/Organization.js | 2 ++ opencga-client/src/main/javascript/Study.js | 2 ++ .../main/python/pyopencga/rest_clients/organization_client.py | 2 ++ .../src/main/python/pyopencga/rest_clients/study_client.py | 2 ++ 12 files changed, 23 insertions(+), 2 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index c0ce6156d34..d5959ab09e3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -12,6 +12,7 @@ import org.opencb.opencga.app.cli.main.options.OrganizationsCommandOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.UpdateAction; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; @@ -214,6 +215,7 @@ private RestResponse updateNotes() throws Exception { ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("tagsAction", commandOptions.tagsAction); queryParams.putIfNotNull("includeResult", commandOptions.includeResult); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 2e8bd7b0040..e2e2180f4d2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -448,6 +448,7 @@ private RestResponse updateNotes() throws Exception { ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("tagsAction", commandOptions.tagsAction); queryParams.putIfNotNull("includeResult", commandOptions.includeResult); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java index 68be35b6cac..7b60a3059b2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java @@ -214,6 +214,9 @@ public class UpdateNotesCommandOptions { @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) public String id; + @Parameter(names = {"--tags-action"}, description = "Action to be performed if the array of tags is being updated.", required = false, arity = 1) + public String tagsAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java index 4e5cf838199..d75022de0b9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java @@ -505,6 +505,9 @@ public class UpdateNotesCommandOptions { @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) public String id; + @Parameter(names = {"--tags-action"}, description = "Action to be performed if the array of tags is being updated.", required = false, arity = 1) + public String tagsAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; diff --git a/opencga-client/src/main/R/R/Organization-methods.R b/opencga-client/src/main/R/R/Organization-methods.R index da241e16fb5..14e44864ed9 100644 --- a/opencga-client/src/main/R/R/Organization-methods.R +++ b/opencga-client/src/main/R/R/Organization-methods.R @@ -23,7 +23,7 @@ #' | createNotes | /{apiVersion}/organizations/notes/create | include, exclude, includeResult, body[*] | #' | searchNotes | /{apiVersion}/organizations/notes/search | include, exclude, creationDate, modificationDate, id, scope, visibility, uuid, userId, tags, version | #' | deleteNotes | /{apiVersion}/organizations/notes/{id}/delete | id[*], includeResult | -#' | updateNotes | /{apiVersion}/organizations/notes/{id}/update | include, exclude, id[*], includeResult, body[*] | +#' | updateNotes | /{apiVersion}/organizations/notes/{id}/update | include, exclude, id[*], tagsAction, includeResult, body[*] | #' | userUpdateStatus | /{apiVersion}/organizations/user/{user}/status/update | include, exclude, user[*], organization, includeResult, body[*] | #' | updateUser | /{apiVersion}/organizations/user/{user}/update | include, exclude, user[*], organization, includeResult, body[*] | #' | updateConfiguration | /{apiVersion}/organizations/{organization}/configuration/update | include, exclude, organization[*], includeResult, authenticationOriginsAction, body[*] | @@ -85,6 +85,7 @@ setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param id Note unique identifier. + #' @param tagsAction Action to be performed if the array of tags is being updated. Allowed values: ['ADD SET REMOVE'] #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing the Note fields to be updated. updateNotes=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="notes", diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index ddbc75ca374..ee3f4509354 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -31,7 +31,7 @@ #' | createNotes | /{apiVersion}/studies/{study}/notes/create | include, exclude, study[*], includeResult, body[*] | #' | searchNotes | /{apiVersion}/studies/{study}/notes/search | include, exclude, study[*], creationDate, modificationDate, id, uuid, userId, tags, visibility, version | #' | deleteNotes | /{apiVersion}/studies/{study}/notes/{id}/delete | study[*], id[*], includeResult | -#' | updateNotes | /{apiVersion}/studies/{study}/notes/{id}/update | include, exclude, study[*], id[*], includeResult, body[*] | +#' | updateNotes | /{apiVersion}/studies/{study}/notes/{id}/update | include, exclude, study[*], id[*], tagsAction, includeResult, body[*] | #' | permissionRules | /{apiVersion}/studies/{study}/permissionRules | study[*], entity[*] | #' | updatePermissionRules | /{apiVersion}/studies/{study}/permissionRules/update | study[*], entity[*], action, body[*] | #' | runTemplates | /{apiVersion}/studies/{study}/templates/run | study[*], jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | @@ -191,6 +191,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, group, id, members, stud #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Note unique identifier. + #' @param tagsAction Action to be performed if the array of tags is being updated. Allowed values: ['ADD SET REMOVE'] #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing the Note fields to be updated. updateNotes=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="notes", diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java index 67d55693b52..b5d5a3b7fa1 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java @@ -128,6 +128,7 @@ public RestResponse deleteNotes(String id, ObjectMap params) throws Client * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. + * tagsAction: Action to be performed if the array of tags is being updated. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index e5b054ae283..b1a8bce6959 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -288,6 +288,7 @@ public RestResponse deleteNotes(String study, String id, ObjectMap params) * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. + * tagsAction: Action to be performed if the array of tags is being updated. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/javascript/Organization.js b/opencga-client/src/main/javascript/Organization.js index 1580a4a7828..09b79429246 100644 --- a/opencga-client/src/main/javascript/Organization.js +++ b/opencga-client/src/main/javascript/Organization.js @@ -97,6 +97,8 @@ export default class Organization extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {"ADD SET REMOVE"} [params.tagsAction = "ADD"] - Action to be performed if the array of tags is being updated. The default + * value is ADD. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 9e013fef0dc..1fa1c7a1c1b 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -219,6 +219,8 @@ export default class Study extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {"ADD SET REMOVE"} [params.tagsAction = "ADD"] - Action to be performed if the array of tags is being updated. The default + * value is ADD. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py index 0c695ca0a0e..0d7dc0ff159 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py @@ -104,6 +104,8 @@ def update_notes(self, id, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. + :param str tags_action: Action to be performed if the array of tags is + being updated. Allowed values: ['ADD SET REMOVE'] :param bool include_result: Flag indicating to include the created or updated document result in the response. """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 93e70823d52..b730dc06d5d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -270,6 +270,8 @@ def update_notes(self, study, id, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. + :param str tags_action: Action to be performed if the array of tags is + being updated. Allowed values: ['ADD SET REMOVE'] :param bool include_result: Flag indicating to include the created or updated document result in the response. """ From ce0a8db2dcece7ee3e0dea0064a4fa78355ba6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 19 Jul 2024 09:29:10 +0100 Subject: [PATCH 078/128] analysis: Fix compilation issues. #TASK-5448 --- .../variant/manager/VariantOperationsTest.java | 4 ++-- opencga-client/src/main/R/R/Operation-methods.R | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index 3452693b855..ef6b79bd153 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -342,7 +342,7 @@ public void testSetup() throws Exception { .setFile(file.getId()) .setAnnotate(false) .setLoadHomRef(YesNoAuto.YES.name()), - Paths.get(opencga.createTmpOutdir("_index")), "index", token); + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); fail("Should have thrown an exception"); } catch (ToolException e) { MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); @@ -389,7 +389,7 @@ public void testSetup() throws Exception { new VariantIndexParams() .setFile(file.getId()) .setLoadHomRef(YesNoAuto.YES.name()), - Paths.get(opencga.createTmpOutdir("_index")), "index", token); + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject(PROJECT_FQN).getMetadataManager(); int studyId = metadataManager.getStudyId(study2fqn); diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 237c0d92d37..1c3fc8d7072 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -42,12 +42,12 @@ #' | variantSecondaryAnnotationIndex | /{apiVersion}/operation/variant/secondary/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | #' | variantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' | configureVariantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index/configure | study, skipRebuild, body | -#' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | -#' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, study, samples | +#' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | +#' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, samples | #' | setupVariant | /{apiVersion}/operation/variant/setup | study, body | -#' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | +#' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation From 9995ddb7f58f10ba3e4b25e11756d3f1bfaef354 Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 19 Jul 2024 11:05:10 +0200 Subject: [PATCH 079/128] app: improve migration, #TASK-5964 --- .../v3_2_0/TASK_5964/AddNewClinicalStatusValues.java | 12 +++++++----- .../ClinicalAnalysisStudyConfiguration.java | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java index 39e1447774d..775cf2c64d7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java @@ -74,8 +74,8 @@ protected void run() throws Exception { private Document fillStatusValues(String entity, String id, Document status, List statusValueList) { ClinicalStatus clinicalStatus = new ClinicalStatus(); - String clinicalId = status != null ? status.getString("id") : null; - if (status == null || StringUtils.isEmpty(clinicalId)) { + String clinicalStatusId = status != null ? status.getString("id") : null; + if (status == null || StringUtils.isEmpty(clinicalStatusId)) { logger.warn("Status is empty or does not contain 'id' field. Setting default status value for {} '{}'", entity, id); for (ClinicalStatusValue clinicalStatusValue : statusValueList) { if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { @@ -86,7 +86,7 @@ private Document fillStatusValues(String entity, String id, Document status, Lis } } else { for (ClinicalStatusValue clinicalStatusValue : statusValueList) { - if (clinicalStatusValue.getId().equals(clinicalId)) { + if (clinicalStatusValue.getId().equals(clinicalStatusId)) { clinicalStatus.setId(clinicalStatusValue.getId()); clinicalStatus.setDescription(clinicalStatusValue.getDescription()); clinicalStatus.setType(clinicalStatusValue.getType()); @@ -94,8 +94,10 @@ private Document fillStatusValues(String entity, String id, Document status, Lis } } if (clinicalStatus.getType() == null) { - logger.warn("Status '{}' not found in the list of available status values. Status type cannot be set for {} '{}'", - clinicalId, entity, id); + logger.warn("Status '{}' not found in the list of available status values. Keeping original status for {} '{}'", + clinicalStatusId, entity, id); + clinicalStatus.setId(clinicalStatusId); + clinicalStatus.setDescription(status.getString("description")); } } clinicalStatus.setDate(TimeUtils.getTime()); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java index 900bfd07d78..5f71fe0c6b9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java @@ -1,10 +1,11 @@ package org.opencb.opencga.core.models.study.configuration; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.common.FlagValue; import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; +import org.opencb.opencga.core.models.common.FlagValue; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; public class ClinicalAnalysisStudyConfiguration { @@ -32,7 +33,6 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { List clinicalStatusValueList = new ArrayList<>(4); List interpretationStatusList = new ArrayList<>(3); List priorities = new ArrayList<>(5); - Map> flags = new HashMap<>(); List clinicalConsentList = new ArrayList<>(); clinicalStatusValueList.add( From b006b5514dc8e7a6800339a77fae9082cb82c635 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 23 Jul 2024 11:49:10 +0200 Subject: [PATCH 080/128] catalog: archive old password, #TASK-6494 --- .../catalog/FixStatusIndexesMigration.java | 2 +- .../storage/AddAllelesColumnToPhoenix.java | 2 +- ...llegalConcurrentFileLoadingsMigration.java | 2 +- .../v3/v3_2_1/AddPasswordHistory.java | 42 ++ .../MoveUserAccountToInternalMigration.java | 47 ++ opencga-catalog/pom.xml | 5 + .../AzureADAuthenticationManager.java | 6 +- .../LDAPAuthenticationManager.java | 8 +- .../opencga/catalog/db/api/UserDBAdaptor.java | 14 +- .../OrganizationMongoDBAdaptorFactory.java | 2 +- .../db/mongodb/UserMongoDBAdaptor.java | 77 +- .../db/mongodb/converters/UserConverter.java | 15 +- .../CatalogAuthenticationException.java | 5 + .../catalog/managers/AbstractManager.java | 36 +- .../catalog/managers/AdminManager.java | 4 +- .../catalog/managers/CatalogManager.java | 3 +- .../opencga/catalog/managers/JobManager.java | 4 +- .../opencga/catalog/managers/NoteManager.java | 16 +- .../catalog/managers/OrganizationManager.java | 6 +- .../catalog/managers/ProjectManager.java | 2 +- .../opencga/catalog/managers/UserManager.java | 140 ++-- .../catalog/managers/AbstractManagerTest.java | 23 + .../catalog/managers/CatalogManagerTest.java | 558 +-------------- .../managers/ClinicalAnalysisManagerTest.java | 10 +- .../managers/OrganizationManagerTest.java | 2 +- .../catalog/managers/ProjectManagerTest.java | 3 +- .../catalog/managers/UserManagerTest.java | 664 ++++++++++++++++++ .../templates/TemplateManagerTest.java | 20 +- .../opencga/core/api/FieldConstants.java | 13 +- .../opencb/opencga/core/common/TimeUtils.java | 7 + .../core/config/AccountConfiguration.java | 42 ++ .../opencga/core/config/Configuration.java | 46 +- .../opencga/core/models/user/Account.java | 46 +- .../opencga/core/models/user/Password.java | 48 ++ .../opencb/opencga/core/models/user/User.java | 68 +- .../core/models/user/UserInternal.java | 30 +- pom.xml | 2 +- 37 files changed, 1270 insertions(+), 750 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v2}/v2_12_4/catalog/FixStatusIndexesMigration.java (97%) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v2}/v2_12_5/storage/AddAllelesColumnToPhoenix.java (95%) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v2}/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java (99%) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistory.java create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java create mode 100644 opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java similarity index 97% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java index ae73bcdc578..e1953673252 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_4.catalog; +package org.opencb.opencga.app.migrations.v2.v2_12_4.catalog; import org.bson.Document; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java similarity index 95% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java index 4efa260d965..93104000f20 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.opencga.app.migrations.StorageMigrationTool; import org.opencb.opencga.catalog.migration.Migration; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java similarity index 99% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java index 132e1357f1b..6d30f1a4de7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistory.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistory.java new file mode 100644 index 00000000000..d0fb6af2d12 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistory.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.lang3.StringUtils; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Migration(id = "add_archivePasswords_array", + description = "Add password history #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class AddPasswordHistory extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists("_archivePasswords", false); + Bson projection = Projections.include("_password"); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + // Add _archivePasswords + String currentPassword = document.getString("_password"); + List archivePasswords = new ArrayList<>(); + if (StringUtils.isNotEmpty(currentPassword)) { + archivePasswords.add(currentPassword); + } + updateDocument.getSet().put("_archivePasswords", archivePasswords); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java new file mode 100644 index 00000000000..64a824ec8ed --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java @@ -0,0 +1,47 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "move_user_account_to_internal", + description = "Move account to internal.account #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class MoveUserAccountToInternalMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists("account", true); + Bson projection = Projections.include("internal", "account"); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + Document account = document.get("account", Document.class); + Document internal = document.get("internal", Document.class); + internal.put("account", account); + + updateDocument.getSet().put("expirationDate", internal.get("lastModified")); + updateDocument.getSet().put("creationDate", account.get("creationDate")); + account.remove("creationDate"); + + Document password = new Document() + .append("expirationDate", null) + .append("lastChangedDate", internal.get("lastModified")); + account.put("password", password); + account.put("failedAttempts", internal.get("failedAttempts")); + internal.remove("failedAttempts"); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } +} diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 7457317446b..c55f55fadd5 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -61,6 +61,11 @@ + + org.mockito + mockito-core + test + org.apache.commons commons-collections4 diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java index f104f8fdf93..6a3a0480c28 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java @@ -39,6 +39,7 @@ import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationProvider; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.user.*; @@ -382,8 +383,9 @@ private List extractUserInformation(List(), attributes); userList.add(user); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java index 35fe615d161..f79eb85b19d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java @@ -198,9 +198,11 @@ public List getRemoteUserInformation(List userStringList) throws C Map attributes = new HashMap<>(); attributes.put("LDAP_RDN", rdn); - User user = new User(uid, displayName, mail, usersSearch, new Account() - .setAuthentication(new Account.AuthenticationOrigin(originId, false)), new UserInternal(new UserStatus()), - new UserQuota(-1, -1, -1, -1), new ArrayList<>(), new HashMap<>(), new LinkedList<>(), attributes); + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(originId, false)); + User user = new User(uid, displayName, mail, usersSearch, TimeUtils.getTime(), TimeUtils.getTime(), + new UserInternal(new UserStatus(), account), + new UserQuota(-1, -1, -1, -1), new HashMap<>(), new LinkedList<>(), attributes); userList.add(user); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index 3099262e917..bc25060061e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -118,14 +118,18 @@ enum QueryParams implements QueryParam { NAME("name", TEXT_ARRAY, ""), EMAIL("email", TEXT_ARRAY, ""), ORGANIZATION("organization", TEXT_ARRAY, ""), + CREATION_DATE("creationDate", TEXT_ARRAY, ""), + MODIFICATION_DATE("modificationDate", TEXT_ARRAY, ""), INTERNAL("internal", OBJECT, ""), - INTERNAL_FAILED_ATTEMPTS("internal.failedAttempts", INTEGER, ""), INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), - ACCOUNT("account", TEXT_ARRAY, ""), - ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), - ACCOUNT_CREATION_DATE("account.creationDate", TEXT, ""), - ACCOUNT_EXPIRATION_DATE("account.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT("internal.account", TEXT_ARRAY, ""), + INTERNAL_ACCOUNT_AUTHENTICATION_ID("internal.account.authentication.id", TEXT, ""), + INTERNAL_ACCOUNT_FAILED_ATTEMPTS("internal.account.failedAttempts", INTEGER, ""), + INTERNAL_ACCOUNT_CREATION_DATE("internal.account.creationDate", TEXT, ""), + INTERNAL_ACCOUNT_EXPIRATION_DATE("internal.account.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE("internal.account.password.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE("internal.account.password.lastChangeDate", TEXT, ""), QUOTA("quota", OBJECT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index b1c1c6704ec..fbc54430a22 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -119,7 +119,7 @@ public class OrganizationMongoDBAdaptorFactory { private final NoteMongoDBAdaptor notesDBAdaptor; private final OrganizationMongoDBAdaptor organizationDBAdaptor; - private final UserMongoDBAdaptor userDBAdaptor; + private UserMongoDBAdaptor userDBAdaptor; private final ProjectMongoDBAdaptor projectDBAdaptor; private final StudyMongoDBAdaptor studyDBAdaptor; private final IndividualMongoDBAdaptor individualDBAdaptor; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 19a9a6cd1f3..581ed6f1620 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -19,6 +19,7 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -54,6 +55,7 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.function.Consumer; @@ -72,6 +74,7 @@ public class UserMongoDBAdaptor extends CatalogMongoDBAdaptor implements UserDBA private UserConverter userConverter; private static final String PRIVATE_PASSWORD = "_password"; + private static final String ARCHIVE_PASSWORD = "_archivePasswords"; public UserMongoDBAdaptor(MongoDBCollection userCollection, MongoDBCollection deletedUserCollection, Configuration configuration, OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { @@ -118,6 +121,7 @@ private void insert(ClientSession clientSession, User user, String password) thr Document userDocument = userConverter.convertToStorageType(user); userDocument.append(ID, user.getId()); userDocument.append(PRIVATE_PASSWORD, encryptPassword(password)); + userDocument.append(ARCHIVE_PASSWORD, Collections.singletonList(encryptPassword(password))); userCollection.insert(clientSession, userDocument, null); } @@ -132,15 +136,7 @@ public OpenCGAResult get(String userId, QueryOptions options) @Override public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) throws CatalogDBException, CatalogAuthenticationException { - Document bson = new Document(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(oldPassword)); - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); - - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); - } - return new OpenCGAResult(result); + return setPassword(userId, oldPassword, newPassword); } @Override @@ -160,15 +156,54 @@ public void authenticate(String userId, String password) throws CatalogAuthentic @Override public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogDBException { - Query query = new Query(QueryParams.ID.key(), userId); - query.append(QueryParams.EMAIL.key(), email); - Bson bson = parseQuery(query); + return setPassword(userId, null, newPassword); + } + + public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, String newPassword) throws CatalogDBException { + List queryFilter = new ArrayList<>(); + queryFilter.add(Filters.eq(QueryParams.ID.key(), userId)); + queryFilter.add(Filters.ne(ARCHIVE_PASSWORD, encryptPassword(newPassword))); + if (StringUtils.isNotEmpty(oldPassword)) { + queryFilter.add(Filters.eq(PRIVATE_PASSWORD, encryptPassword(oldPassword))); + } + Bson query = Filters.and(queryFilter); - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); + UpdateDocument updateDocument = new UpdateDocument(); + String encryptedPassword = encryptPassword(newPassword); + updateDocument.getSet().put(PRIVATE_PASSWORD, encryptedPassword); + updateDocument.getPush().put(ARCHIVE_PASSWORD, encryptedPassword); + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE.key(), TimeUtils.getTime()); + if (configuration.getAccount().getPasswordExpirationDays() > 0) { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + String stringDate = TimeUtils.getTime(date); + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE.key(), stringDate); + } + Document update = updateDocument.toFinalUpdateDocument(); - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw new CatalogDBException("Bad user or email"); + logger.debug("Change password: query '{}'; update: '{}'", query.toBsonDocument(), update); + DataResult result = userCollection.update(query, update, null); + if (result.getNumUpdated() == 0) { + if (result.getNumMatches() == 0) { + Query userQuery = new Query(QueryParams.ID.key(), userId); + OpenCGAResult queryResult = nativeGet(userQuery, + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(PRIVATE_PASSWORD, ARCHIVE_PASSWORD))); + if (queryResult.getNumResults() == 0) { + throw new CatalogDBException("Could not update the password. User not found."); + } + Document userDocument = queryResult.first(); + if (StringUtils.isNotEmpty(oldPassword)) { + String dbPassword = userDocument.getString(PRIVATE_PASSWORD); + if (!encryptPassword(oldPassword).equals(dbPassword)) { + throw new CatalogDBException("Could not update the password. Please, verify that the current password is correct."); + } + } + List archivePassword = userDocument.getList(ARCHIVE_PASSWORD, String.class); + if (archivePassword.contains(encryptedPassword)) { + throw new CatalogDBException("Could not update the password. The new password has already been used. Please, use" + + " a different one."); + } + } + throw new CatalogDBException("Could not update the password. Please, verify that the current password is correct."); } return new OpenCGAResult(result); } @@ -409,7 +444,7 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog for (Document user : queryResult.getResults()) { ArrayList projects = (ArrayList) user.get("projects"); - if (projects.size() > 0) { + if (CollectionUtils.isNotEmpty(projects)) { List projectsTmp = new ArrayList<>(projects.size()); for (Document project : projects) { Query query1 = new Query(ProjectDBAdaptor.QueryParams.UID.key(), project.get(ProjectDBAdaptor @@ -429,7 +464,7 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException { UpdateDocument document = new UpdateDocument(); - final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), ACCOUNT_EXPIRATION_DATE.key()}; + final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), INTERNAL_ACCOUNT_EXPIRATION_DATE.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); if (parameters.containsKey(QueryParams.INTERNAL_STATUS_ID.key())) { @@ -437,7 +472,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer document.getSet().put(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()); } - final String[] acceptedIntParams = {INTERNAL_FAILED_ATTEMPTS.key()}; + final String[] acceptedIntParams = {INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key()}; filterIntParams(parameters, document.getSet(), acceptedIntParams); final String[] acceptedObjectParams = {QueryParams.QUOTA.key()}; @@ -670,8 +705,8 @@ private Bson parseQuery(Query query) throws CatalogDBException { case EMAIL: case ORGANIZATION: case INTERNAL_STATUS_DATE: - case ACCOUNT_AUTHENTICATION_ID: - case ACCOUNT_CREATION_DATE: + case INTERNAL_ACCOUNT_AUTHENTICATION_ID: + case INTERNAL_ACCOUNT_CREATION_DATE: case TOOL_ID: case TOOL_NAME: case TOOL_ALIAS: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java index 604a6073d0f..32420556a73 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java @@ -33,17 +33,12 @@ public UserConverter() { @Override public User convertToDataModelType(Document document) { - // TODO: Remove this piece of code once we are sure User contains the migrated new account type from 1.4.2 - Document account = (Document) document.get("account"); - if (account != null && account.get("authentication") == null) { - String authOrigin = account.getString("authOrigin"); - Document authentication = new Document() - .append("id", authOrigin) - .append("application", false); - account.put("authentication", authentication); + User user = super.convertToDataModelType(document); + // Add account to deprecated place + if (user.getInternal() != null && user.getInternal().getAccount() != null) { + user.setAccount(user.getInternal().getAccount()); } - - return super.convertToDataModelType(document); + return user; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java index 3853e91607f..8ac1b5c385f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java @@ -76,6 +76,11 @@ public static CatalogAuthenticationException accountIsExpired(String userId, Str + " talk to your organization owner/administrator."); } + public static CatalogAuthenticationException passwordExpired(String userId, String expirationDate) { + return new CatalogAuthenticationException("The password for the user account '" + userId + "' expired on " + expirationDate + + ". Please, reset your password or talk to your organization owner/administrator."); + } + public static CatalogAuthenticationException userNotAllowed(String domain) { return new CatalogAuthenticationException(domain + ": User not allowed to access the system."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java index 3d3611c5414..da7566340ac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java @@ -73,68 +73,72 @@ public abstract class AbstractManager { logger = LoggerFactory.getLogger(this.getClass()); } + protected DBAdaptorFactory getCatalogDBAdaptorFactory() { + return catalogDBAdaptorFactory; + } + protected MigrationDBAdaptor getMigrationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getMigrationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getMigrationDBAdaptor(organization); } protected MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogMetaDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogMetaDBAdaptor(organization); } protected OrganizationDBAdaptor getOrganizationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogOrganizationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogOrganizationDBAdaptor(organization); } protected UserDBAdaptor getUserDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogUserDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogUserDBAdaptor(organization); } protected ProjectDBAdaptor getProjectDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogProjectDbAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogProjectDbAdaptor(organization); } protected StudyDBAdaptor getStudyDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogStudyDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogStudyDBAdaptor(organization); } protected FileDBAdaptor getFileDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogFileDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogFileDBAdaptor(organization); } protected SampleDBAdaptor getSampleDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogSampleDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogSampleDBAdaptor(organization); } protected IndividualDBAdaptor getIndividualDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogIndividualDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogIndividualDBAdaptor(organization); } protected JobDBAdaptor getJobDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organization); } protected AuditDBAdaptor getAuditDbAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogAuditDbAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogAuditDbAdaptor(organization); } protected CohortDBAdaptor getCohortDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogCohortDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogCohortDBAdaptor(organization); } protected PanelDBAdaptor getPanelDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogPanelDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogPanelDBAdaptor(organization); } protected FamilyDBAdaptor getFamilyDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogFamilyDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogFamilyDBAdaptor(organization); } protected ClinicalAnalysisDBAdaptor getClinicalAnalysisDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getClinicalAnalysisDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getClinicalAnalysisDBAdaptor(organization); } protected InterpretationDBAdaptor getInterpretationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getInterpretationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getInterpretationDBAdaptor(organization); } protected void fixQueryObject(Query query) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java index 09bc57aabf1..7edf1030496 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java @@ -67,7 +67,7 @@ public OpenCGAResult userSearch(String organizationId, Query query, QueryO query.remove(ParamConstants.USER); } if (query.containsKey(ParamConstants.USER_CREATION_DATE)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); query.remove(ParamConstants.USER_CREATION_DATE); } @@ -134,7 +134,7 @@ public List getOrganizationIds(String token) throws CatalogException { JwtPayload payload = catalogManager.getUserManager().validateToken(token); try { authorizationManager.checkIsOpencgaAdministrator(payload); - List organizationIds = catalogDBAdaptorFactory.getOrganizationIds(); + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); auditManager.audit(ParamConstants.ADMIN_ORGANIZATION, payload.getUserId(), Enums.Action.FETCH_ORGANIZATION_IDS, Enums.Resource.STUDY, "", "", "", "", new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 1409b1e9d4c..030505395d0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -48,7 +48,6 @@ import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; @@ -278,7 +277,7 @@ private void privateInstall(String algorithm, String secretKey, String password, organizationConfiguration, null), QueryOptions.empty(), null); - User user = new User(OPENCGA, new Account().setExpirationDate("")) + User user = new User(OPENCGA) .setEmail(StringUtils.isEmpty(email) ? "opencga@admin.com" : email) .setOrganization(ADMIN_ORGANIZATION); userManager.create(user, password, null); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index b8b9ba55bae..fe7ffc18e44 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -421,7 +421,7 @@ public OpenCGAResult kill(String studyStr, String jobId, String token) thro } ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_KILL_JOB_REQUESTED.key(), true); - OpenCGAResult update = catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, QueryOptions.empty()); auditManager.audit(organizationId, userId, Enums.Action.KILL_JOB, Enums.Resource.JOB, jobId, jobUuid, study.getId(), @@ -816,7 +816,7 @@ public DBIterator iterator(String studyId, Query query, QueryOptions option public OpenCGAResult countInOrganization(String organizationId, Query query, String token) throws CatalogException { JwtPayload jwtPayload = userManager.validateToken(token); authorizationManager.checkIsOpencgaAdministrator(jwtPayload); - return catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organizationId).count(query); + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).count(query); } @Override diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java index b3c7139bd30..4628126dc51 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java @@ -54,7 +54,7 @@ private OpenCGAResult internalGet(String organizationId, long studyUid, St } else { query.put(NoteDBAdaptor.QueryParams.ID.key(), noteId); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(query, QueryOptions.empty()); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, QueryOptions.empty()); if (result.getNumResults() == 0) { throw CatalogException.notFound("note", Collections.singletonList(noteId)); } @@ -88,7 +88,7 @@ public OpenCGAResult searchOrganizationNote(Query query, QueryOptions opti queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; @@ -131,7 +131,7 @@ public OpenCGAResult searchStudyNote(String studyStr, Query query, QueryOp queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; @@ -209,11 +209,11 @@ public OpenCGAResult createStudyNote(String studyStr, NoteCreateParams not private OpenCGAResult create(Note note, QueryOptions options, JwtPayload tokenPayload) throws CatalogException { String organizationId = tokenPayload.getOrganization(); validateNewNote(note, tokenPayload.getUserId()); - OpenCGAResult insert = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).insert(note); + OpenCGAResult insert = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).insert(note); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created note Query query = new Query(NoteDBAdaptor.QueryParams.UID.key(), note.getUid()); - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(query, options); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } return insert; @@ -302,11 +302,11 @@ private OpenCGAResult update(long noteUid, NoteUpdateParams noteUpdatePara // Write who's performing the update updateMap.put(NoteDBAdaptor.QueryParams.USER_ID.key(), tokenPayload.getUserId()); - OpenCGAResult update = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, QueryOptions.empty()); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated note - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); update.setResults(result.getResults()); } return update; @@ -379,7 +379,7 @@ public OpenCGAResult deleteStudyNote(String studyStr, String noteId, Query } private OpenCGAResult delete(Note note, QueryOptions options, JwtPayload jwtPayload) throws CatalogException { - OpenCGAResult delete = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(jwtPayload.getOrganization()).delete(note); + OpenCGAResult delete = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(jwtPayload.getOrganization()).delete(note); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { delete.setResults(Collections.singletonList(note)); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java index 90b83a92728..2612ea7f633 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java @@ -185,7 +185,7 @@ public OpenCGAResult create(OrganizationCreateParams organizationC organization = organizationCreateParams.toOrganization(); validateOrganizationForCreation(organization, userId); - queryResult = catalogDBAdaptorFactory.createOrganization(organization, options, userId); + queryResult = getCatalogDBAdaptorFactory().createOrganization(organization, options, userId); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { OpenCGAResult result = getOrganizationDBAdaptor(organization.getId()).get(options); organization = result.first(); @@ -211,7 +211,7 @@ public OpenCGAResult create(OrganizationCreateParams organizationC auditManager.auditCreate(ParamConstants.ADMIN_ORGANIZATION, userId, Enums.Resource.ORGANIZATION, organization.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); try { - catalogDBAdaptorFactory.deleteOrganization(organization); + getCatalogDBAdaptorFactory().deleteOrganization(organization); } catch (Exception e1) { logger.error("Error deleting organization from catalog after failing creating the folder in the filesystem", e1); throw e; @@ -592,7 +592,7 @@ Set getOrganizationOwnerAndAdmins(String organizationId) throws CatalogE public List getOrganizationIds(String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); authorizationManager.checkIsOpencgaAdministrator(tokenPayload, "get all organization ids"); - return catalogDBAdaptorFactory.getOrganizationIds(); + return getCatalogDBAdaptorFactory().getOrganizationIds(); } private void privatizeResults(OpenCGAResult result) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index beda16bfa2e..dcfecf341f6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -562,7 +562,7 @@ public void importReleases(String organizationId, String owner, String inputDirS } OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(owner, new QueryOptions(QueryOptions.INCLUDE, - Collections.singletonList(UserDBAdaptor.QueryParams.ACCOUNT.key()))); + Collections.singletonList(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT.key()))); if (userDataResult.getNumResults() == 0) { throw new CatalogException("User " + owner + " not found"); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index 35c8bd46b64..9fa16ae9d64 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -36,10 +36,12 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupUpdateParams; @@ -60,8 +62,8 @@ */ public class UserManager extends AbstractManager { - static final QueryOptions INCLUDE_ACCOUNT_AND_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.ACCOUNT.key(), UserDBAdaptor.QueryParams.INTERNAL.key())); + static final QueryOptions INCLUDE_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( + UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.INTERNAL.key())); protected static Logger logger = LoggerFactory.getLogger(UserManager.class); private final CatalogIOManager catalogIOManager; private final AuthenticationFactory authenticationFactory; @@ -130,36 +132,54 @@ public OpenCGAResult create(User user, String password, String token) thro // Initialise fields ParamUtils.checkObj(user, "User"); ParamUtils.checkValidUserId(user.getId()); - ParamUtils.checkParameter(user.getName(), "name"); + user.setName(ParamUtils.defaultString(user.getName(), user.getId())); user.setEmail(ParamUtils.defaultString(user.getEmail(), "")); if (StringUtils.isNotEmpty(user.getEmail())) { checkEmail(user.getEmail()); } - user.setAccount(ParamUtils.defaultObject(user.getAccount(), Account::new)); - user.getAccount().setCreationDate(TimeUtils.getTime()); - if (StringUtils.isEmpty(user.getAccount().getExpirationDate())) { - // By default, user accounts will be valid for 1 year when they are created. - user.getAccount().setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); - Date date = TimeUtils.add1YeartoDate(new Date()); - user.getAccount().setExpirationDate(TimeUtils.getTime(date)); - } else { - // Validate expiration date is not over - ParamUtils.checkDateIsNotExpired(user.getAccount().getExpirationDate(), "account.expirationDate"); - } - user.setInternal(new UserInternal(new UserStatus(UserStatus.READY))); + user.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(user.getCreationDate(), + UserDBAdaptor.QueryParams.CREATION_DATE.key())); + user.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(user.getModificationDate(), + UserDBAdaptor.QueryParams.MODIFICATION_DATE.key())); + + user.setInternal(ParamUtils.defaultObject(user.getInternal(), UserInternal::new)); + user.getInternal().setStatus(new UserStatus(InternalStatus.READY)); user.setQuota(ParamUtils.defaultObject(user.getQuota(), UserQuota::new)); user.setProjects(ParamUtils.defaultObject(user.getProjects(), Collections::emptyList)); user.setConfigs(ParamUtils.defaultObject(user.getConfigs(), HashMap::new)); user.setFilters(ParamUtils.defaultObject(user.getFilters(), LinkedList::new)); user.setAttributes(ParamUtils.defaultObject(user.getAttributes(), Collections::emptyMap)); + // Init account + user.getInternal().setAccount(ParamUtils.defaultObject(user.getInternal().getAccount(), Account::new)); + Account account = user.getInternal().getAccount(); + account.setPassword(ParamUtils.defaultObject(account.getPassword(), Password::new)); + if (StringUtils.isEmpty(account.getExpirationDate())) { + account.setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); + } else { + // Validate expiration date is not over + ParamUtils.checkDateIsNotExpired(account.getExpirationDate(), UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key()); + } if (StringUtils.isEmpty(password)) { Map authOriginMap = authenticationFactory.getOrganizationAuthenticationManagers(organizationId); - if (!authOriginMap.containsKey(user.getAccount().getAuthentication().getId())) { - throw new CatalogException("Unknown authentication origin id '" + user.getAccount().getAuthentication() + "'"); + if (!authOriginMap.containsKey(account.getAuthentication().getId())) { + throw new CatalogException("Unknown authentication origin id '" + account.getAuthentication() + "'"); } } else { - user.getAccount().setAuthentication(new Account.AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, false)); + account.setAuthentication(new Account.AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, false)); + } + + // Set password expiration + if (AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId())) { + account.getPassword().setLastChangeDate(TimeUtils.getTime()); + } + if (!AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId()) + || configuration.getAccount().getPasswordExpirationDays() <= 0) { + // User password doesn't expire or it's not managed by OpenCGA + account.getPassword().setExpirationDate(null); + } else { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + account.getPassword().setExpirationDate(TimeUtils.getTime(date)); } if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) || !OPENCGA.equals(user.getId())) { @@ -213,7 +233,6 @@ public OpenCGAResult create(User user, String password, String token) thro public OpenCGAResult create(String id, String name, String email, String password, String organization, Long quota, String token) throws CatalogException { User user = new User(id, name, email, organization, new UserInternal(new UserStatus())) - .setAccount(new Account("", "", null)) .setQuota(new UserQuota().setMaxDisk(quota != null ? quota : -1)); return create(user, password, token); } @@ -244,7 +263,8 @@ public OpenCGAResult search(@Nullable String organizationId, Query query, // Fix query params if (query.containsKey(ParamConstants.USER_AUTHENTICATION_ORIGIN)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_AUTHENTICATION_ID.key(), query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), + query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); query.remove(ParamConstants.USER_AUTHENTICATION_ORIGIN); } @@ -269,11 +289,12 @@ public JwtPayload validateToken(String token) throws CatalogException { authOrigin = CatalogAuthenticationManager.OPENCGA; } else { OpenCGAResult userResult = getUserDBAdaptor(jwtPayload.getOrganization()).get(jwtPayload.getUserId(), - INCLUDE_ACCOUNT_AND_INTERNAL); + INCLUDE_INTERNAL); if (userResult.getNumResults() == 0) { throw new CatalogException("User '" + jwtPayload.getUserId() + "' could not be found."); } - authOrigin = userResult.first().getAccount().getAuthentication().getId(); + User user = userResult.first(); + authOrigin = user.getInternal().getAccount().getAuthentication().getId(); } authenticationFactory.validateToken(jwtPayload.getOrganization(), authOrigin, token); @@ -478,8 +499,10 @@ public void importRemoteEntities(String organizationId, String authOrigin, List< } } else { for (String applicationId : idList) { - User application = new User(applicationId, new Account() - .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true))) + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true)); + User application = new User(applicationId) + .setInternal(new UserInternal(new UserStatus(UserStatus.READY), account)) .setEmail("mail@mail.co.uk"); application.setOrganization(organizationId); create(application, null, token); @@ -785,7 +808,7 @@ public AuthenticationResponse login(String organizationId, String username, Stri if (OPENCGA.equals(username)) { organizationId = ParamConstants.ADMIN_ORGANIZATION; } else { - List organizationIds = catalogDBAdaptorFactory.getOrganizationIds(); + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); if (organizationIds.size() == 2) { organizationId = organizationIds.stream().filter(s -> !ParamConstants.ADMIN_ORGANIZATION.equals(s)).findFirst().get(); } else { @@ -794,38 +817,56 @@ public AuthenticationResponse login(String organizationId, String username, Stri } } - OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_ACCOUNT_AND_INTERNAL); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { User user = userOpenCGAResult.first(); // Only local OPENCGA users that are not superadmins can be automatically banned or their accounts be expired boolean userCanBeBanned = !ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) - && CatalogAuthenticationManager.OPENCGA.equals(user.getAccount().getAuthentication().getId()); + && CatalogAuthenticationManager.OPENCGA.equals(user.getInternal().getAccount().getAuthentication().getId()); // We check if (userCanBeBanned) { // Check user is not banned, suspended or has an expired account if (UserStatus.BANNED.equals(user.getInternal().getStatus().getId())) { throw CatalogAuthenticationException.userIsBanned(username); } - Date date = TimeUtils.toDate(user.getAccount().getExpirationDate()); - if (date == null) { - throw new CatalogException("Unexpected null expiration date for user '" + username + "'."); + if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { + throw CatalogAuthenticationException.userIsSuspended(username); } - if (date.before(new Date())) { - throw CatalogAuthenticationException.accountIsExpired(username, user.getAccount().getExpirationDate()); + Account account2 = user.getInternal().getAccount(); + if (account2.getPassword().getExpirationDate() != null) { + Account account1 = user.getInternal().getAccount(); + Date passwordExpirationDate = TimeUtils.toDate(account1.getPassword().getExpirationDate()); + if (passwordExpirationDate == null) { + throw new CatalogException("Unexpected null 'passwordExpirationDate' for user '" + username + "'."); + } + if (passwordExpirationDate.before(new Date())) { + Account account = user.getInternal().getAccount(); + throw CatalogAuthenticationException.passwordExpired(username, account.getPassword().getExpirationDate()); + } + } + if (user.getInternal().getAccount().getExpirationDate() != null) { + Date date = TimeUtils.toDate(user.getInternal().getAccount().getExpirationDate()); + if (date == null) { + throw new CatalogException("Unexpected null 'expirationDate' for user '" + username + "'."); + } + if (date.before(new Date())) { + throw CatalogAuthenticationException.accountIsExpired(username, + user.getInternal().getAccount().getExpirationDate()); + } } } - if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { - throw CatalogAuthenticationException.userIsSuspended(username); - } - authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); + User user1 = userOpenCGAResult.first(); + authId = user1.getInternal().getAccount().getAuthentication().getId(); try { response = authenticationFactory.authenticate(organizationId, authId, username, password); } catch (CatalogAuthenticationException e) { if (userCanBeBanned) { // We can only lock the account if it is not the root user - int failedAttempts = userOpenCGAResult.first().getInternal().getFailedAttempts(); - ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), failedAttempts + 1); - if (failedAttempts >= (configuration.getMaxLoginAttempts() - 1)) { + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + int failedAttempts = userInternal.getAccount().getFailedAttempts(); + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), + failedAttempts + 1); + if (failedAttempts >= (configuration.getAccount().getMaxLoginAttempts() - 1)) { // Ban the account updateParams.append(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), UserStatus.BANNED); } @@ -838,10 +879,13 @@ public AuthenticationResponse login(String organizationId, String username, Stri } // If it was a local user and the counter of failed attempts was greater than 0, we reset it - if (userCanBeBanned && userOpenCGAResult.first().getInternal().getFailedAttempts() > 0) { - // Reset login failed attempts counter - ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); - getUserDBAdaptor(organizationId).update(username, updateParams); + if (userCanBeBanned) { + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + if (userInternal.getAccount().getFailedAttempts() > 0) { + // Reset login failed attempts counter + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); + getUserDBAdaptor(organizationId).update(username, updateParams); + } } } else { // We attempt to login the user with the different authentication managers @@ -992,7 +1036,7 @@ public OpenCGAResult changeStatus(String organizationId, String userId, St // Update user status and reset failed attempts to 0 ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), status); if (UserStatus.READY.equals(status)) { - updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); + updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); } OpenCGAResult result = getUserDBAdaptor(userIdOrganization).update(userId, updateParams); @@ -1059,9 +1103,10 @@ public String getNonExpiringToken(String organizationId, String userId, Map userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_ACCOUNT_AND_INTERNAL); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { - String authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); + User user1 = userOpenCGAResult.first(); + String authId = user1.getInternal().getAccount().getAuthentication().getId(); return authenticationFactory.getOrganizationAuthenticationManager(organizationId, authId); } else { throw new CatalogException("User '" + user + "' not found."); @@ -1492,7 +1537,8 @@ private String getAuthenticationOriginId(String organizationId, String userId) t if (user == null || user.getNumResults() == 0) { throw new CatalogException(userId + " user not found"); } - return user.first().getAccount().getAuthentication().getId(); + User user1 = user.first(); + return user1.getInternal().getAccount().getAuthentication().getId(); } /** diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java index 9044dd2f7c0..4bafb0cc4e0 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java @@ -23,13 +23,17 @@ import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; import org.junit.rules.TestName; +import org.mockito.Mockito; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -474,4 +478,23 @@ private void createDummyData(CatalogManager catalogManager) throws CatalogExcept } + protected CatalogManager mockCatalogManager() throws CatalogDBException { + CatalogManager spy = Mockito.spy(catalogManager); + UserManager userManager = spy.getUserManager(); + UserManager userManagerSpy = Mockito.spy(userManager); + Mockito.doReturn(userManagerSpy).when(spy).getUserManager(); + MongoDBAdaptorFactory mongoDBAdaptorFactory = mockMongoDBAdaptorFactory(); + Mockito.doReturn(mongoDBAdaptorFactory).when(userManagerSpy).getCatalogDBAdaptorFactory(); + return spy; + } + + protected MongoDBAdaptorFactory mockMongoDBAdaptorFactory() throws CatalogDBException { + MongoDBAdaptorFactory catalogDBAdaptorFactory = (MongoDBAdaptorFactory) catalogManager.getUserManager().getCatalogDBAdaptorFactory(); + MongoDBAdaptorFactory dbAdaptorFactorySpy = Mockito.spy(catalogDBAdaptorFactory); + UserDBAdaptor userDBAdaptor = dbAdaptorFactorySpy.getCatalogUserDBAdaptor(organizationId); + UserDBAdaptor userDBAdaptorSpy = Mockito.spy(userDBAdaptor); + Mockito.doReturn(userDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogUserDBAdaptor(organizationId); + return dbAdaptorFactorySpy; + } + } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java index 73cab79045b..8f0bce1401e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java @@ -16,13 +16,10 @@ package org.opencb.opencga.catalog.managers; -import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.common.Status; @@ -34,7 +31,10 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -43,7 +43,6 @@ import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; -import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; import org.opencb.opencga.core.models.common.AnnotationSet; @@ -61,11 +60,10 @@ import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.study.*; -import org.opencb.opencga.core.models.user.*; +import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import javax.naming.NamingException; import java.io.IOException; import java.util.*; import java.util.concurrent.ExecutorService; @@ -73,22 +71,12 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; -import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; @Category(MediumTests.class) public class CatalogManagerTest extends AbstractManagerTest { - @Test - public void createOpencgaUserTest() throws CatalogException { - thrown.expect(CatalogException.class); - thrown.expectMessage("forbidden"); - catalogManager.getUserManager().create(new User().setId(ParamConstants.OPENCGA_USER_ID).setName(orgOwnerUserId) - .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - } - @Test public void createStudyFailMoreThanOneProject() throws CatalogException { catalogManager.getProjectManager().incrementRelease(project1, ownerToken); @@ -102,443 +90,6 @@ public void createStudyFailMoreThanOneProject() throws CatalogException { null, null, null, null, ownerToken); } - @Test - public void testAdminUserExists() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - JwtPayload payload = catalogManager.getUserManager().validateToken(token); - assertEquals(ParamConstants.OPENCGA_USER_ID, payload.getUserId()); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, payload.getOrganization()); - } - - @Test - public void searchUsersTest() throws CatalogException { - OpenCGAResult search = catalogManager.getUserManager().search(organizationId, new Query(), QueryOptions.empty(), opencgaToken); - assertEquals(8, search.getNumResults()); - for (User user : search.getResults()) { - if (noAccessUserId1.equals(user.getId())) { - assertEquals(0, user.getProjects().size()); - } else if (user.getId().startsWith("normalUser")) { - assertEquals(1, user.getProjects().size()); - } else { - assertEquals(2, user.getProjects().size()); - } - } - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), ownerToken); - assertEquals(8, search.getNumResults()); - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken2); - assertEquals(8, search.getNumResults()); - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken1); - assertEquals(8, search.getNumResults()); - - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), - QueryOptions.empty(), studyAdminToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), - QueryOptions.empty(), normalToken1)); - } - - @Test - public void testGetToken() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - Map claims = new HashMap<>(); - claims.put("a", "hola"); - claims.put("ab", "byw"); - // Create a token valid for 1 second - String expiringToken = catalogManager.getUserManager().getToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, 1L, token); - assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); - - String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, token); - assertEquals("opencga", catalogManager.getUserManager().validateToken(nonExpiringToken).getUserId()); - - Thread.sleep(1000); - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage("expired"); - assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); - } - - @Test - public void loginWithoutOrganizationId() throws CatalogException { - String token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - JwtPayload jwtPayload = new JwtPayload(token); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); - - token = catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - jwtPayload = new JwtPayload(token); - assertEquals(organizationId, jwtPayload.getOrganization()); - - // Create a third organization - catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("other").setName("Test"), QueryOptions.empty(), opencgaToken); - token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - jwtPayload = new JwtPayload(token); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); - - thrown.expect(CatalogParameterException.class); - thrown.expectMessage("organization"); - catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD); - } - - @Test - public void testCreateExistingUser() throws Exception { - thrown.expect(CatalogException.class); - thrown.expectMessage(containsString("already exists")); - catalogManager.getUserManager().create(orgOwnerUserId, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, - null, opencgaToken); - } - - @Test - public void testCreateAnonymousUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, - organizationId, null, opencgaToken); - } - - @Test - public void testCreateRegisteredUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, - opencgaToken); - } - - @Test - public void testLogin() throws Exception { - catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD); - - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); - catalogManager.getUserManager().login(organizationId, normalUserId1, "fakePassword"); - } - - @Test - public void refreshTokenTest() throws Exception { - String refreshToken = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getRefreshToken(); - AuthenticationResponse authenticationResponse = catalogManager.getUserManager().refreshToken(refreshToken); - assertNotNull(authenticationResponse); - assertNotNull(authenticationResponse.getToken()); - } - - @Test - public void anonymousUserLoginTest() throws CatalogException { - AuthenticationResponse authResponse = catalogManager.getUserManager().loginAnonymous(organizationId); - assertNotNull(authResponse.getToken()); - - String org2 = "otherOrg"; - catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); - catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); - String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); - Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() - .setId("project") - .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), - INCLUDE_RESULT, owner2Token).first(); - Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); - - try { - catalogManager.getUserManager().loginAnonymous(org2); - fail("Anonymous user should not get a token for that organization as it has not been granted any kind of access"); - } catch (Exception e) { - assertEquals(CatalogAuthenticationException.class, e.getClass()); - assertTrue(e.getMessage().contains("not found")); - } - - catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), owner2Token); - authResponse = catalogManager.getUserManager().loginAnonymous(org2); - assertNotNull(authResponse.getToken()); - - - catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Collections.singletonList("*")), owner2Token); - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().loginAnonymous(org2); - } - - @Test - public void incrementLoginAttemptsTest() throws CatalogException { - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(1, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - - for (int i = 2; i < 5; i++) { - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(i, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - } - - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(5, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); - - CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - assertTrue(incorrect.getMessage().contains("banned")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(5, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); - - CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); - assertTrue(authException.getMessage().contains("banned")); - - // Remove ban from user - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(0, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - - String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); - assertNotNull(token); - } - - @Test - public void changeUserStatusTest() throws CatalogException { - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); - assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); - - catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); - CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); - assertTrue(authException.getMessage().contains("own account")); - - authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); - assertTrue(authException.getMessage().contains("suspend administrators")); - - CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); - assertTrue(incorrect.getMessage().contains("suspended")); - - catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); - String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); - assertNotNull(token); - - CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); - assertTrue(paramException.getMessage().contains("Invalid status")); - - CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); - assertTrue(dbException.getMessage().contains("not exist")); - } - - @Test - public void loginExpiredAccountTest() throws CatalogException { - // Expire account of normalUserId1 - ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); - catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); - - CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); - assertTrue(authException.getMessage().contains("expired")); - - // Ensure it doesn't matter whether opencga account is expired or not - catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); - String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertNotNull(token); - } - - @Test - public void updateUserTest() throws JsonProcessingException, CatalogException { - UserUpdateParams userUpdateParams = new UserUpdateParams() - .setName("newName") - .setEmail("mail@mail.com"); - ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); - User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); - assertEquals(userUpdateParams.getName(), user.getName()); - assertEquals(userUpdateParams.getEmail(), user.getEmail()); - - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); - - userUpdateParams = new UserUpdateParams() - .setEmail("notAnEmail"); - ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); - assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); - } - - @Test - public void testGetUserInfo() throws CatalogException { - // OpenCGA administrator - DataResult user = catalogManager.getUserManager().get(organizationId, - Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), opencgaToken); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - // Organization owner - user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), - new QueryOptions(), ownerToken); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - // Organization administrator - user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), - new QueryOptions(), orgAdminToken1); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - thrown.expect(CatalogAuthorizationException.class); - thrown.expectMessage("organization"); - catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), - studyAdminToken1); - } - - @Test - public void testGetProjectsFromUserInfo() throws CatalogException { - String userId = organizationId; - catalogManager.getUserManager().create(userId, "test", "mail@mail.com", TestParamConstants.PASSWORD, organizationId, null, - opencgaToken); - catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("test")), ownerToken); - String token = catalogManager.getUserManager().login(organizationId, userId, TestParamConstants.PASSWORD).getToken(); - - DataResult user = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(), token); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, normalUserId3, new QueryOptions(), normalToken3); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(), ownerToken); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, new QueryOptions(), studyAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - - user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), normalToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, normalUserId3, new QueryOptions(), normalToken3); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, orgOwnerUserId, new QueryOptions(), ownerToken); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, orgAdminUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, studyAdminUserId1, new QueryOptions(), studyAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - } - - @Test - public void testModifyUser() throws CatalogException, InterruptedException, IOException { - ObjectMap params = new ObjectMap(); - String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); - String newPassword = RandomStringUtils.randomAlphanumeric(10); - String newEmail = "new@email.ac.uk"; - - params.put("name", newName); - - Thread.sleep(10); - - catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); - catalogManager.getUserManager().update(orgOwnerUserId, new ObjectMap("email", newEmail), null, ownerToken); - catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD, newPassword); - - List userList = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(QueryOptions - .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), - UserDBAdaptor.QueryParams.ATTRIBUTES.key())), ownerToken).getResults(); - User userPost = userList.get(0); - System.out.println("userPost = " + userPost); - assertEquals(userPost.getName(), newName); - assertEquals(userPost.getEmail(), newEmail); - - catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); - catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD); - catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD); - - try { - params = new ObjectMap(); - params.put("password", "1234321"); - catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); - fail("Expected exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - - try { - catalogManager.getUserManager().update(orgOwnerUserId, params, null, orgAdminToken1); - fail("Expected exception"); - } catch (CatalogException e) { - System.out.println(e); - } - } - - @Test - public void testUpdateUserConfig() throws CatalogException { - Map map = new HashMap<>(); - map.put("key1", "value1"); - map.put("key2", "value2"); - catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); - - Map config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); - assertEquals(2, config.size()); - assertEquals("value1", config.get("key1")); - assertEquals("value2", config.get("key2")); - - map = new HashMap<>(); - map.put("key2", "value3"); - catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); - config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); - assertEquals(1, config.size()); - assertEquals("value3", config.get("key2")); - - catalogManager.getUserManager().deleteConfig(normalUserId1, "a", normalToken1); - - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1); - } - - private String getAdminToken() throws CatalogException, IOException { - return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - } - - @Test - public void createUserUsingMailAsId() throws CatalogException { - catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); - AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); - assertNotNull(login); - User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); - assertEquals("hello.mail@mymail.org", user.getId()); - } - @Test public void getGroupsTest() throws CatalogException { Group group = new Group("groupId", Arrays.asList(normalUserId2, normalUserId3)).setSyncedFrom(new Group.Sync("ldap", "bio")); @@ -564,82 +115,6 @@ public void getGroupsTest() throws CatalogException { catalogManager.getStudyManager().getCustomGroups(studyFqn, group.getId(), normalToken2); } - @Ignore - @Test - public void importLdapUsers() throws CatalogException, NamingException, IOException { - // Action only for admins - catalogManager.getUserManager().importRemoteEntities(organizationId, "ldap", Arrays.asList("pfurio", "imedina"), false, null, null, - getAdminToken()); - // TODO: Validate the users have been imported - } - - // To make this test work we will need to add a correct user and password to be able to login - @Ignore - @Test - public void loginNotRegisteredUsers() throws CatalogException { - // Action only for admins - Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); - catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); - catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", "view_only"), - ParamUtils.AclAction.SET, ownerToken); - String token = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, "password").getToken(); - - assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - - // We remove the permissions for group ldap - catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", ""), - ParamUtils.AclAction.RESET, this.ownerToken); - - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - } - - @Ignore - @Test - public void syncUsers() throws CatalogException { - // Action only for admins - String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", "bio", "bio", studyFqn, true, token); - DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); - - assertEquals(1, bio.getNumResults()); - assertEquals(0, bio.first().getUserIds().size()); - - catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, studyFqn, "ldap", token); - bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); - - assertEquals(1, bio.getNumResults()); - assertTrue(!bio.first().getUserIds().isEmpty()); - } - - @Ignore - @Test - public void importLdapGroups() throws CatalogException, IOException { - // Action only for admins - String remoteGroup = "bio"; - String internalGroup = "test"; - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); - - DataResult test = catalogManager.getStudyManager().getGroup(studyFqn, "test", ownerToken); - assertEquals(1, test.getNumResults()); - assertEquals("@test", test.first().getId()); - assertTrue(test.first().getUserIds().size() > 0); - -// internalGroup = "test1"; -// try { -// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); -// fail("Should not be possible creating another group containing the same users that belong to a different group"); -// } catch (CatalogException e) { -// System.out.println(e.getMessage()); -// } - - remoteGroup = "bioo"; - internalGroup = "test2"; - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); - } - @Test public void createEmptyGroup() throws CatalogException { catalogManager.getUserManager().create("test", "test", "test@mail.com", TestParamConstants.PASSWORD, organizationId, 100L, opencgaToken); @@ -681,29 +156,6 @@ public void testAssignPermissions() throws CatalogException { assertTrue(acls.stream().map(x -> String.valueOf(x.get("member"))).collect(Collectors.toSet()).contains("@group_cancer_some_thing_else")); } - @Test - public void getUserInfoTest() throws CatalogException { - OpenCGAResult result = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, QueryOptions.empty(), ownerToken); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, QueryOptions.empty(), orgAdminToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, QueryOptions.empty(), studyAdminToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), normalToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(1, result.first().getProjects().size()); - } - /** * Project methods *************************** */ diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index cf267f04b71..18a20c3667b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -259,8 +259,8 @@ public void createMultipleCasesSameFamily() throws CatalogException { public void updateClinicalAnalystsTest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); // Add analysts OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), @@ -312,7 +312,7 @@ public void updateClinicalAnalysisRequest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); assertTrue(StringUtils.isEmpty(case1.getRequest().getId())); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setEmail("mail@mail.com").setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setEmail("mail@mail.com"), TestParamConstants.PASSWORD, opencgaToken); ClinicalRequest request = new ClinicalRequest("requestId", "bla", null, new ClinicalResponsible().setId("u1"), new HashMap<>()); @@ -350,8 +350,8 @@ public void updateClinicalAnalysisResponsible() throws CatalogException { assertEquals(orgOwnerUserId, case1.getResponsible().getId()); catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setEmail("mail@mail.com") - .setOrganization(organizationId) - .setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); + .setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); ClinicalResponsible responsible = new ClinicalResponsible().setId("u1"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java index ca9323b422d..a72d52eace3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java @@ -287,7 +287,7 @@ private void updateAndAssertChanges(String orgId, OrganizationUserUpdateParams u User user = catalogManager.getOrganizationManager().updateUser(orgId, normalUserId1, userUpdateParams, INCLUDE_RESULT, token).first(); assertEquals(userUpdateParams.getName(), user.getName()); assertEquals(userUpdateParams.getEmail(), user.getEmail()); - assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getAccount().getExpirationDate()); + assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getInternal().getAccount().getExpirationDate()); assertEquals(userUpdateParams.getQuota().getCpuUsage(), user.getQuota().getCpuUsage()); assertEquals(userUpdateParams.getQuota().getDiskUsage(), user.getQuota().getDiskUsage()); assertEquals(userUpdateParams.getQuota().getMaxCpu(), user.getQuota().getMaxCpu()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java index 3a0a4e4a808..85e13914da5 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java @@ -38,7 +38,6 @@ import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.GroupUpdateParams; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -89,7 +88,7 @@ public void searchSampleNoPermissions() throws CatalogException { public void searchProjects() throws CatalogException { String org2 = "otherOrg"; catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); - catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setAccount(new Account()).setOrganization(org2), + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java new file mode 100644 index 00000000000..7bfda8c5261 --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java @@ -0,0 +1,664 @@ +package org.opencb.opencga.catalog.managers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.PasswordUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; +import org.opencb.opencga.core.models.study.Group; +import org.opencb.opencga.core.models.study.GroupUpdateParams; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.StudyAclParams; +import org.opencb.opencga.core.models.user.*; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.MediumTests; + +import javax.naming.NamingException; +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +@Category(MediumTests.class) +public class UserManagerTest extends AbstractManagerTest { + + @Test + public void createOpencgaUserTest() throws CatalogException { + thrown.expect(CatalogException.class); + thrown.expectMessage("forbidden"); + catalogManager.getUserManager().create(new User().setId(ParamConstants.OPENCGA_USER_ID).setName(orgOwnerUserId) + .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + } + + + @Test + public void testAdminUserExists() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + assertEquals(ParamConstants.OPENCGA_USER_ID, payload.getUserId()); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, payload.getOrganization()); + } + + @Test + public void searchUsersTest() throws CatalogException { + OpenCGAResult search = catalogManager.getUserManager().search(organizationId, new Query(), QueryOptions.empty(), opencgaToken); + assertEquals(8, search.getNumResults()); + for (User user : search.getResults()) { + if (noAccessUserId1.equals(user.getId())) { + assertEquals(0, user.getProjects().size()); + } else if (user.getId().startsWith("normalUser")) { + assertEquals(1, user.getProjects().size()); + } else { + assertEquals(2, user.getProjects().size()); + } + } + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), ownerToken); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken2); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken1); + assertEquals(8, search.getNumResults()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), normalToken1)); + } + + @Test + public void testGetToken() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + Map claims = new HashMap<>(); + claims.put("a", "hola"); + claims.put("ab", "byw"); + // Create a token valid for 1 second + String expiringToken = catalogManager.getUserManager().getToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, 1L, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + + String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(nonExpiringToken).getUserId()); + + Thread.sleep(1000); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("expired"); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + } + + @Test + public void loginWithoutOrganizationId() throws CatalogException { + String token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + JwtPayload jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + token = catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(organizationId, jwtPayload.getOrganization()); + + // Create a third organization + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("other").setName("Test"), QueryOptions.empty(), opencgaToken); + token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + thrown.expect(CatalogParameterException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD); + } + + @Test + public void testCreateExistingUser() throws Exception { + thrown.expect(CatalogException.class); + thrown.expectMessage(containsString("already exists")); + catalogManager.getUserManager().create(orgOwnerUserId, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, + null, opencgaToken); + } + + @Test + public void testCreateAnonymousUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, + organizationId, null, opencgaToken); + } + + @Test + public void testCreateRegisteredUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + } + + @Test + public void testLogin() throws Exception { + catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD); + + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); + catalogManager.getUserManager().login(organizationId, normalUserId1, "fakePassword"); + } + + @Test + public void refreshTokenTest() throws Exception { + String refreshToken = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getRefreshToken(); + AuthenticationResponse authenticationResponse = catalogManager.getUserManager().refreshToken(refreshToken); + assertNotNull(authenticationResponse); + assertNotNull(authenticationResponse.getToken()); + } + + @Test + public void anonymousUserLoginTest() throws CatalogException { + AuthenticationResponse authResponse = catalogManager.getUserManager().loginAnonymous(organizationId); + assertNotNull(authResponse.getToken()); + + String org2 = "otherOrg"; + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); + String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); + Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() + .setId("project") + .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), + INCLUDE_RESULT, owner2Token).first(); + Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); + + try { + catalogManager.getUserManager().loginAnonymous(org2); + fail("Anonymous user should not get a token for that organization as it has not been granted any kind of access"); + } catch (Exception e) { + assertEquals(CatalogAuthenticationException.class, e.getClass()); + assertTrue(e.getMessage().contains("not found")); + } + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + authResponse = catalogManager.getUserManager().loginAnonymous(org2); + assertNotNull(authResponse.getToken()); + + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().loginAnonymous(org2); + } + + @Test + public void incrementLoginAttemptsTest() throws CatalogException { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal3 = user.getInternal(); + assertEquals(1, userInternal3.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + for (int i = 2; i < 5; i++) { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(i, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + } + + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal2 = user.getInternal(); + assertEquals(5, userInternal2.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + assertTrue(incorrect.getMessage().contains("banned")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal1 = user.getInternal(); + assertEquals(5, userInternal1.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("banned")); + + // Remove ban from user + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(0, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void changeUserStatusTest() throws CatalogException { + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); + assertTrue(authException.getMessage().contains("own account")); + + authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(authException.getMessage().contains("suspend administrators")); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); + assertTrue(incorrect.getMessage().contains("suspended")); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); + String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + + CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); + assertTrue(paramException.getMessage().contains("Invalid status")); + + CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(dbException.getMessage().contains("not exist")); + } + + @Test + public void loginExpiredAccountTest() throws CatalogException { + // Expire account of normalUserId1 + ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); + catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("expired")); + + // Ensure it doesn't matter whether opencga account is expired or not + catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); + String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void updateUserTest() throws JsonProcessingException, CatalogException { + UserUpdateParams userUpdateParams = new UserUpdateParams() + .setName("newName") + .setEmail("mail@mail.com"); + ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); + assertEquals(userUpdateParams.getName(), user.getName()); + assertEquals(userUpdateParams.getEmail(), user.getEmail()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); + + userUpdateParams = new UserUpdateParams() + .setEmail("notAnEmail"); + ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); + } + + @Test + public void testGetUserInfo() throws CatalogException { + // OpenCGA administrator + DataResult user = catalogManager.getUserManager().get(organizationId, + Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), opencgaToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization owner + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), ownerToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization administrator + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), orgAdminToken1); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), + studyAdminToken1); + } + + @Test + public void testGetProjectsFromUserInfo() throws CatalogException { + String userId = organizationId; + catalogManager.getUserManager().create(userId, "test", "mail@mail.com", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("test")), ownerToken); + String token = catalogManager.getUserManager().login(organizationId, userId, TestParamConstants.PASSWORD).getToken(); + + DataResult user = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(), token); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), normalToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + } + + @Test + public void testModifyUser() throws CatalogException, InterruptedException, IOException { + ObjectMap params = new ObjectMap(); + String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); + String newPassword = PasswordUtils.getStrongRandomPassword(); + String newEmail = "new@email.ac.uk"; + + params.put("name", newName); + + Thread.sleep(10); + + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + catalogManager.getUserManager().update(orgOwnerUserId, new ObjectMap("email", newEmail), null, ownerToken); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD, newPassword); + + List userList = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(QueryOptions + .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), + UserDBAdaptor.QueryParams.ATTRIBUTES.key())), ownerToken).getResults(); + User userPost = userList.get(0); + System.out.println("userPost = " + userPost); + assertEquals(userPost.getName(), newName); + assertEquals(userPost.getEmail(), newEmail); + + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); + CatalogDBException exception = assertThrows(CatalogDBException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("The new password has already been used")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, anotherPassword); + + try { + params = new ObjectMap(); + params.put("password", "1234321"); + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + fail("Expected exception"); + } catch (CatalogDBException e) { + System.out.println(e); + } + + try { + catalogManager.getUserManager().update(orgOwnerUserId, params, null, orgAdminToken1); + fail("Expected exception"); + } catch (CatalogException e) { + System.out.println(e); + } + } + + @Test + public void automaticPasswordExpirationTest() throws CatalogException { + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(1); + + String oneDay = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(1)); + String twoDays = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(2)); + + User user = new User().setId("tempUser"); + String password = PasswordUtils.getStrongRandomPassword(); + User storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account2 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(oneDay) <= Long.parseLong(account2.getPassword().getExpirationDate())); + Account account1 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(twoDays) > Long.parseLong(account1.getPassword().getExpirationDate())); + + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(-5); + user = new User().setId("tempUser2"); + storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account = storedUser.getInternal().getAccount(); + assertNull(account.getPassword().getExpirationDate()); + } + + @Test + public void loginUserPasswordExpiredTest() throws CatalogException { + try (CatalogManager mockCatalogManager = mockCatalogManager()) { + UserDBAdaptor userDBAdaptor = mockCatalogManager.getUserManager().getUserDBAdaptor(organizationId); + + OpenCGAResult result = mockCatalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), normalToken1); + + // Set password expired 2 days ago + Date date = TimeUtils.addDaysToCurrentDate(-2); + String beforeYesterday = TimeUtils.getTime(date); + User user = result.first(); + user.getInternal().getAccount().getPassword().setExpirationDate(beforeYesterday); + + Mockito.doReturn(result).when(userDBAdaptor).get(normalUserId1, UserManager.INCLUDE_INTERNAL); + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> mockCatalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("expired on " + beforeYesterday)); + } + } + + @Test + public void changePasswordTest() throws CatalogException { + String newPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, newPassword); + + CatalogDBException exception = assertThrows(CatalogDBException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword)); + assertTrue(exception.getMessage().contains("verify that the current password is correct")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, anotherPassword); + } + + @Test + public void testUpdateUserConfig() throws CatalogException { + Map map = new HashMap<>(); + map.put("key1", "value1"); + map.put("key2", "value2"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + + Map config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(2, config.size()); + assertEquals("value1", config.get("key1")); + assertEquals("value2", config.get("key2")); + + map = new HashMap<>(); + map.put("key2", "value3"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(1, config.size()); + assertEquals("value3", config.get("key2")); + + catalogManager.getUserManager().deleteConfig(normalUserId1, "a", normalToken1); + + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1); + } + + private String getAdminToken() throws CatalogException, IOException { + return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + } + + @Test + public void createUserUsingMailAsId() throws CatalogException { + catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); + AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); + assertNotNull(login); + User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); + assertEquals("hello.mail@mymail.org", user.getId()); + } + + @Test + public void getUserInfoTest() throws CatalogException { + OpenCGAResult result = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, QueryOptions.empty(), ownerToken); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, QueryOptions.empty(), orgAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, QueryOptions.empty(), studyAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), normalToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(1, result.first().getProjects().size()); + } + + @Ignore + @Test + public void importLdapUsers() throws CatalogException, NamingException, IOException { + // Action only for admins + catalogManager.getUserManager().importRemoteEntities(organizationId, "ldap", Arrays.asList("pfurio", "imedina"), false, null, null, + getAdminToken()); + // TODO: Validate the users have been imported + } + + // To make this test work we will need to add a correct user and password to be able to login + @Ignore + @Test + public void loginNotRegisteredUsers() throws CatalogException { + // Action only for admins + Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); + catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", "view_only"), + ParamUtils.AclAction.SET, ownerToken); + String token = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, "password").getToken(); + + assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + + // We remove the permissions for group ldap + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", ""), + ParamUtils.AclAction.RESET, this.ownerToken); + + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + } + + @Ignore + @Test + public void syncUsers() throws CatalogException { + // Action only for admins + String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", "bio", "bio", studyFqn, true, token); + DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertEquals(0, bio.first().getUserIds().size()); + + catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, studyFqn, "ldap", token); + bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertTrue(!bio.first().getUserIds().isEmpty()); + } + + @Ignore + @Test + public void importLdapGroups() throws CatalogException, IOException { + // Action only for admins + String remoteGroup = "bio"; + String internalGroup = "test"; + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + + DataResult test = catalogManager.getStudyManager().getGroup(studyFqn, "test", ownerToken); + assertEquals(1, test.getNumResults()); + assertEquals("@test", test.first().getId()); + assertTrue(test.first().getUserIds().size() > 0); + +// internalGroup = "test1"; +// try { +// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); +// fail("Should not be possible creating another group containing the same users that belong to a different group"); +// } catch (CatalogException e) { +// System.out.println(e.getMessage()); +// } + + remoteGroup = "bioo"; + internalGroup = "test2"; + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + } + + +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java index f7b28721987..ca96f875326 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java @@ -11,7 +11,6 @@ import org.opencb.opencga.catalog.templates.TemplateManager; import org.opencb.opencga.catalog.templates.config.TemplateManifest; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -25,11 +24,14 @@ public class TemplateManagerTest extends AbstractManagerTest { public void test() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); catalogManager.getStudyManager().create("project", new Study().setId("study"), QueryOptions.empty(), ownerToken); @@ -54,13 +56,13 @@ public void test() throws Exception { public void test_yaml() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 835f89a61a6..d4f3142bc1d 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -441,9 +441,16 @@ public class FieldConstants { public static final String USER_ACCOUNT = "User account."; //Account - public static final String ACCOUNT_TYPE = "User account type can have the values GUEST, FULL and ADMINISTRATOR."; - public static final String ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; - public static final String ACCOUNT_AUTHENTICATION = "How the account is authenticated"; + public static final String INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; + public static final String INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION = "Number of consecutive failed attempts. When the user logs" + + " in successfully, this field is automatically reset to 0."; + public static final String INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION = "Object containing the last time the password was changed and the" + + " expiration date."; + public static final String INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE_DESCRIPTION = "Date the user's password was last changed. This " + + "field will be null if the user account origin is different from OpenCGA."; + public static final String INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION = "Date the user's password expires. This field will" + + " be null if the user account origin is different from OpenCGA and the passwordExpiration feature is disabled."; + public static final String INTERNAL_ACCOUNT_AUTHENTICATION = "How the account is authenticated"; public static final String USER_QUOTA = "User quota"; public static final String USER_PROJECTS = "A List with related projects."; public static final String USER_SHARED_PROJECTS = "A List with shared projects."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java index da6aa39f1f6..c0574f0afdf 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java @@ -135,6 +135,13 @@ public static Date add1YeartoDate(Date date) { return new Date(cal.getTimeInMillis()); } + public static Date addDaysToCurrentDate(int days) { + Calendar cal = new GregorianCalendar(); + cal.setTime(new Date()); + cal.add(Calendar.DAY_OF_YEAR, days); + return new Date(cal.getTimeInMillis()); + } + public static Date toDate(String dateStr) { Date date = null; try { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java new file mode 100644 index 00000000000..f467a93d37d --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.core.config; + +public class AccountConfiguration { + + private int maxLoginAttempts; + private int passwordExpirationDays; + + public AccountConfiguration() { + } + + public AccountConfiguration(int maxLoginAttempts, int passwordExpirationDays) { + this.maxLoginAttempts = maxLoginAttempts; + this.passwordExpirationDays = passwordExpirationDays; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("AccountConfiguration{"); + sb.append("maxLoginAttempts=").append(maxLoginAttempts); + sb.append(", passwordExpirationDays=").append(passwordExpirationDays); + sb.append('}'); + return sb.toString(); + } + + public int getMaxLoginAttempts() { + return maxLoginAttempts; + } + + public AccountConfiguration setMaxLoginAttempts(int maxLoginAttempts) { + this.maxLoginAttempts = maxLoginAttempts; + return this; + } + + public int getPasswordExpirationDays() { + return passwordExpirationDays; + } + + public AccountConfiguration setPasswordExpirationDays(int passwordExpirationDays) { + this.passwordExpirationDays = passwordExpirationDays; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index 600def109ff..b7b869c728c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -44,7 +44,7 @@ public class Configuration { private String workspace; private String jobDir; - private int maxLoginAttempts; + private AccountConfiguration account; private Monitor monitor; private HealthCheck healthCheck; @@ -79,6 +79,7 @@ public Configuration() { analysis = new Analysis(); panel = new Panel(); server = new ServerConfiguration(); + account = new AccountConfiguration(); } public void serialize(OutputStream configurationOututStream) throws IOException { @@ -113,7 +114,6 @@ public static Configuration load(InputStream configurationInputStream, String fo } catch (IOException e) { throw new IOException("Configuration file could not be parsed: " + e.getMessage(), e); } - addDefaultValueIfMissing(configuration); // We must always overwrite configuration with environment parameters @@ -122,8 +122,15 @@ public static Configuration load(InputStream configurationInputStream, String fo } private static void addDefaultValueIfMissing(Configuration configuration) { - if (configuration.getMaxLoginAttempts() <= 0) { - configuration.setMaxLoginAttempts(5); + if (configuration.getAccount() == null) { + configuration.setAccount(new AccountConfiguration()); + } + if (configuration.getAccount().getMaxLoginAttempts() <= 0) { + configuration.getAccount().setMaxLoginAttempts(5); + } + if (configuration.getAccount().getPasswordExpirationDays() < 0) { + // Disable password expiration by default + configuration.getAccount().setPasswordExpirationDays(0); } } @@ -147,7 +154,8 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio configuration.getMonitor().setPort(Integer.parseInt(value)); break; case "OPENCGA.MAX_LOGIN_ATTEMPTS": - configuration.setMaxLoginAttempts(Integer.parseInt(value)); + case "OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS": + configuration.getAccount().setMaxLoginAttempts(Integer.parseInt(value)); break; case "OPENCGA_EXECUTION_MODE": case "OPENCGA_EXECUTION_ID": @@ -203,6 +211,16 @@ public static void reportUnusedField(String field, Object value) { } } + public static void reportMovedField(String previousField, String newField, Object value) { + // Report only if the value is not null and not an empty string + if (value != null && !(value instanceof String && ((String) value).isEmpty())) { + if (reportedFields.add(previousField)) { + // Only log the first time a field is found + logger.warn("Option '{}' with value '{}' was moved to '{}'.", previousField, value, newField); + } + } + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("Configuration{"); @@ -211,7 +229,7 @@ public String toString() { sb.append(", databasePrefix='").append(databasePrefix).append('\''); sb.append(", workspace='").append(workspace).append('\''); sb.append(", jobDir='").append(jobDir).append('\''); - sb.append(", maxLoginAttempts=").append(maxLoginAttempts); + sb.append(", account=").append(account); sb.append(", monitor=").append(monitor); sb.append(", healthCheck=").append(healthCheck); sb.append(", audit=").append(audit); @@ -292,12 +310,24 @@ public Configuration setJobDir(String jobDir) { return this; } + public AccountConfiguration getAccount() { + return account; + } + + public Configuration setAccount(AccountConfiguration account) { + this.account = account; + return this; + } + + @Deprecated public int getMaxLoginAttempts() { - return maxLoginAttempts; + return account.getMaxLoginAttempts(); } + @Deprecated public Configuration setMaxLoginAttempts(int maxLoginAttempts) { - this.maxLoginAttempts = maxLoginAttempts; + reportMovedField("configuration.yml#maxLoginAttempts", "configuration.yml#account.maxLoginAttempts", maxLoginAttempts); + account.setMaxLoginAttempts(maxLoginAttempts); return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java index f31d4d477b4..293ade331be 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java @@ -27,44 +27,49 @@ */ public class Account { - @DataField(id = "creationDate", indexed = true, - description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) - private String creationDate; - @DataField(id = "expirationDate", indexed = true, - description = FieldConstants.ACCOUNT_EXPIRATION_DATE_DESCRIPTION) + description = FieldConstants.INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION) private String expirationDate; + @DataField(id = "password", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION) + private Password password; + + @DataField(id = "failedAttempts", description = FieldConstants.INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION) + private int failedAttempts; @DataField(id = "authentication", indexed = true, uncommentedClasses = {"AccountType"}, - description = FieldConstants.ACCOUNT_AUTHENTICATION) + description = FieldConstants.INTERNAL_ACCOUNT_AUTHENTICATION) private AuthenticationOrigin authentication; public Account() { String creationDate = TimeUtils.getTime(); + // Default 1 year Calendar cal = Calendar.getInstance(); cal.setTime(TimeUtils.toDate(creationDate)); cal.add(Calendar.YEAR, +1); String expirationDate = TimeUtils.getTime(cal.getTime()); - this.creationDate = creationDate; this.expirationDate = expirationDate; + this.password = new Password(); + this.failedAttempts = 0; this.authentication = null; } - public Account(String creationDate, String expirationDate, AuthenticationOrigin authentication) { + public Account(String expirationDate, Password password, int failedAttempts, AuthenticationOrigin authentication) { this.expirationDate = expirationDate; - this.creationDate = creationDate; + this.password = password; + this.failedAttempts = failedAttempts; this.authentication = authentication; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Account{"); - sb.append("creationDate='").append(creationDate).append('\''); - sb.append(", expirationDate='").append(expirationDate).append('\''); - sb.append(", authentication='").append(authentication).append('\''); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", password=").append(password); + sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", authentication=").append(authentication); sb.append('}'); return sb.toString(); } @@ -78,12 +83,21 @@ public Account setExpirationDate(String expirationDate) { return this; } - public String getCreationDate() { - return creationDate; + public Password getPassword() { + return password; + } + + public Account setPassword(Password password) { + this.password = password; + return this; + } + + public int getFailedAttempts() { + return failedAttempts; } - public Account setCreationDate(String creationDate) { - this.creationDate = creationDate; + public Account setFailedAttempts(int failedAttempts) { + this.failedAttempts = failedAttempts; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java new file mode 100644 index 00000000000..6cc9d5f338d --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java @@ -0,0 +1,48 @@ +package org.opencb.opencga.core.models.user; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +public class Password { + + @DataField(id = "expirationDate", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION) + private String expirationDate; + + @DataField(id = "lastChangeDate", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE_DESCRIPTION) + private String lastChangeDate; + + public Password() { + } + + public Password(String expirationDate, String lastChangeDate) { + this.expirationDate = expirationDate; + this.lastChangeDate = lastChangeDate; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Password{"); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", lastChangeDate='").append(lastChangeDate).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getExpirationDate() { + return expirationDate; + } + + public Password setExpirationDate(String expirationDate) { + this.expirationDate = expirationDate; + return this; + } + + public String getLastChangeDate() { + return lastChangeDate; + } + + public Password setLastChangeDate(String lastChangeDate) { + this.lastChangeDate = lastChangeDate; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java index b4d3ef26c02..8694320cee2 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java @@ -16,6 +16,7 @@ package org.opencb.opencga.core.models.user; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; import org.opencb.commons.datastore.core.ObjectMap; @@ -49,14 +50,11 @@ public class User { @DataField(id = "organization", indexed = true, description = FieldConstants.USER_ORGANIZATION) private String organization; - @DataField(id = "account", indexed = true, description = FieldConstants.USER_ACCOUNT) - private Account account; + @DataField(id = "creationDate", since = "3.2.1", description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; - /** - * An object describing the internal information of the User. This is managed by OpenCGA. - * - * @apiNote Internal - */ + @DataField(id = "modificationDate", since = "3.2.1", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; @DataField(id = "internal", indexed = true, description = FieldConstants.GENERIC_INTERNAL) private UserInternal internal; @@ -89,22 +87,21 @@ public class User { public User() { } - public User(String id, Account account) { - this(id, id, null, null, account, new UserInternal(new UserStatus()), null, Collections.emptyList(), Collections.emptyMap(), + public User(String id) { + this(id, id, null, null, null, null, new UserInternal(new UserStatus()), null, Collections.emptyMap(), new LinkedList<>(), Collections.emptyMap()); } public User(String id, String name, String email, String organization, UserInternal internal) { - this(id, name, email, organization, null, internal, null, new ArrayList<>(), new HashMap<>(), new LinkedList<>(), new HashMap<>()); + this(id, name, email, organization, null, null, internal, null, new HashMap<>(), new LinkedList<>(), new HashMap<>()); } - public User(String id, String name, String email, String organization, Account account, UserInternal internal, UserQuota quota, - List projects, Map configs, List filters, Map attributes) { + public User(String id, String name, String email, String organization, UserInternal internal, UserQuota quota, List projects, + Map configs, List filters, Map attributes) { this.id = id; this.name = name; this.email = email; this.organization = organization; - this.account = account != null ? account : new Account(); this.internal = internal; this.quota = quota; this.projects = projects; @@ -113,6 +110,22 @@ public User(String id, String name, String email, String organization, Account a this.attributes = attributes; } + public User(String id, String name, String email, String organization, String creationDate, String modificationDate, + UserInternal internal, UserQuota quota, Map configs, List filters, + Map attributes) { + this.id = id; + this.name = name; + this.email = email; + this.organization = organization; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.internal = internal; + this.quota = quota; + this.configs = configs; + this.filters = filters; + this.attributes = attributes; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("User{"); @@ -120,7 +133,8 @@ public String toString() { sb.append(", name='").append(name).append('\''); sb.append(", email='").append(email).append('\''); sb.append(", organization='").append(organization).append('\''); - sb.append(", account=").append(account); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", internal=").append(internal); sb.append(", quota=").append(quota); sb.append(", projects=").append(projects); @@ -167,12 +181,34 @@ public User setOrganization(String organization) { return this; } + public String getCreationDate() { + return creationDate; + } + + public User setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public User setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + @JsonIgnore + @Deprecated public Account getAccount() { - return account; + return getInternal().getAccount(); } + @JsonIgnore + @Deprecated public User setAccount(Account account) { - this.account = account; + getInternal().setAccount(account); return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java index 329db613b5b..55a75709bd1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java @@ -16,32 +16,41 @@ package org.opencb.opencga.core.models.user; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; public class UserInternal extends Internal { + @DataField(id = "status", description = FieldConstants.INTERNAL_STATUS_DESCRIPTION) private UserStatus status; - private int failedAttempts; + + @DataField(id = "account", since = "3.2.1", description = FieldConstants.USER_ACCOUNT) + private Account account; public UserInternal() { } public UserInternal(UserStatus status) { - this(TimeUtils.getTime(), TimeUtils.getTime(), status); + this(TimeUtils.getTime(), TimeUtils.getTime(), status, new Account()); + } + + public UserInternal(UserStatus status, Account account) { + this(TimeUtils.getTime(), TimeUtils.getTime(), status, account); } - public UserInternal(String registrationDate, String lastModified, UserStatus status) { + public UserInternal(String registrationDate, String lastModified, UserStatus status1, Account account) { super(null, registrationDate, lastModified); - this.status = status; - this.failedAttempts = 0; + this.status = status1; + this.account = account; } @Override public String toString() { final StringBuilder sb = new StringBuilder("UserInternal{"); sb.append("status=").append(status); - sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", account=").append(account); sb.append(", registrationDate='").append(registrationDate).append('\''); sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); @@ -57,12 +66,13 @@ public UserInternal setStatus(UserStatus status) { return this; } - public int getFailedAttempts() { - return failedAttempts; + public Account getAccount() { + return account; } - public UserInternal setFailedAttempts(int failedAttempts) { - this.failedAttempts = failedAttempts; + public UserInternal setAccount(Account account) { + this.account = account; return this; } + } diff --git a/pom.xml b/pom.xml index 84e170213b8..fd542d22621 100644 --- a/pom.xml +++ b/pom.xml @@ -1369,7 +1369,7 @@ opencga LOCAL - 5 + 5 https://ws.opencb.org/opencga-prod From 7c8520961279439b774c3c03c3fc515f77999fc6 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 24 Jul 2024 11:26:33 +0200 Subject: [PATCH 081/128] core: rename lastChangeDate for lastModified, #TASK-6494 --- .../MoveUserAccountToInternalMigration.java | 7 ++- .../opencga/catalog/db/api/UserDBAdaptor.java | 3 +- .../db/mongodb/UserMongoDBAdaptor.java | 2 +- .../db/mongodb/converters/UserConverter.java | 54 ++++++++++++++++++- .../opencga/catalog/managers/UserManager.java | 6 +-- .../opencga/core/api/FieldConstants.java | 2 +- .../opencga/core/models/user/Password.java | 18 +++---- .../opencb/opencga/core/models/user/User.java | 11 ++-- .../src/main/resources/configuration.yml | 2 +- 9 files changed, 80 insertions(+), 25 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java index 64a824ec8ed..68f2cc00e7c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java @@ -30,17 +30,20 @@ protected void run() throws Exception { Document internal = document.get("internal", Document.class); internal.put("account", account); - updateDocument.getSet().put("expirationDate", internal.get("lastModified")); + updateDocument.getSet().put("modificationDate", internal.get("lastModified")); updateDocument.getSet().put("creationDate", account.get("creationDate")); account.remove("creationDate"); Document password = new Document() .append("expirationDate", null) - .append("lastChangedDate", internal.get("lastModified")); + .append("lastModified", internal.get("lastModified")); account.put("password", password); account.put("failedAttempts", internal.get("failedAttempts")); internal.remove("failedAttempts"); + updateDocument.getSet().put("internal", internal); + updateDocument.getUnset().add("account"); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); }); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index bc25060061e..8c67639017c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -120,6 +120,7 @@ enum QueryParams implements QueryParam { ORGANIZATION("organization", TEXT_ARRAY, ""), CREATION_DATE("creationDate", TEXT_ARRAY, ""), MODIFICATION_DATE("modificationDate", TEXT_ARRAY, ""), + DEPRECATED_ACCOUNT("account", OBJECT, ""), // Deprecated since 3.2.1 #TASK-6494 TODO: Remove in future releases INTERNAL("internal", OBJECT, ""), INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), @@ -129,7 +130,7 @@ enum QueryParams implements QueryParam { INTERNAL_ACCOUNT_CREATION_DATE("internal.account.creationDate", TEXT, ""), INTERNAL_ACCOUNT_EXPIRATION_DATE("internal.account.expirationDate", TEXT, ""), INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE("internal.account.password.expirationDate", TEXT, ""), - INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE("internal.account.password.lastChangeDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED("internal.account.password.lastModified", TEXT, ""), QUOTA("quota", OBJECT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 581ed6f1620..76bc3bd9926 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -172,7 +172,7 @@ public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, St String encryptedPassword = encryptPassword(newPassword); updateDocument.getSet().put(PRIVATE_PASSWORD, encryptedPassword); updateDocument.getPush().put(ARCHIVE_PASSWORD, encryptedPassword); - updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE.key(), TimeUtils.getTime()); + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED.key(), TimeUtils.getTime()); if (configuration.getAccount().getPasswordExpirationDays() > 0) { Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); String stringDate = TimeUtils.getTime(date); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java index 32420556a73..837349b6b8a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java @@ -18,27 +18,77 @@ import org.apache.avro.generic.GenericRecord; import org.bson.Document; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserInternal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Created by pfurio on 19/01/16. */ public class UserConverter extends OpenCgaMongoConverter { + protected static Logger logger = LoggerFactory.getLogger(UserConverter.class); + public UserConverter() { super(User.class); getObjectMapper().addMixIn(GenericRecord.class, GenericRecordAvroJsonMixin.class); } + @Override + public Document convertToStorageType(User object) { + Document userDocument = super.convertToStorageType(object); + removeDeprecatedAccountObject(userDocument); + return userDocument; + } + @Override public User convertToDataModelType(Document document) { User user = super.convertToDataModelType(document); + + restoreFromDeprecatedAccountObject(user); + addToDeprecatedAccountObject(user); + + return user; + } + + /** + * Remove 'account' object from the User document so it is no longer stored in the database. + * Remove after a few releases. + * + * @param userDocument User document. + */ + @Deprecated + private void removeDeprecatedAccountObject(Document userDocument) { + userDocument.remove(UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key()); + } + + /** + * Restores information from the account object to the corresponding internal.account object. + * Added to maintain backwards compatibility with the deprecated account object in TASK-6494 (v3.2.1) + * Remove after a few releases. + * + * @param user User object. + */ + @Deprecated + private void restoreFromDeprecatedAccountObject(User user) { + if (user.getAccount() != null) { + if (user.getInternal() == null) { + user.setInternal(new UserInternal()); + } + user.getInternal().setAccount(user.getAccount()); + logger.warn("Restoring user account information from deprecated account object to internal.account object. " + + "Please, run 'opencga-admin.sh migration run'."); + } + } + + private void addToDeprecatedAccountObject(User user) { // Add account to deprecated place - if (user.getInternal() != null && user.getInternal().getAccount() != null) { + if (user.getInternal() != null && user.getInternal().getAccount() != null && user.getAccount() == null) { user.setAccount(user.getInternal().getAccount()); } - return user; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index 9fa16ae9d64..8070d177233 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -62,8 +62,8 @@ */ public class UserManager extends AbstractManager { - static final QueryOptions INCLUDE_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.INTERNAL.key())); + static final QueryOptions INCLUDE_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.ID.key(), + UserDBAdaptor.QueryParams.INTERNAL.key(), UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key())); protected static Logger logger = LoggerFactory.getLogger(UserManager.class); private final CatalogIOManager catalogIOManager; private final AuthenticationFactory authenticationFactory; @@ -171,7 +171,7 @@ public OpenCGAResult create(User user, String password, String token) thro // Set password expiration if (AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId())) { - account.getPassword().setLastChangeDate(TimeUtils.getTime()); + account.getPassword().setLastModified(TimeUtils.getTime()); } if (!AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId()) || configuration.getAccount().getPasswordExpirationDays() <= 0) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index d4f3142bc1d..c5aaf9dadf9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -446,7 +446,7 @@ public class FieldConstants { + " in successfully, this field is automatically reset to 0."; public static final String INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION = "Object containing the last time the password was changed and the" + " expiration date."; - public static final String INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE_DESCRIPTION = "Date the user's password was last changed. This " + public static final String INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION = "Date the user's password was last changed. This " + "field will be null if the user account origin is different from OpenCGA."; public static final String INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION = "Date the user's password expires. This field will" + " be null if the user account origin is different from OpenCGA and the passwordExpiration feature is disabled."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java index 6cc9d5f338d..5b375936bd3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java @@ -8,22 +8,22 @@ public class Password { @DataField(id = "expirationDate", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION) private String expirationDate; - @DataField(id = "lastChangeDate", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_LAST_CHANGE_DATE_DESCRIPTION) - private String lastChangeDate; + @DataField(id = "lastModified", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION) + private String lastModified; public Password() { } - public Password(String expirationDate, String lastChangeDate) { + public Password(String expirationDate, String lastModified) { this.expirationDate = expirationDate; - this.lastChangeDate = lastChangeDate; + this.lastModified = lastModified; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Password{"); sb.append("expirationDate='").append(expirationDate).append('\''); - sb.append(", lastChangeDate='").append(lastChangeDate).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); return sb.toString(); } @@ -37,12 +37,12 @@ public Password setExpirationDate(String expirationDate) { return this; } - public String getLastChangeDate() { - return lastChangeDate; + public String getLastModified() { + return lastModified; } - public Password setLastChangeDate(String lastChangeDate) { - this.lastChangeDate = lastChangeDate; + public Password setLastModified(String lastModified) { + this.lastModified = lastModified; return this; } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java index 8694320cee2..98273a80528 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java @@ -16,7 +16,6 @@ package org.opencb.opencga.core.models.user; -import com.fasterxml.jackson.annotation.JsonIgnore; import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; import org.opencb.commons.datastore.core.ObjectMap; @@ -56,6 +55,10 @@ public class User { @DataField(id = "modificationDate", since = "3.2.1", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) private String modificationDate; + @DataField(id = "account", description = FieldConstants.USER_ACCOUNT) + @Deprecated + private Account account; + @DataField(id = "internal", indexed = true, description = FieldConstants.GENERIC_INTERNAL) private UserInternal internal; @@ -199,16 +202,14 @@ public User setModificationDate(String modificationDate) { return this; } - @JsonIgnore @Deprecated public Account getAccount() { - return getInternal().getAccount(); + return account; } - @JsonIgnore @Deprecated public User setAccount(Account account) { - getInternal().setAccount(account); + this.account = account; return this; } diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index e3661d0167a..4d16f4aa68d 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -7,7 +7,7 @@ workspace: ${OPENCGA.USER.WORKSPACE} jobDir: ${OPENCGA.USER.WORKSPACE}/jobs # Maximum number of login attempts before banning a user account -maxLoginAttempts: ${OPENCGA.MAX_LOGIN_ATTEMPTS} +maxLoginAttempts: ${OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS} panel: host: "http://resources.opencb.org/opencb/opencga/disease-panels" From 0f34c370a1c48b51664e7d8c44a2c8f7f1d412d9 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 24 Jul 2024 12:00:16 +0200 Subject: [PATCH 082/128] server: change executor cli generator to consider maps, #TASK-6626 --- .../executors/OpencgaCommandExecutor.java | 22 +++++++++---------- .../cli/ExecutorsCliRestApiWriter.java | 6 +++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java index 72f9bdd0e8c..f1e0c47ad70 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java @@ -17,8 +17,8 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; @@ -36,19 +36,13 @@ import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.server.generator.models.RestCategory; -import org.opencb.opencga.server.generator.models.RestEndpoint; -import org.opencb.opencga.server.generator.models.RestParameter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.Date; -import java.util.List; +import java.util.*; /** * Created on 27/05/16. @@ -266,18 +260,22 @@ public RestResponse refreshToken(AuthenticationResponse return res; } - public Object putNestedIfNotNull(ObjectMap map, String key, Object value, boolean parents) { + public void putNestedIfNotNull(ObjectMap map, String key, Object value, boolean parents) { if (value != null) { map.putNested(key, value, parents); } - return null; } - public Object putNestedIfNotEmpty(ObjectMap map, String key, String value, boolean parents) { + public void putNestedIfNotEmpty(ObjectMap map, String key, String value, boolean parents) { if (StringUtils.isNotEmpty(value)) { map.putNested(key, value, parents); } - return null; + } + + public void putNestedMapIfNotEmpty(ObjectMap map, String key, Map value, boolean parents) { + if (MapUtils.isNotEmpty(value)) { + map.putNested(key, value, parents); + } } public boolean checkExpiredSession(String[] args) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java index 8befc2876c5..faa3dd8626e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java @@ -401,9 +401,11 @@ private String getBodyParams(RestParameter body) { String javaCommandOptionsField = "commandOptions." + getJavaFieldName(restParameter); String label = StringUtils.isEmpty(restParameter.getParentName()) ? restParameter.getName() : restParameter.getParentName() + "." + restParameter.getName(); if (restParameter.getTypeClass().equals("java.lang.String;")) { - sb.append(" putNestedIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\"," + javaCommandOptionsField + ", true);\n"); + sb.append(" putNestedIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); + } else if (isValidMap(restParameter)) { + sb.append(" putNestedMapIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); } else { - sb.append(" putNestedIfNotNull(beanParams, \"" + label.replaceAll("body_", "") + "\"," + javaCommandOptionsField + ", true);\n"); + sb.append(" putNestedIfNotNull(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); } } } From b0609587a1568705a1c0687e223261405d2eca24 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 24 Jul 2024 12:11:23 +0200 Subject: [PATCH 083/128] app: regenerate cli executors, #TASK-6626 --- .../main/executors/AdminCommandExecutor.java | 44 +- .../AnalysisAlignmentCommandExecutor.java | 62 +- .../AnalysisClinicalCommandExecutor.java | 352 ++++----- .../AnalysisVariantCommandExecutor.java | 684 +++++++++--------- .../executors/CohortsCommandExecutor.java | 66 +- .../DiseasePanelsCommandExecutor.java | 52 +- .../executors/FamiliesCommandExecutor.java | 52 +- .../main/executors/FilesCommandExecutor.java | 154 ++-- .../executors/IndividualsCommandExecutor.java | 154 ++-- .../main/executors/JobsCommandExecutor.java | 56 +- ...erationsVariantStorageCommandExecutor.java | 272 +++---- .../OrganizationsCommandExecutor.java | 64 +- .../executors/ProjectsCommandExecutor.java | 42 +- .../executors/SamplesCommandExecutor.java | 120 +-- .../executors/StudiesCommandExecutor.java | 112 +-- .../main/executors/UsersCommandExecutor.java | 28 +- 16 files changed, 1157 insertions(+), 1157 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java index e78430b3f96..99135ede6b0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java @@ -130,9 +130,9 @@ private RestResponse installCatalog() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InstallationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "secretKey",commandOptions.secretKey, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "secretKey", commandOptions.secretKey, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); installationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -161,7 +161,7 @@ private RestResponse jwtCatalog() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JWTParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "secretKey",commandOptions.secretKey, true); + putNestedIfNotEmpty(beanParams, "secretKey", commandOptions.secretKey, true); jWTParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -186,11 +186,11 @@ private RestResponse createUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "organization", commandOptions.organization, true); userCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -219,11 +219,11 @@ private RestResponse importUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserImportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "authenticationOriginId",commandOptions.authenticationOriginId, true); - putNestedIfNotNull(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "resourceType",commandOptions.resourceType, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotEmpty(beanParams, "studyGroup",commandOptions.studyGroup, true); + putNestedIfNotEmpty(beanParams, "authenticationOriginId", commandOptions.authenticationOriginId, true); + putNestedIfNotNull(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "resourceType", commandOptions.resourceType, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotEmpty(beanParams, "studyGroup", commandOptions.studyGroup, true); userImportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -287,12 +287,12 @@ private RestResponse syncUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupSyncParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "authenticationOriginId",commandOptions.authenticationOriginId, true); - putNestedIfNotEmpty(beanParams, "from",commandOptions.from, true); - putNestedIfNotEmpty(beanParams, "to",commandOptions.to, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotNull(beanParams, "syncAll",commandOptions.syncAll, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotEmpty(beanParams, "authenticationOriginId", commandOptions.authenticationOriginId, true); + putNestedIfNotEmpty(beanParams, "from", commandOptions.from, true); + putNestedIfNotEmpty(beanParams, "to", commandOptions.to, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotNull(beanParams, "syncAll", commandOptions.syncAll, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); groupSyncParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -322,8 +322,8 @@ private RestResponse usersUpdateGroups() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserUpdateGroup.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "studyIds",commandOptions.studyIds, true); - putNestedIfNotNull(beanParams, "groupIds",commandOptions.groupIds, true); + putNestedIfNotNull(beanParams, "studyIds", commandOptions.studyIds, true); + putNestedIfNotNull(beanParams, "groupIds", commandOptions.groupIds, true); userUpdateGroup = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index ddc77f0009b..566389f36b2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -140,12 +140,12 @@ private RestResponse runBwa() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), BwaWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "fastaFile",commandOptions.fastaFile, true); - putNestedIfNotEmpty(beanParams, "fastq1File",commandOptions.fastq1File, true); - putNestedIfNotEmpty(beanParams, "fastq2File",commandOptions.fastq2File, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "bwaParams",commandOptions.bwaParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "fastaFile", commandOptions.fastaFile, true); + putNestedIfNotEmpty(beanParams, "fastq1File", commandOptions.fastq1File, true); + putNestedIfNotEmpty(beanParams, "fastq2File", commandOptions.fastq2File, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "bwaParams", commandOptions.bwaParams, true); bwaWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -184,9 +184,9 @@ private RestResponse runCoverageIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CoverageIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFileId",commandOptions.bamFileId, true); - putNestedIfNotEmpty(beanParams, "baiFileId",commandOptions.baiFileId, true); - putNestedIfNotNull(beanParams, "windowSize",commandOptions.windowSize, true); + putNestedIfNotEmpty(beanParams, "bamFileId", commandOptions.bamFileId, true); + putNestedIfNotEmpty(beanParams, "baiFileId", commandOptions.baiFileId, true); + putNestedIfNotNull(beanParams, "windowSize", commandOptions.windowSize, true); coverageIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -225,9 +225,9 @@ private RestResponse coverageQcGeneCoverageStatsRun() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentGeneCoverageStatsParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFile",commandOptions.bamFile, true); - putNestedIfNotNull(beanParams, "genes",commandOptions.genes, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "bamFile", commandOptions.bamFile, true); + putNestedIfNotNull(beanParams, "genes", commandOptions.genes, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); alignmentGeneCoverageStatsParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -323,9 +323,9 @@ private RestResponse runDeeptools() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), DeeptoolsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "deeptoolsParams",commandOptions.deeptoolsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "deeptoolsParams", commandOptions.deeptoolsParams, true); deeptoolsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -364,9 +364,9 @@ private RestResponse runFastqc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FastqcWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "inputFile",commandOptions.inputFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "fastqcParams",commandOptions.fastqcParams, true); + putNestedIfNotEmpty(beanParams, "inputFile", commandOptions.inputFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "fastqcParams", commandOptions.fastqcParams, true); fastqcWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -405,8 +405,8 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "fileId",commandOptions.fileId, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "fileId", commandOptions.fileId, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); alignmentIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -445,9 +445,9 @@ private RestResponse runPicard() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PicardWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "picardParams",commandOptions.picardParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "picardParams", commandOptions.picardParams, true); picardWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -486,10 +486,10 @@ private RestResponse runQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentQcParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFile",commandOptions.bamFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "bamFile", commandOptions.bamFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); alignmentQcParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -560,10 +560,10 @@ private RestResponse runSamtools() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SamtoolsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "inputFile",commandOptions.inputFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "samtoolsParams",commandOptions.samtoolsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "inputFile", commandOptions.inputFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "samtoolsParams", commandOptions.samtoolsParams, true); samtoolsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index 84eeffe1be6..0a1db0995cf 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -238,8 +238,8 @@ private RestResponse updateAcl() throws Exception .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); clinicalAnalysisAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -273,7 +273,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -305,7 +305,7 @@ private RestResponse updateClinicalConfiguration() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisStudyConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "interpretation.defaultFilter",commandOptions.interpretationDefaultFilter, true); + putNestedMapIfNotEmpty(beanParams, "interpretation.defaultFilter", commandOptions.interpretationDefaultFilter, true); clinicalAnalysisStudyConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -341,47 +341,47 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "disorder.id",commandOptions.disorderId, true); - putNestedIfNotEmpty(beanParams, "proband.id",commandOptions.probandId, true); - putNestedIfNotEmpty(beanParams, "family.id",commandOptions.familyId, true); - putNestedIfNotNull(beanParams, "panelLock",commandOptions.panelLock, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "report.title",commandOptions.reportTitle, true); - putNestedIfNotEmpty(beanParams, "report.overview",commandOptions.reportOverview, true); - putNestedIfNotEmpty(beanParams, "report.logo",commandOptions.reportLogo, true); - putNestedIfNotEmpty(beanParams, "report.signedBy",commandOptions.reportSignedBy, true); - putNestedIfNotEmpty(beanParams, "report.signature",commandOptions.reportSignature, true); - putNestedIfNotEmpty(beanParams, "report.date",commandOptions.reportDate, true); - putNestedIfNotEmpty(beanParams, "request.id",commandOptions.requestId, true); - putNestedIfNotEmpty(beanParams, "request.justification",commandOptions.requestJustification, true); - putNestedIfNotEmpty(beanParams, "request.date",commandOptions.requestDate, true); - putNestedIfNotNull(beanParams, "request.attributes",commandOptions.requestAttributes, true); - putNestedIfNotEmpty(beanParams, "responsible.id",commandOptions.responsibleId, true); - putNestedIfNotEmpty(beanParams, "responsible.name",commandOptions.responsibleName, true); - putNestedIfNotEmpty(beanParams, "responsible.email",commandOptions.responsibleEmail, true); - putNestedIfNotEmpty(beanParams, "responsible.organization",commandOptions.responsibleOrganization, true); - putNestedIfNotEmpty(beanParams, "responsible.department",commandOptions.responsibleDepartment, true); - putNestedIfNotEmpty(beanParams, "responsible.address",commandOptions.responsibleAddress, true); - putNestedIfNotEmpty(beanParams, "responsible.city",commandOptions.responsibleCity, true); - putNestedIfNotEmpty(beanParams, "responsible.postcode",commandOptions.responsiblePostcode, true); - putNestedIfNotEmpty(beanParams, "interpretation.description",commandOptions.interpretationDescription, true); - putNestedIfNotEmpty(beanParams, "interpretation.clinicalAnalysisId",commandOptions.interpretationClinicalAnalysisId, true); - putNestedIfNotEmpty(beanParams, "interpretation.creationDate",commandOptions.interpretationCreationDate, true); - putNestedIfNotEmpty(beanParams, "interpretation.modificationDate",commandOptions.interpretationModificationDate, true); - putNestedIfNotNull(beanParams, "interpretation.locked",commandOptions.interpretationLocked, true); - putNestedIfNotNull(beanParams, "interpretation.attributes",commandOptions.interpretationAttributes, true); - putNestedIfNotNull(beanParams, "qualityControl.summary",commandOptions.qualityControlSummary, true); - putNestedIfNotNull(beanParams, "qualityControl.comments",commandOptions.qualityControlComments, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "dueDate",commandOptions.dueDate, true); - putNestedIfNotEmpty(beanParams, "priority.id",commandOptions.priorityId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "disorder.id", commandOptions.disorderId, true); + putNestedIfNotEmpty(beanParams, "proband.id", commandOptions.probandId, true); + putNestedIfNotEmpty(beanParams, "family.id", commandOptions.familyId, true); + putNestedIfNotNull(beanParams, "panelLock", commandOptions.panelLock, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "report.title", commandOptions.reportTitle, true); + putNestedIfNotEmpty(beanParams, "report.overview", commandOptions.reportOverview, true); + putNestedIfNotEmpty(beanParams, "report.logo", commandOptions.reportLogo, true); + putNestedIfNotEmpty(beanParams, "report.signedBy", commandOptions.reportSignedBy, true); + putNestedIfNotEmpty(beanParams, "report.signature", commandOptions.reportSignature, true); + putNestedIfNotEmpty(beanParams, "report.date", commandOptions.reportDate, true); + putNestedIfNotEmpty(beanParams, "request.id", commandOptions.requestId, true); + putNestedIfNotEmpty(beanParams, "request.justification", commandOptions.requestJustification, true); + putNestedIfNotEmpty(beanParams, "request.date", commandOptions.requestDate, true); + putNestedMapIfNotEmpty(beanParams, "request.attributes", commandOptions.requestAttributes, true); + putNestedIfNotEmpty(beanParams, "responsible.id", commandOptions.responsibleId, true); + putNestedIfNotEmpty(beanParams, "responsible.name", commandOptions.responsibleName, true); + putNestedIfNotEmpty(beanParams, "responsible.email", commandOptions.responsibleEmail, true); + putNestedIfNotEmpty(beanParams, "responsible.organization", commandOptions.responsibleOrganization, true); + putNestedIfNotEmpty(beanParams, "responsible.department", commandOptions.responsibleDepartment, true); + putNestedIfNotEmpty(beanParams, "responsible.address", commandOptions.responsibleAddress, true); + putNestedIfNotEmpty(beanParams, "responsible.city", commandOptions.responsibleCity, true); + putNestedIfNotEmpty(beanParams, "responsible.postcode", commandOptions.responsiblePostcode, true); + putNestedIfNotEmpty(beanParams, "interpretation.description", commandOptions.interpretationDescription, true); + putNestedIfNotEmpty(beanParams, "interpretation.clinicalAnalysisId", commandOptions.interpretationClinicalAnalysisId, true); + putNestedIfNotEmpty(beanParams, "interpretation.creationDate", commandOptions.interpretationCreationDate, true); + putNestedIfNotEmpty(beanParams, "interpretation.modificationDate", commandOptions.interpretationModificationDate, true); + putNestedIfNotNull(beanParams, "interpretation.locked", commandOptions.interpretationLocked, true); + putNestedMapIfNotEmpty(beanParams, "interpretation.attributes", commandOptions.interpretationAttributes, true); + putNestedIfNotNull(beanParams, "qualityControl.summary", commandOptions.qualityControlSummary, true); + putNestedIfNotNull(beanParams, "qualityControl.comments", commandOptions.qualityControlComments, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "dueDate", commandOptions.dueDate, true); + putNestedIfNotEmpty(beanParams, "priority.id", commandOptions.priorityId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); clinicalAnalysisCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -537,9 +537,9 @@ private RestResponse runInterpreterCancerTiering() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CancerTieringInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "discardedVariants",commandOptions.discardedVariants, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "discardedVariants", commandOptions.discardedVariants, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); cancerTieringInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -578,7 +578,7 @@ private RestResponse runInterpreterExomiser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ExomiserInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); exomiserInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -617,10 +617,10 @@ private RestResponse runInterpreterTeam() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TeamInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "panels",commandOptions.panels, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "panels", commandOptions.panels, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); teamInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -659,10 +659,10 @@ private RestResponse runInterpreterTiering() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TieringInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "panels",commandOptions.panels, true); - putNestedIfNotNull(beanParams, "penetrance",commandOptions.penetrance, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "panels", commandOptions.panels, true); + putNestedIfNotNull(beanParams, "penetrance", commandOptions.penetrance, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); tieringInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -701,57 +701,57 @@ private RestResponse runInterpreterZetta() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ZettaInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotEmpty(beanParams, "fileData",commandOptions.fileData, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "sampleData",commandOptions.sampleData, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "sampleMetadata",commandOptions.sampleMetadata, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMgf",commandOptions.cohortStatsMgf, true); - putNestedIfNotEmpty(beanParams, "cohortStatsPass",commandOptions.cohortStatsPass, true); - putNestedIfNotEmpty(beanParams, "score",commandOptions.score, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "familyDisorder",commandOptions.familyDisorder, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotEmpty(beanParams, "familyMembers",commandOptions.familyMembers, true); - putNestedIfNotEmpty(beanParams, "familyProband",commandOptions.familyProband, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "geneTraitId",commandOptions.geneTraitId, true); - putNestedIfNotEmpty(beanParams, "go",commandOptions.go, true); - putNestedIfNotEmpty(beanParams, "expression",commandOptions.expression, true); - putNestedIfNotEmpty(beanParams, "proteinKeyword",commandOptions.proteinKeyword, true); - putNestedIfNotEmpty(beanParams, "drug",commandOptions.drug, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "customAnnotation",commandOptions.customAnnotation, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotEmpty(beanParams, "trait",commandOptions.trait, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotEmpty(beanParams, "fileData", commandOptions.fileData, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "sampleData", commandOptions.sampleData, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "sampleMetadata", commandOptions.sampleMetadata, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMgf", commandOptions.cohortStatsMgf, true); + putNestedIfNotEmpty(beanParams, "cohortStatsPass", commandOptions.cohortStatsPass, true); + putNestedIfNotEmpty(beanParams, "score", commandOptions.score, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "familyDisorder", commandOptions.familyDisorder, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotEmpty(beanParams, "familyMembers", commandOptions.familyMembers, true); + putNestedIfNotEmpty(beanParams, "familyProband", commandOptions.familyProband, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "geneTraitId", commandOptions.geneTraitId, true); + putNestedIfNotEmpty(beanParams, "go", commandOptions.go, true); + putNestedIfNotEmpty(beanParams, "expression", commandOptions.expression, true); + putNestedIfNotEmpty(beanParams, "proteinKeyword", commandOptions.proteinKeyword, true); + putNestedIfNotEmpty(beanParams, "drug", commandOptions.drug, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "customAnnotation", commandOptions.customAnnotation, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotEmpty(beanParams, "trait", commandOptions.trait, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); zettaInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -790,7 +790,7 @@ private RestResponse load() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisLoadParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); clinicalAnalysisLoadParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -945,7 +945,7 @@ private RestResponse runRgaIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RgaAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); rgaAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1288,42 +1288,42 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "disorder.id",commandOptions.disorderId, true); - putNestedIfNotNull(beanParams, "panelLock",commandOptions.panelLock, true); - putNestedIfNotEmpty(beanParams, "proband.id",commandOptions.probandId, true); - putNestedIfNotEmpty(beanParams, "family.id",commandOptions.familyId, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "report.title",commandOptions.reportTitle, true); - putNestedIfNotEmpty(beanParams, "report.overview",commandOptions.reportOverview, true); - putNestedIfNotEmpty(beanParams, "report.logo",commandOptions.reportLogo, true); - putNestedIfNotEmpty(beanParams, "report.signedBy",commandOptions.reportSignedBy, true); - putNestedIfNotEmpty(beanParams, "report.signature",commandOptions.reportSignature, true); - putNestedIfNotEmpty(beanParams, "report.date",commandOptions.reportDate, true); - putNestedIfNotEmpty(beanParams, "request.id",commandOptions.requestId, true); - putNestedIfNotEmpty(beanParams, "request.justification",commandOptions.requestJustification, true); - putNestedIfNotEmpty(beanParams, "request.date",commandOptions.requestDate, true); - putNestedIfNotNull(beanParams, "request.attributes",commandOptions.requestAttributes, true); - putNestedIfNotEmpty(beanParams, "responsible.id",commandOptions.responsibleId, true); - putNestedIfNotEmpty(beanParams, "responsible.name",commandOptions.responsibleName, true); - putNestedIfNotEmpty(beanParams, "responsible.email",commandOptions.responsibleEmail, true); - putNestedIfNotEmpty(beanParams, "responsible.organization",commandOptions.responsibleOrganization, true); - putNestedIfNotEmpty(beanParams, "responsible.department",commandOptions.responsibleDepartment, true); - putNestedIfNotEmpty(beanParams, "responsible.address",commandOptions.responsibleAddress, true); - putNestedIfNotEmpty(beanParams, "responsible.city",commandOptions.responsibleCity, true); - putNestedIfNotEmpty(beanParams, "responsible.postcode",commandOptions.responsiblePostcode, true); - putNestedIfNotNull(beanParams, "qualityControl.summary",commandOptions.qualityControlSummary, true); - putNestedIfNotNull(beanParams, "qualityControl.comments",commandOptions.qualityControlComments, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "dueDate",commandOptions.dueDate, true); - putNestedIfNotEmpty(beanParams, "priority.id",commandOptions.priorityId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "disorder.id", commandOptions.disorderId, true); + putNestedIfNotNull(beanParams, "panelLock", commandOptions.panelLock, true); + putNestedIfNotEmpty(beanParams, "proband.id", commandOptions.probandId, true); + putNestedIfNotEmpty(beanParams, "family.id", commandOptions.familyId, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "report.title", commandOptions.reportTitle, true); + putNestedIfNotEmpty(beanParams, "report.overview", commandOptions.reportOverview, true); + putNestedIfNotEmpty(beanParams, "report.logo", commandOptions.reportLogo, true); + putNestedIfNotEmpty(beanParams, "report.signedBy", commandOptions.reportSignedBy, true); + putNestedIfNotEmpty(beanParams, "report.signature", commandOptions.reportSignature, true); + putNestedIfNotEmpty(beanParams, "report.date", commandOptions.reportDate, true); + putNestedIfNotEmpty(beanParams, "request.id", commandOptions.requestId, true); + putNestedIfNotEmpty(beanParams, "request.justification", commandOptions.requestJustification, true); + putNestedIfNotEmpty(beanParams, "request.date", commandOptions.requestDate, true); + putNestedMapIfNotEmpty(beanParams, "request.attributes", commandOptions.requestAttributes, true); + putNestedIfNotEmpty(beanParams, "responsible.id", commandOptions.responsibleId, true); + putNestedIfNotEmpty(beanParams, "responsible.name", commandOptions.responsibleName, true); + putNestedIfNotEmpty(beanParams, "responsible.email", commandOptions.responsibleEmail, true); + putNestedIfNotEmpty(beanParams, "responsible.organization", commandOptions.responsibleOrganization, true); + putNestedIfNotEmpty(beanParams, "responsible.department", commandOptions.responsibleDepartment, true); + putNestedIfNotEmpty(beanParams, "responsible.address", commandOptions.responsibleAddress, true); + putNestedIfNotEmpty(beanParams, "responsible.city", commandOptions.responsibleCity, true); + putNestedIfNotEmpty(beanParams, "responsible.postcode", commandOptions.responsiblePostcode, true); + putNestedIfNotNull(beanParams, "qualityControl.summary", commandOptions.qualityControlSummary, true); + putNestedIfNotNull(beanParams, "qualityControl.comments", commandOptions.qualityControlComments, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "dueDate", commandOptions.dueDate, true); + putNestedIfNotEmpty(beanParams, "priority.id", commandOptions.priorityId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); clinicalAnalysisUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1403,17 +1403,17 @@ private RestResponse createInterpretation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InterpretationCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysisId",commandOptions.clinicalAnalysisId, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "method.name",commandOptions.methodName, true); - putNestedIfNotEmpty(beanParams, "method.version",commandOptions.methodVersion, true); - putNestedIfNotEmpty(beanParams, "method.commit",commandOptions.methodCommit, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysisId", commandOptions.clinicalAnalysisId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "method.name", commandOptions.methodName, true); + putNestedIfNotEmpty(beanParams, "method.version", commandOptions.methodVersion, true); + putNestedIfNotEmpty(beanParams, "method.commit", commandOptions.methodCommit, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); interpretationCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1492,16 +1492,16 @@ private RestResponse updateInterpretation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InterpretationUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "method.name",commandOptions.methodName, true); - putNestedIfNotEmpty(beanParams, "method.version",commandOptions.methodVersion, true); - putNestedIfNotEmpty(beanParams, "method.commit",commandOptions.methodCommit, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "method.name", commandOptions.methodName, true); + putNestedIfNotEmpty(beanParams, "method.version", commandOptions.methodVersion, true); + putNestedIfNotEmpty(beanParams, "method.commit", commandOptions.methodCommit, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); interpretationUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1537,15 +1537,15 @@ private RestResponse updateReport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalReport.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "title",commandOptions.title, true); - putNestedIfNotEmpty(beanParams, "overview",commandOptions.overview, true); - putNestedIfNotEmpty(beanParams, "discussion.author",commandOptions.discussionAuthor, true); - putNestedIfNotEmpty(beanParams, "discussion.date",commandOptions.discussionDate, true); - putNestedIfNotEmpty(beanParams, "discussion.text",commandOptions.discussionText, true); - putNestedIfNotEmpty(beanParams, "logo",commandOptions.logo, true); - putNestedIfNotEmpty(beanParams, "signedBy",commandOptions.signedBy, true); - putNestedIfNotEmpty(beanParams, "signature",commandOptions.signature, true); - putNestedIfNotEmpty(beanParams, "date",commandOptions.date, true); + putNestedIfNotEmpty(beanParams, "title", commandOptions.title, true); + putNestedIfNotEmpty(beanParams, "overview", commandOptions.overview, true); + putNestedIfNotEmpty(beanParams, "discussion.author", commandOptions.discussionAuthor, true); + putNestedIfNotEmpty(beanParams, "discussion.date", commandOptions.discussionDate, true); + putNestedIfNotEmpty(beanParams, "discussion.text", commandOptions.discussionText, true); + putNestedIfNotEmpty(beanParams, "logo", commandOptions.logo, true); + putNestedIfNotEmpty(beanParams, "signedBy", commandOptions.signedBy, true); + putNestedIfNotEmpty(beanParams, "signature", commandOptions.signature, true); + putNestedIfNotEmpty(beanParams, "date", commandOptions.date, true); clinicalReport = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 3427a69163c..316985784d5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -318,10 +318,10 @@ private RestResponse runCircos() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CircosAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "title",commandOptions.title, true); - putNestedIfNotEmpty(beanParams, "density",commandOptions.density, true); - putNestedIfNotNull(beanParams, "query",commandOptions.query, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "title", commandOptions.title, true); + putNestedIfNotEmpty(beanParams, "density", commandOptions.density, true); + putNestedMapIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); circosAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -389,11 +389,11 @@ private RestResponse runCohortStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortVariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); cohortVariantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -432,9 +432,9 @@ private RestResponse runExomiser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ExomiserWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysisType",commandOptions.clinicalAnalysisType, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysisType", commandOptions.clinicalAnalysisType, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); exomiserWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -476,93 +476,93 @@ private RestResponse runExport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantExportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotNull(beanParams, "panelIntersection",commandOptions.panelIntersection, true); - putNestedIfNotEmpty(beanParams, "panelFeatureType",commandOptions.panelFeatureType, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "project",commandOptions.bodyProject, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotEmpty(beanParams, "savedFilter",commandOptions.savedFilter, true); - putNestedIfNotEmpty(beanParams, "chromosome",commandOptions.chromosome, true); - putNestedIfNotEmpty(beanParams, "reference",commandOptions.reference, true); - putNestedIfNotEmpty(beanParams, "alternate",commandOptions.alternate, true); - putNestedIfNotEmpty(beanParams, "release",commandOptions.release, true); - putNestedIfNotEmpty(beanParams, "includeStudy",commandOptions.includeStudy, true); - putNestedIfNotEmpty(beanParams, "includeSample",commandOptions.includeSample, true); - putNestedIfNotEmpty(beanParams, "includeFile",commandOptions.includeFile, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotNull(beanParams, "includeSampleId",commandOptions.includeSampleId, true); - putNestedIfNotNull(beanParams, "includeGenotype",commandOptions.includeGenotype, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "fileData",commandOptions.fileData, true); - putNestedIfNotEmpty(beanParams, "genotype",commandOptions.genotype, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "sampleLimit",commandOptions.sampleLimit, true); - putNestedIfNotNull(beanParams, "sampleSkip",commandOptions.sampleSkip, true); - putNestedIfNotEmpty(beanParams, "sampleData",commandOptions.sampleData, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "familyMembers",commandOptions.familyMembers, true); - putNestedIfNotEmpty(beanParams, "familyDisorder",commandOptions.familyDisorder, true); - putNestedIfNotEmpty(beanParams, "familyProband",commandOptions.familyProband, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "cohortStatsPass",commandOptions.cohortStatsPass, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMgf",commandOptions.cohortStatsMgf, true); - putNestedIfNotEmpty(beanParams, "missingAlleles",commandOptions.missingAlleles, true); - putNestedIfNotEmpty(beanParams, "missingGenotypes",commandOptions.missingGenotypes, true); - putNestedIfNotNull(beanParams, "annotationExists",commandOptions.annotationExists, true); - putNestedIfNotEmpty(beanParams, "score",commandOptions.score, true); - putNestedIfNotEmpty(beanParams, "polyphen",commandOptions.polyphen, true); - putNestedIfNotEmpty(beanParams, "sift",commandOptions.sift, true); - putNestedIfNotEmpty(beanParams, "geneRoleInCancer",commandOptions.geneRoleInCancer, true); - putNestedIfNotEmpty(beanParams, "geneTraitId",commandOptions.geneTraitId, true); - putNestedIfNotEmpty(beanParams, "geneTraitName",commandOptions.geneTraitName, true); - putNestedIfNotEmpty(beanParams, "trait",commandOptions.trait, true); - putNestedIfNotEmpty(beanParams, "cosmic",commandOptions.cosmic, true); - putNestedIfNotEmpty(beanParams, "clinvar",commandOptions.clinvar, true); - putNestedIfNotEmpty(beanParams, "hpo",commandOptions.hpo, true); - putNestedIfNotEmpty(beanParams, "go",commandOptions.go, true); - putNestedIfNotEmpty(beanParams, "expression",commandOptions.expression, true); - putNestedIfNotEmpty(beanParams, "proteinKeyword",commandOptions.proteinKeyword, true); - putNestedIfNotEmpty(beanParams, "drug",commandOptions.drug, true); - putNestedIfNotEmpty(beanParams, "customAnnotation",commandOptions.customAnnotation, true); - putNestedIfNotEmpty(beanParams, "unknownGenotype",commandOptions.unknownGenotype, true); - putNestedIfNotNull(beanParams, "sampleMetadata",commandOptions.sampleMetadata, true); - putNestedIfNotNull(beanParams, "sort",commandOptions.sort, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotEmpty(beanParams, "outputFileFormat",commandOptions.outputFileFormat, true); - putNestedIfNotEmpty(beanParams, "variantsFile",commandOptions.variantsFile, true); - putNestedIfNotEmpty(beanParams, "include",commandOptions.bodyInclude, true); - putNestedIfNotEmpty(beanParams, "exclude",commandOptions.bodyExclude, true); - putNestedIfNotNull(beanParams, "limit",commandOptions.limit, true); - putNestedIfNotNull(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotNull(beanParams, "summary",commandOptions.summary, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotNull(beanParams, "panelIntersection", commandOptions.panelIntersection, true); + putNestedIfNotEmpty(beanParams, "panelFeatureType", commandOptions.panelFeatureType, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "project", commandOptions.bodyProject, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotEmpty(beanParams, "savedFilter", commandOptions.savedFilter, true); + putNestedIfNotEmpty(beanParams, "chromosome", commandOptions.chromosome, true); + putNestedIfNotEmpty(beanParams, "reference", commandOptions.reference, true); + putNestedIfNotEmpty(beanParams, "alternate", commandOptions.alternate, true); + putNestedIfNotEmpty(beanParams, "release", commandOptions.release, true); + putNestedIfNotEmpty(beanParams, "includeStudy", commandOptions.includeStudy, true); + putNestedIfNotEmpty(beanParams, "includeSample", commandOptions.includeSample, true); + putNestedIfNotEmpty(beanParams, "includeFile", commandOptions.includeFile, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotNull(beanParams, "includeSampleId", commandOptions.includeSampleId, true); + putNestedIfNotNull(beanParams, "includeGenotype", commandOptions.includeGenotype, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "fileData", commandOptions.fileData, true); + putNestedIfNotEmpty(beanParams, "genotype", commandOptions.genotype, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "sampleLimit", commandOptions.sampleLimit, true); + putNestedIfNotNull(beanParams, "sampleSkip", commandOptions.sampleSkip, true); + putNestedIfNotEmpty(beanParams, "sampleData", commandOptions.sampleData, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "familyMembers", commandOptions.familyMembers, true); + putNestedIfNotEmpty(beanParams, "familyDisorder", commandOptions.familyDisorder, true); + putNestedIfNotEmpty(beanParams, "familyProband", commandOptions.familyProband, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "cohortStatsPass", commandOptions.cohortStatsPass, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMgf", commandOptions.cohortStatsMgf, true); + putNestedIfNotEmpty(beanParams, "missingAlleles", commandOptions.missingAlleles, true); + putNestedIfNotEmpty(beanParams, "missingGenotypes", commandOptions.missingGenotypes, true); + putNestedIfNotNull(beanParams, "annotationExists", commandOptions.annotationExists, true); + putNestedIfNotEmpty(beanParams, "score", commandOptions.score, true); + putNestedIfNotEmpty(beanParams, "polyphen", commandOptions.polyphen, true); + putNestedIfNotEmpty(beanParams, "sift", commandOptions.sift, true); + putNestedIfNotEmpty(beanParams, "geneRoleInCancer", commandOptions.geneRoleInCancer, true); + putNestedIfNotEmpty(beanParams, "geneTraitId", commandOptions.geneTraitId, true); + putNestedIfNotEmpty(beanParams, "geneTraitName", commandOptions.geneTraitName, true); + putNestedIfNotEmpty(beanParams, "trait", commandOptions.trait, true); + putNestedIfNotEmpty(beanParams, "cosmic", commandOptions.cosmic, true); + putNestedIfNotEmpty(beanParams, "clinvar", commandOptions.clinvar, true); + putNestedIfNotEmpty(beanParams, "hpo", commandOptions.hpo, true); + putNestedIfNotEmpty(beanParams, "go", commandOptions.go, true); + putNestedIfNotEmpty(beanParams, "expression", commandOptions.expression, true); + putNestedIfNotEmpty(beanParams, "proteinKeyword", commandOptions.proteinKeyword, true); + putNestedIfNotEmpty(beanParams, "drug", commandOptions.drug, true); + putNestedIfNotEmpty(beanParams, "customAnnotation", commandOptions.customAnnotation, true); + putNestedIfNotEmpty(beanParams, "unknownGenotype", commandOptions.unknownGenotype, true); + putNestedIfNotNull(beanParams, "sampleMetadata", commandOptions.sampleMetadata, true); + putNestedIfNotNull(beanParams, "sort", commandOptions.sort, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotEmpty(beanParams, "outputFileFormat", commandOptions.outputFileFormat, true); + putNestedIfNotEmpty(beanParams, "variantsFile", commandOptions.variantsFile, true); + putNestedIfNotEmpty(beanParams, "include", commandOptions.bodyInclude, true); + putNestedIfNotEmpty(beanParams, "exclude", commandOptions.bodyExclude, true); + putNestedIfNotNull(beanParams, "limit", commandOptions.limit, true); + putNestedIfNotNull(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotNull(beanParams, "summary", commandOptions.summary, true); variantExportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -619,10 +619,10 @@ private RestResponse runFamilyQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "relatednessMethod",commandOptions.relatednessMethod, true); - putNestedIfNotEmpty(beanParams, "relatednessMaf",commandOptions.relatednessMaf, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "relatednessMethod", commandOptions.relatednessMethod, true); + putNestedIfNotEmpty(beanParams, "relatednessMaf", commandOptions.relatednessMaf, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); familyQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -684,9 +684,9 @@ private RestResponse runGatk() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GatkWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "gatkParams",commandOptions.gatkParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "gatkParams", commandOptions.gatkParams, true); gatkWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -725,11 +725,11 @@ private RestResponse runGenomePlot() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GenomePlotAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "configFile",commandOptions.configFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "configFile", commandOptions.configFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); genomePlotAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -768,18 +768,18 @@ private RestResponse runGwas() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GwasAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "phenotype",commandOptions.phenotype, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "indexScoreId",commandOptions.indexScoreId, true); - putNestedIfNotNull(beanParams, "method",commandOptions.method, true); - putNestedIfNotNull(beanParams, "fisherMode",commandOptions.fisherMode, true); - putNestedIfNotEmpty(beanParams, "caseCohort",commandOptions.caseCohort, true); - putNestedIfNotEmpty(beanParams, "caseCohortSamplesAnnotation",commandOptions.caseCohortSamplesAnnotation, true); - putNestedIfNotNull(beanParams, "caseCohortSamples",commandOptions.caseCohortSamples, true); - putNestedIfNotEmpty(beanParams, "controlCohort",commandOptions.controlCohort, true); - putNestedIfNotEmpty(beanParams, "controlCohortSamplesAnnotation",commandOptions.controlCohortSamplesAnnotation, true); - putNestedIfNotNull(beanParams, "controlCohortSamples",commandOptions.controlCohortSamples, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "phenotype", commandOptions.phenotype, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "indexScoreId", commandOptions.indexScoreId, true); + putNestedIfNotNull(beanParams, "method", commandOptions.method, true); + putNestedIfNotNull(beanParams, "fisherMode", commandOptions.fisherMode, true); + putNestedIfNotEmpty(beanParams, "caseCohort", commandOptions.caseCohort, true); + putNestedIfNotEmpty(beanParams, "caseCohortSamplesAnnotation", commandOptions.caseCohortSamplesAnnotation, true); + putNestedIfNotNull(beanParams, "caseCohortSamples", commandOptions.caseCohortSamples, true); + putNestedIfNotEmpty(beanParams, "controlCohort", commandOptions.controlCohort, true); + putNestedIfNotEmpty(beanParams, "controlCohortSamplesAnnotation", commandOptions.controlCohortSamplesAnnotation, true); + putNestedIfNotNull(beanParams, "controlCohortSamples", commandOptions.controlCohortSamples, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); gwasAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -818,19 +818,19 @@ private RestResponse runHrDetect() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), HRDetectAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "sampleId",commandOptions.sampleId, true); - putNestedIfNotEmpty(beanParams, "snvFittingId",commandOptions.snvFittingId, true); - putNestedIfNotEmpty(beanParams, "svFittingId",commandOptions.svFittingId, true); - putNestedIfNotEmpty(beanParams, "cnvQuery",commandOptions.cnvQuery, true); - putNestedIfNotEmpty(beanParams, "indelQuery",commandOptions.indelQuery, true); - putNestedIfNotEmpty(beanParams, "snv3CustomName",commandOptions.snv3CustomName, true); - putNestedIfNotEmpty(beanParams, "snv8CustomName",commandOptions.snv8CustomName, true); - putNestedIfNotEmpty(beanParams, "sv3CustomName",commandOptions.sv3CustomName, true); - putNestedIfNotEmpty(beanParams, "sv8CustomName",commandOptions.sv8CustomName, true); - putNestedIfNotNull(beanParams, "bootstrap",commandOptions.bootstrap, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sampleId", commandOptions.sampleId, true); + putNestedIfNotEmpty(beanParams, "snvFittingId", commandOptions.snvFittingId, true); + putNestedIfNotEmpty(beanParams, "svFittingId", commandOptions.svFittingId, true); + putNestedIfNotEmpty(beanParams, "cnvQuery", commandOptions.cnvQuery, true); + putNestedIfNotEmpty(beanParams, "indelQuery", commandOptions.indelQuery, true); + putNestedIfNotEmpty(beanParams, "snv3CustomName", commandOptions.snv3CustomName, true); + putNestedIfNotEmpty(beanParams, "snv8CustomName", commandOptions.snv8CustomName, true); + putNestedIfNotEmpty(beanParams, "sv3CustomName", commandOptions.sv3CustomName, true); + putNestedIfNotEmpty(beanParams, "sv8CustomName", commandOptions.sv8CustomName, true); + putNestedIfNotNull(beanParams, "bootstrap", commandOptions.bootstrap, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); hRDetectAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -869,36 +869,36 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "transform",commandOptions.transform, true); - putNestedIfNotNull(beanParams, "gvcf",commandOptions.gvcf, true); - putNestedIfNotNull(beanParams, "normalizationSkip",commandOptions.normalizationSkip, true); - putNestedIfNotEmpty(beanParams, "referenceGenome",commandOptions.referenceGenome, true); - putNestedIfNotEmpty(beanParams, "failOnMalformedLines",commandOptions.failOnMalformedLines, true); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "load",commandOptions.load, true); - putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); - putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); - putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "loadArchive",commandOptions.loadArchive, true); - putNestedIfNotEmpty(beanParams, "loadHomRef",commandOptions.loadHomRef, true); - putNestedIfNotEmpty(beanParams, "postLoadCheck",commandOptions.postLoadCheck, true); - putNestedIfNotEmpty(beanParams, "includeGenotypes",commandOptions.includeGenotypes, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotEmpty(beanParams, "merge",commandOptions.merge, true); - putNestedIfNotEmpty(beanParams, "deduplicationPolicy",commandOptions.deduplicationPolicy, true); - putNestedIfNotNull(beanParams, "calculateStats",commandOptions.calculateStats, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexSearch",commandOptions.indexSearch, true); - putNestedIfNotNull(beanParams, "skipIndexedFiles",commandOptions.skipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "transform", commandOptions.transform, true); + putNestedIfNotNull(beanParams, "gvcf", commandOptions.gvcf, true); + putNestedIfNotNull(beanParams, "normalizationSkip", commandOptions.normalizationSkip, true); + putNestedIfNotEmpty(beanParams, "referenceGenome", commandOptions.referenceGenome, true); + putNestedIfNotEmpty(beanParams, "failOnMalformedLines", commandOptions.failOnMalformedLines, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotNull(beanParams, "load", commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload", commandOptions.forceReload, true); + putNestedIfNotEmpty(beanParams, "loadSplitData", commandOptions.loadSplitData, true); + putNestedIfNotNull(beanParams, "loadMultiFileData", commandOptions.loadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "loadSampleIndex", commandOptions.loadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "loadArchive", commandOptions.loadArchive, true); + putNestedIfNotEmpty(beanParams, "loadHomRef", commandOptions.loadHomRef, true); + putNestedIfNotEmpty(beanParams, "postLoadCheck", commandOptions.postLoadCheck, true); + putNestedIfNotEmpty(beanParams, "includeGenotypes", commandOptions.includeGenotypes, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotEmpty(beanParams, "merge", commandOptions.merge, true); + putNestedIfNotEmpty(beanParams, "deduplicationPolicy", commandOptions.deduplicationPolicy, true); + putNestedIfNotNull(beanParams, "calculateStats", commandOptions.calculateStats, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexSearch", commandOptions.indexSearch, true); + putNestedIfNotNull(beanParams, "skipIndexedFiles", commandOptions.skipIndexedFiles, true); variantIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -937,10 +937,10 @@ private RestResponse runIndividualQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "inferredSexMethod",commandOptions.inferredSexMethod, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "inferredSexMethod", commandOptions.inferredSexMethod, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); individualQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -979,9 +979,9 @@ private RestResponse runInferredSex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InferredSexAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); inferredSexAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1054,16 +1054,16 @@ private RestResponse runKnockout() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), KnockoutAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotNull(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "consequenceType",commandOptions.consequenceType, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotNull(beanParams, "skipGenesFile",commandOptions.skipGenesFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotNull(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "consequenceType", commandOptions.consequenceType, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotNull(beanParams, "skipGenesFile", commandOptions.skipGenesFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); knockoutAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1102,10 +1102,10 @@ private RestResponse runMendelianError() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), MendelianErrorAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); mendelianErrorAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1197,22 +1197,22 @@ private RestResponse runMutationalSignature() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), MutationalSignatureAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "query",commandOptions.query, true); - putNestedIfNotEmpty(beanParams, "fitId",commandOptions.fitId, true); - putNestedIfNotEmpty(beanParams, "fitMethod",commandOptions.fitMethod, true); - putNestedIfNotNull(beanParams, "fitNBoot",commandOptions.fitNBoot, true); - putNestedIfNotEmpty(beanParams, "fitSigVersion",commandOptions.fitSigVersion, true); - putNestedIfNotEmpty(beanParams, "fitOrgan",commandOptions.fitOrgan, true); - putNestedIfNotNull(beanParams, "fitThresholdPerc",commandOptions.fitThresholdPerc, true); - putNestedIfNotNull(beanParams, "fitThresholdPval",commandOptions.fitThresholdPval, true); - putNestedIfNotNull(beanParams, "fitMaxRareSigs",commandOptions.fitMaxRareSigs, true); - putNestedIfNotEmpty(beanParams, "fitSignaturesFile",commandOptions.fitSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "fitRareSignaturesFile",commandOptions.fitRareSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotEmpty(beanParams, "fitId", commandOptions.fitId, true); + putNestedIfNotEmpty(beanParams, "fitMethod", commandOptions.fitMethod, true); + putNestedIfNotNull(beanParams, "fitNBoot", commandOptions.fitNBoot, true); + putNestedIfNotEmpty(beanParams, "fitSigVersion", commandOptions.fitSigVersion, true); + putNestedIfNotEmpty(beanParams, "fitOrgan", commandOptions.fitOrgan, true); + putNestedIfNotNull(beanParams, "fitThresholdPerc", commandOptions.fitThresholdPerc, true); + putNestedIfNotNull(beanParams, "fitThresholdPval", commandOptions.fitThresholdPval, true); + putNestedIfNotNull(beanParams, "fitMaxRareSigs", commandOptions.fitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "fitSignaturesFile", commandOptions.fitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "fitRareSignaturesFile", commandOptions.fitRareSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); mutationalSignatureAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1251,8 +1251,8 @@ private RestResponse runPlink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PlinkWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "plinkParams",commandOptions.plinkParams, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "plinkParams", commandOptions.plinkParams, true); plinkWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1381,11 +1381,11 @@ private RestResponse runRelatedness() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RelatednessAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "individuals",commandOptions.individuals, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "minorAlleleFreq",commandOptions.minorAlleleFreq, true); - putNestedIfNotEmpty(beanParams, "method",commandOptions.method, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "individuals", commandOptions.individuals, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "minorAlleleFreq", commandOptions.minorAlleleFreq, true); + putNestedIfNotEmpty(beanParams, "method", commandOptions.method, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); relatednessAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1424,9 +1424,9 @@ private RestResponse runRvtests() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RvtestsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "rvtestsParams",commandOptions.rvtestsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "rvtestsParams", commandOptions.rvtestsParams, true); rvtestsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1500,9 +1500,9 @@ private RestResponse runSampleEligibility() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleEligibilityAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "query",commandOptions.query, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "cohortId",commandOptions.cohortId, true); + putNestedIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "cohortId", commandOptions.cohortId, true); sampleEligibilityAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1541,53 +1541,53 @@ private RestResponse runSampleQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "vsId",commandOptions.vsId, true); - putNestedIfNotEmpty(beanParams, "vsDescription",commandOptions.vsDescription, true); - putNestedIfNotEmpty(beanParams, "vsQuery.id",commandOptions.vsQueryId, true); - putNestedIfNotEmpty(beanParams, "vsQuery.region",commandOptions.vsQueryRegion, true); - putNestedIfNotEmpty(beanParams, "vsQuery.gene",commandOptions.vsQueryGene, true); - putNestedIfNotEmpty(beanParams, "vsQuery.type",commandOptions.vsQueryType, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panel",commandOptions.vsQueryPanel, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelModeOfInheritance",commandOptions.vsQueryPanelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelConfidence",commandOptions.vsQueryPanelConfidence, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelRoleInCancer",commandOptions.vsQueryPanelRoleInCancer, true); - putNestedIfNotNull(beanParams, "vsQuery.panelIntersection",commandOptions.vsQueryPanelIntersection, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelFeatureType",commandOptions.vsQueryPanelFeatureType, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsRef",commandOptions.vsQueryCohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsAlt",commandOptions.vsQueryCohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsMaf",commandOptions.vsQueryCohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "vsQuery.ct",commandOptions.vsQueryCt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.xref",commandOptions.vsQueryXref, true); - putNestedIfNotEmpty(beanParams, "vsQuery.biotype",commandOptions.vsQueryBiotype, true); - putNestedIfNotEmpty(beanParams, "vsQuery.proteinSubstitution",commandOptions.vsQueryProteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "vsQuery.conservation",commandOptions.vsQueryConservation, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyMaf",commandOptions.vsQueryPopulationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyAlt",commandOptions.vsQueryPopulationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyRef",commandOptions.vsQueryPopulationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "vsQuery.transcriptFlag",commandOptions.vsQueryTranscriptFlag, true); - putNestedIfNotEmpty(beanParams, "vsQuery.functionalScore",commandOptions.vsQueryFunctionalScore, true); - putNestedIfNotEmpty(beanParams, "vsQuery.clinical",commandOptions.vsQueryClinical, true); - putNestedIfNotEmpty(beanParams, "vsQuery.clinicalSignificance",commandOptions.vsQueryClinicalSignificance, true); - putNestedIfNotNull(beanParams, "vsQuery.clinicalConfirmedStatus",commandOptions.vsQueryClinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "msId",commandOptions.msId, true); - putNestedIfNotEmpty(beanParams, "msDescription",commandOptions.msDescription, true); - putNestedIfNotEmpty(beanParams, "msQuery",commandOptions.msQuery, true); - putNestedIfNotEmpty(beanParams, "msFitId",commandOptions.msFitId, true); - putNestedIfNotEmpty(beanParams, "msFitMethod",commandOptions.msFitMethod, true); - putNestedIfNotNull(beanParams, "msFitNBoot",commandOptions.msFitNBoot, true); - putNestedIfNotEmpty(beanParams, "msFitSigVersion",commandOptions.msFitSigVersion, true); - putNestedIfNotEmpty(beanParams, "msFitOrgan",commandOptions.msFitOrgan, true); - putNestedIfNotNull(beanParams, "msFitThresholdPerc",commandOptions.msFitThresholdPerc, true); - putNestedIfNotNull(beanParams, "msFitThresholdPval",commandOptions.msFitThresholdPval, true); - putNestedIfNotNull(beanParams, "msFitMaxRareSigs",commandOptions.msFitMaxRareSigs, true); - putNestedIfNotEmpty(beanParams, "msFitSignaturesFile",commandOptions.msFitSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "msFitRareSignaturesFile",commandOptions.msFitRareSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "gpId",commandOptions.gpId, true); - putNestedIfNotEmpty(beanParams, "gpDescription",commandOptions.gpDescription, true); - putNestedIfNotEmpty(beanParams, "gpConfigFile",commandOptions.gpConfigFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "vsId", commandOptions.vsId, true); + putNestedIfNotEmpty(beanParams, "vsDescription", commandOptions.vsDescription, true); + putNestedIfNotEmpty(beanParams, "vsQuery.id", commandOptions.vsQueryId, true); + putNestedIfNotEmpty(beanParams, "vsQuery.region", commandOptions.vsQueryRegion, true); + putNestedIfNotEmpty(beanParams, "vsQuery.gene", commandOptions.vsQueryGene, true); + putNestedIfNotEmpty(beanParams, "vsQuery.type", commandOptions.vsQueryType, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panel", commandOptions.vsQueryPanel, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelModeOfInheritance", commandOptions.vsQueryPanelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelConfidence", commandOptions.vsQueryPanelConfidence, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelRoleInCancer", commandOptions.vsQueryPanelRoleInCancer, true); + putNestedIfNotNull(beanParams, "vsQuery.panelIntersection", commandOptions.vsQueryPanelIntersection, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelFeatureType", commandOptions.vsQueryPanelFeatureType, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsRef", commandOptions.vsQueryCohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsAlt", commandOptions.vsQueryCohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsMaf", commandOptions.vsQueryCohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "vsQuery.ct", commandOptions.vsQueryCt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.xref", commandOptions.vsQueryXref, true); + putNestedIfNotEmpty(beanParams, "vsQuery.biotype", commandOptions.vsQueryBiotype, true); + putNestedIfNotEmpty(beanParams, "vsQuery.proteinSubstitution", commandOptions.vsQueryProteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "vsQuery.conservation", commandOptions.vsQueryConservation, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyMaf", commandOptions.vsQueryPopulationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyAlt", commandOptions.vsQueryPopulationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyRef", commandOptions.vsQueryPopulationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "vsQuery.transcriptFlag", commandOptions.vsQueryTranscriptFlag, true); + putNestedIfNotEmpty(beanParams, "vsQuery.functionalScore", commandOptions.vsQueryFunctionalScore, true); + putNestedIfNotEmpty(beanParams, "vsQuery.clinical", commandOptions.vsQueryClinical, true); + putNestedIfNotEmpty(beanParams, "vsQuery.clinicalSignificance", commandOptions.vsQueryClinicalSignificance, true); + putNestedIfNotNull(beanParams, "vsQuery.clinicalConfirmedStatus", commandOptions.vsQueryClinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "msId", commandOptions.msId, true); + putNestedIfNotEmpty(beanParams, "msDescription", commandOptions.msDescription, true); + putNestedIfNotEmpty(beanParams, "msQuery", commandOptions.msQuery, true); + putNestedIfNotEmpty(beanParams, "msFitId", commandOptions.msFitId, true); + putNestedIfNotEmpty(beanParams, "msFitMethod", commandOptions.msFitMethod, true); + putNestedIfNotNull(beanParams, "msFitNBoot", commandOptions.msFitNBoot, true); + putNestedIfNotEmpty(beanParams, "msFitSigVersion", commandOptions.msFitSigVersion, true); + putNestedIfNotEmpty(beanParams, "msFitOrgan", commandOptions.msFitOrgan, true); + putNestedIfNotNull(beanParams, "msFitThresholdPerc", commandOptions.msFitThresholdPerc, true); + putNestedIfNotNull(beanParams, "msFitThresholdPval", commandOptions.msFitThresholdPval, true); + putNestedIfNotNull(beanParams, "msFitMaxRareSigs", commandOptions.msFitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "msFitSignaturesFile", commandOptions.msFitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "msFitRareSignaturesFile", commandOptions.msFitRareSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "gpId", commandOptions.gpId, true); + putNestedIfNotEmpty(beanParams, "gpDescription", commandOptions.gpDescription, true); + putNestedIfNotEmpty(beanParams, "gpConfigFile", commandOptions.gpConfigFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); sampleQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1644,36 +1644,36 @@ private RestResponse runSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleVariantFilterParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotNull(beanParams, "panelIntersection",commandOptions.panelIntersection, true); - putNestedIfNotEmpty(beanParams, "panelFeatureType",commandOptions.panelFeatureType, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotNull(beanParams, "genotypes",commandOptions.genotypes, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "samplesInAllVariants",commandOptions.samplesInAllVariants, true); - putNestedIfNotNull(beanParams, "maxVariants",commandOptions.maxVariants, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotNull(beanParams, "panelIntersection", commandOptions.panelIntersection, true); + putNestedIfNotEmpty(beanParams, "panelFeatureType", commandOptions.panelFeatureType, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotNull(beanParams, "genotypes", commandOptions.genotypes, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "samplesInAllVariants", commandOptions.samplesInAllVariants, true); + putNestedIfNotNull(beanParams, "maxVariants", commandOptions.maxVariants, true); sampleVariantFilterParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1740,42 +1740,42 @@ private RestResponse runSampleStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleVariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "variantQuery.id",commandOptions.variantQueryId, true); - putNestedIfNotEmpty(beanParams, "variantQuery.region",commandOptions.variantQueryRegion, true); - putNestedIfNotEmpty(beanParams, "variantQuery.gene",commandOptions.variantQueryGene, true); - putNestedIfNotEmpty(beanParams, "variantQuery.type",commandOptions.variantQueryType, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panel",commandOptions.variantQueryPanel, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelModeOfInheritance",commandOptions.variantQueryPanelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelConfidence",commandOptions.variantQueryPanelConfidence, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelRoleInCancer",commandOptions.variantQueryPanelRoleInCancer, true); - putNestedIfNotNull(beanParams, "variantQuery.panelIntersection",commandOptions.variantQueryPanelIntersection, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelFeatureType",commandOptions.variantQueryPanelFeatureType, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsRef",commandOptions.variantQueryCohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsAlt",commandOptions.variantQueryCohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsMaf",commandOptions.variantQueryCohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "variantQuery.ct",commandOptions.variantQueryCt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.xref",commandOptions.variantQueryXref, true); - putNestedIfNotEmpty(beanParams, "variantQuery.biotype",commandOptions.variantQueryBiotype, true); - putNestedIfNotEmpty(beanParams, "variantQuery.proteinSubstitution",commandOptions.variantQueryProteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "variantQuery.conservation",commandOptions.variantQueryConservation, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyMaf",commandOptions.variantQueryPopulationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyAlt",commandOptions.variantQueryPopulationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyRef",commandOptions.variantQueryPopulationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "variantQuery.transcriptFlag",commandOptions.variantQueryTranscriptFlag, true); - putNestedIfNotEmpty(beanParams, "variantQuery.functionalScore",commandOptions.variantQueryFunctionalScore, true); - putNestedIfNotEmpty(beanParams, "variantQuery.clinical",commandOptions.variantQueryClinical, true); - putNestedIfNotEmpty(beanParams, "variantQuery.clinicalSignificance",commandOptions.variantQueryClinicalSignificance, true); - putNestedIfNotNull(beanParams, "variantQuery.clinicalConfirmedStatus",commandOptions.variantQueryClinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "variantQuery.sampleData",commandOptions.variantQuerySampleData, true); - putNestedIfNotEmpty(beanParams, "variantQuery.fileData",commandOptions.variantQueryFileData, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotNull(beanParams, "indexOverwrite",commandOptions.indexOverwrite, true); - putNestedIfNotEmpty(beanParams, "indexId",commandOptions.indexId, true); - putNestedIfNotEmpty(beanParams, "indexDescription",commandOptions.indexDescription, true); - putNestedIfNotNull(beanParams, "batchSize",commandOptions.batchSize, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "variantQuery.id", commandOptions.variantQueryId, true); + putNestedIfNotEmpty(beanParams, "variantQuery.region", commandOptions.variantQueryRegion, true); + putNestedIfNotEmpty(beanParams, "variantQuery.gene", commandOptions.variantQueryGene, true); + putNestedIfNotEmpty(beanParams, "variantQuery.type", commandOptions.variantQueryType, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panel", commandOptions.variantQueryPanel, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelModeOfInheritance", commandOptions.variantQueryPanelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelConfidence", commandOptions.variantQueryPanelConfidence, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelRoleInCancer", commandOptions.variantQueryPanelRoleInCancer, true); + putNestedIfNotNull(beanParams, "variantQuery.panelIntersection", commandOptions.variantQueryPanelIntersection, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelFeatureType", commandOptions.variantQueryPanelFeatureType, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsRef", commandOptions.variantQueryCohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsAlt", commandOptions.variantQueryCohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsMaf", commandOptions.variantQueryCohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "variantQuery.ct", commandOptions.variantQueryCt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.xref", commandOptions.variantQueryXref, true); + putNestedIfNotEmpty(beanParams, "variantQuery.biotype", commandOptions.variantQueryBiotype, true); + putNestedIfNotEmpty(beanParams, "variantQuery.proteinSubstitution", commandOptions.variantQueryProteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "variantQuery.conservation", commandOptions.variantQueryConservation, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyMaf", commandOptions.variantQueryPopulationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyAlt", commandOptions.variantQueryPopulationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyRef", commandOptions.variantQueryPopulationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "variantQuery.transcriptFlag", commandOptions.variantQueryTranscriptFlag, true); + putNestedIfNotEmpty(beanParams, "variantQuery.functionalScore", commandOptions.variantQueryFunctionalScore, true); + putNestedIfNotEmpty(beanParams, "variantQuery.clinical", commandOptions.variantQueryClinical, true); + putNestedIfNotEmpty(beanParams, "variantQuery.clinicalSignificance", commandOptions.variantQueryClinicalSignificance, true); + putNestedIfNotNull(beanParams, "variantQuery.clinicalConfirmedStatus", commandOptions.variantQueryClinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "variantQuery.sampleData", commandOptions.variantQuerySampleData, true); + putNestedIfNotEmpty(beanParams, "variantQuery.fileData", commandOptions.variantQueryFileData, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotNull(beanParams, "indexOverwrite", commandOptions.indexOverwrite, true); + putNestedIfNotEmpty(beanParams, "indexId", commandOptions.indexId, true); + putNestedIfNotEmpty(beanParams, "indexDescription", commandOptions.indexDescription, true); + putNestedIfNotNull(beanParams, "batchSize", commandOptions.batchSize, true); sampleVariantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1815,11 +1815,11 @@ private RestResponse runStatsExport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsExportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohorts",commandOptions.cohorts, true); - putNestedIfNotEmpty(beanParams, "output",commandOptions.output, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "outputFileFormat",commandOptions.outputFileFormat, true); + putNestedIfNotNull(beanParams, "cohorts", commandOptions.cohorts, true); + putNestedIfNotEmpty(beanParams, "output", commandOptions.output, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "outputFileFormat", commandOptions.outputFileFormat, true); variantStatsExportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1858,14 +1858,14 @@ private RestResponse runStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); variantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java index b240857de8b..287c2223a00 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java @@ -128,8 +128,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); cohortAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -163,7 +163,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -200,16 +200,16 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); cohortCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -285,16 +285,16 @@ private RestResponse generate() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortGenerateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.bodyId, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.bodyCreationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.bodyModificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.bodyId, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.bodyCreationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.bodyModificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); cohortGenerateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -411,16 +411,16 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); cohortUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java index 00dbbbc130f..d2fde239e71 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java @@ -117,8 +117,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); panelAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -153,17 +153,17 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "author",commandOptions.author, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.version",commandOptions.sourceVersion, true); - putNestedIfNotEmpty(beanParams, "source.author",commandOptions.sourceAuthor, true); - putNestedIfNotEmpty(beanParams, "source.project",commandOptions.sourceProject, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "author", commandOptions.author, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.version", commandOptions.sourceVersion, true); + putNestedIfNotEmpty(beanParams, "source.author", commandOptions.sourceAuthor, true); + putNestedIfNotEmpty(beanParams, "source.project", commandOptions.sourceProject, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); panelCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -234,8 +234,8 @@ private RestResponse importPanels() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelImportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "source",commandOptions.source, true); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "source", commandOptions.source, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); panelImportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -355,17 +355,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "author",commandOptions.author, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.version",commandOptions.sourceVersion, true); - putNestedIfNotEmpty(beanParams, "source.author",commandOptions.sourceAuthor, true); - putNestedIfNotEmpty(beanParams, "source.project",commandOptions.sourceProject, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "author", commandOptions.author, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.version", commandOptions.sourceVersion, true); + putNestedIfNotEmpty(beanParams, "source.author", commandOptions.sourceAuthor, true); + putNestedIfNotEmpty(beanParams, "source.project", commandOptions.sourceProject, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); panelUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java index 5e93767f51c..dc5a1878e40 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java @@ -126,10 +126,10 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); familyAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -163,7 +163,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -199,16 +199,16 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "expectedSize",commandOptions.expectedSize, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "expectedSize", commandOptions.expectedSize, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); familyCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -362,17 +362,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "expectedSize",commandOptions.expectedSize, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "expectedSize", commandOptions.expectedSize, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); familyUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java index f9f6f5fedb8..873a16baac3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java @@ -197,9 +197,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); fileAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -233,7 +233,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -273,27 +273,27 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotNull(beanParams, "format",commandOptions.format, true); - putNestedIfNotNull(beanParams, "bioformat",commandOptions.bioformat, true); - putNestedIfNotNull(beanParams, "sampleIds",commandOptions.sampleIds, true); - putNestedIfNotEmpty(beanParams, "software.name",commandOptions.softwareName, true); - putNestedIfNotEmpty(beanParams, "software.version",commandOptions.softwareVersion, true); - putNestedIfNotEmpty(beanParams, "software.repository",commandOptions.softwareRepository, true); - putNestedIfNotEmpty(beanParams, "software.commit",commandOptions.softwareCommit, true); - putNestedIfNotEmpty(beanParams, "software.website",commandOptions.softwareWebsite, true); - putNestedIfNotNull(beanParams, "software.params",commandOptions.softwareParams, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "jobId",commandOptions.jobId, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "format", commandOptions.format, true); + putNestedIfNotNull(beanParams, "bioformat", commandOptions.bioformat, true); + putNestedIfNotNull(beanParams, "sampleIds", commandOptions.sampleIds, true); + putNestedIfNotEmpty(beanParams, "software.name", commandOptions.softwareName, true); + putNestedIfNotEmpty(beanParams, "software.version", commandOptions.softwareVersion, true); + putNestedIfNotEmpty(beanParams, "software.repository", commandOptions.softwareRepository, true); + putNestedIfNotEmpty(beanParams, "software.commit", commandOptions.softwareCommit, true); + putNestedIfNotEmpty(beanParams, "software.website", commandOptions.softwareWebsite, true); + putNestedMapIfNotEmpty(beanParams, "software.params", commandOptions.softwareParams, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "jobId", commandOptions.jobId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); fileCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -371,8 +371,8 @@ private RestResponse fetch() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileFetch.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "url",commandOptions.url, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "url", commandOptions.url, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); fileFetch = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -412,16 +412,16 @@ private RestResponse link() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileLinkParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "uri",commandOptions.uri, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "virtualFileName",commandOptions.virtualFileName, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "internal.sampleMap",commandOptions.internalSampleMap, true); + putNestedIfNotEmpty(beanParams, "uri", commandOptions.uri, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "virtualFileName", commandOptions.virtualFileName, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "internal.sampleMap", commandOptions.internalSampleMap, true); fileLinkParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -460,12 +460,12 @@ private RestResponse runLink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileLinkToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "uri",commandOptions.uri, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "virtualFileName",commandOptions.virtualFileName, true); - putNestedIfNotNull(beanParams, "parents",commandOptions.parents, true); - putNestedIfNotNull(beanParams, "skipPostLink",commandOptions.skipPostLink, true); + putNestedIfNotNull(beanParams, "uri", commandOptions.uri, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "virtualFileName", commandOptions.virtualFileName, true); + putNestedIfNotNull(beanParams, "parents", commandOptions.parents, true); + putNestedIfNotNull(beanParams, "skipPostLink", commandOptions.skipPostLink, true); fileLinkToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -504,8 +504,8 @@ private RestResponse runPostlink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PostLinkToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "files",commandOptions.files, true); - putNestedIfNotNull(beanParams, "batchSize",commandOptions.batchSize, true); + putNestedIfNotNull(beanParams, "files", commandOptions.files, true); + putNestedIfNotNull(beanParams, "batchSize", commandOptions.batchSize, true); postLinkToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -669,37 +669,37 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "sampleIds",commandOptions.sampleIds, true); - putNestedIfNotNull(beanParams, "format",commandOptions.format, true); - putNestedIfNotNull(beanParams, "bioformat",commandOptions.bioformat, true); - putNestedIfNotEmpty(beanParams, "software.name",commandOptions.softwareName, true); - putNestedIfNotEmpty(beanParams, "software.version",commandOptions.softwareVersion, true); - putNestedIfNotEmpty(beanParams, "software.repository",commandOptions.softwareRepository, true); - putNestedIfNotEmpty(beanParams, "software.commit",commandOptions.softwareCommit, true); - putNestedIfNotEmpty(beanParams, "software.website",commandOptions.softwareWebsite, true); - putNestedIfNotNull(beanParams, "software.params",commandOptions.softwareParams, true); - putNestedIfNotNull(beanParams, "experiment.technology",commandOptions.experimentTechnology, true); - putNestedIfNotNull(beanParams, "experiment.method",commandOptions.experimentMethod, true); - putNestedIfNotNull(beanParams, "experiment.nucleicAcidType",commandOptions.experimentNucleicAcidType, true); - putNestedIfNotEmpty(beanParams, "experiment.manufacturer",commandOptions.experimentManufacturer, true); - putNestedIfNotEmpty(beanParams, "experiment.platform",commandOptions.experimentPlatform, true); - putNestedIfNotEmpty(beanParams, "experiment.library",commandOptions.experimentLibrary, true); - putNestedIfNotEmpty(beanParams, "experiment.date",commandOptions.experimentDate, true); - putNestedIfNotEmpty(beanParams, "experiment.center",commandOptions.experimentCenter, true); - putNestedIfNotEmpty(beanParams, "experiment.lab",commandOptions.experimentLab, true); - putNestedIfNotEmpty(beanParams, "experiment.responsible",commandOptions.experimentResponsible, true); - putNestedIfNotEmpty(beanParams, "experiment.description",commandOptions.experimentDescription, true); - putNestedIfNotNull(beanParams, "experiment.attributes",commandOptions.experimentAttributes, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "stats",commandOptions.stats, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "sampleIds", commandOptions.sampleIds, true); + putNestedIfNotNull(beanParams, "format", commandOptions.format, true); + putNestedIfNotNull(beanParams, "bioformat", commandOptions.bioformat, true); + putNestedIfNotEmpty(beanParams, "software.name", commandOptions.softwareName, true); + putNestedIfNotEmpty(beanParams, "software.version", commandOptions.softwareVersion, true); + putNestedIfNotEmpty(beanParams, "software.repository", commandOptions.softwareRepository, true); + putNestedIfNotEmpty(beanParams, "software.commit", commandOptions.softwareCommit, true); + putNestedIfNotEmpty(beanParams, "software.website", commandOptions.softwareWebsite, true); + putNestedMapIfNotEmpty(beanParams, "software.params", commandOptions.softwareParams, true); + putNestedIfNotNull(beanParams, "experiment.technology", commandOptions.experimentTechnology, true); + putNestedIfNotNull(beanParams, "experiment.method", commandOptions.experimentMethod, true); + putNestedIfNotNull(beanParams, "experiment.nucleicAcidType", commandOptions.experimentNucleicAcidType, true); + putNestedIfNotEmpty(beanParams, "experiment.manufacturer", commandOptions.experimentManufacturer, true); + putNestedIfNotEmpty(beanParams, "experiment.platform", commandOptions.experimentPlatform, true); + putNestedIfNotEmpty(beanParams, "experiment.library", commandOptions.experimentLibrary, true); + putNestedIfNotEmpty(beanParams, "experiment.date", commandOptions.experimentDate, true); + putNestedIfNotEmpty(beanParams, "experiment.center", commandOptions.experimentCenter, true); + putNestedIfNotEmpty(beanParams, "experiment.lab", commandOptions.experimentLab, true); + putNestedIfNotEmpty(beanParams, "experiment.responsible", commandOptions.experimentResponsible, true); + putNestedIfNotEmpty(beanParams, "experiment.description", commandOptions.experimentDescription, true); + putNestedMapIfNotEmpty(beanParams, "experiment.attributes", commandOptions.experimentAttributes, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedMapIfNotEmpty(beanParams, "stats", commandOptions.stats, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); fileUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -820,7 +820,7 @@ private RestResponse move() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileMoveParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); fileMoveParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java index 495341543d6..dcf24216d8d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java @@ -135,9 +135,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); individualAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -171,7 +171,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -207,42 +207,42 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "father.id",commandOptions.fatherId, true); - putNestedIfNotEmpty(beanParams, "father.uuid",commandOptions.fatherUuid, true); - putNestedIfNotEmpty(beanParams, "mother.id",commandOptions.motherId, true); - putNestedIfNotEmpty(beanParams, "mother.uuid",commandOptions.motherUuid, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "location.address",commandOptions.locationAddress, true); - putNestedIfNotEmpty(beanParams, "location.postalCode",commandOptions.locationPostalCode, true); - putNestedIfNotEmpty(beanParams, "location.city",commandOptions.locationCity, true); - putNestedIfNotEmpty(beanParams, "location.state",commandOptions.locationState, true); - putNestedIfNotEmpty(beanParams, "location.country",commandOptions.locationCountry, true); - putNestedIfNotEmpty(beanParams, "sex.id",commandOptions.sexId, true); - putNestedIfNotEmpty(beanParams, "sex.name",commandOptions.sexName, true); - putNestedIfNotEmpty(beanParams, "sex.description",commandOptions.sexDescription, true); - putNestedIfNotEmpty(beanParams, "sex.source",commandOptions.sexSource, true); - putNestedIfNotEmpty(beanParams, "sex.url",commandOptions.sexUrl, true); - putNestedIfNotNull(beanParams, "sex.attributes",commandOptions.sexAttributes, true); - putNestedIfNotEmpty(beanParams, "ethnicity.id",commandOptions.ethnicityId, true); - putNestedIfNotEmpty(beanParams, "ethnicity.name",commandOptions.ethnicityName, true); - putNestedIfNotEmpty(beanParams, "ethnicity.description",commandOptions.ethnicityDescription, true); - putNestedIfNotEmpty(beanParams, "ethnicity.source",commandOptions.ethnicitySource, true); - putNestedIfNotEmpty(beanParams, "ethnicity.url",commandOptions.ethnicityUrl, true); - putNestedIfNotNull(beanParams, "ethnicity.attributes",commandOptions.ethnicityAttributes, true); - putNestedIfNotNull(beanParams, "parentalConsanguinity",commandOptions.parentalConsanguinity, true); - putNestedIfNotEmpty(beanParams, "population.name",commandOptions.populationName, true); - putNestedIfNotEmpty(beanParams, "population.subpopulation",commandOptions.populationSubpopulation, true); - putNestedIfNotEmpty(beanParams, "population.description",commandOptions.populationDescription, true); - putNestedIfNotEmpty(beanParams, "dateOfBirth",commandOptions.dateOfBirth, true); - putNestedIfNotNull(beanParams, "karyotypicSex",commandOptions.karyotypicSex, true); - putNestedIfNotNull(beanParams, "lifeStatus",commandOptions.lifeStatus, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "father.id", commandOptions.fatherId, true); + putNestedIfNotEmpty(beanParams, "father.uuid", commandOptions.fatherUuid, true); + putNestedIfNotEmpty(beanParams, "mother.id", commandOptions.motherId, true); + putNestedIfNotEmpty(beanParams, "mother.uuid", commandOptions.motherUuid, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "location.address", commandOptions.locationAddress, true); + putNestedIfNotEmpty(beanParams, "location.postalCode", commandOptions.locationPostalCode, true); + putNestedIfNotEmpty(beanParams, "location.city", commandOptions.locationCity, true); + putNestedIfNotEmpty(beanParams, "location.state", commandOptions.locationState, true); + putNestedIfNotEmpty(beanParams, "location.country", commandOptions.locationCountry, true); + putNestedIfNotEmpty(beanParams, "sex.id", commandOptions.sexId, true); + putNestedIfNotEmpty(beanParams, "sex.name", commandOptions.sexName, true); + putNestedIfNotEmpty(beanParams, "sex.description", commandOptions.sexDescription, true); + putNestedIfNotEmpty(beanParams, "sex.source", commandOptions.sexSource, true); + putNestedIfNotEmpty(beanParams, "sex.url", commandOptions.sexUrl, true); + putNestedMapIfNotEmpty(beanParams, "sex.attributes", commandOptions.sexAttributes, true); + putNestedIfNotEmpty(beanParams, "ethnicity.id", commandOptions.ethnicityId, true); + putNestedIfNotEmpty(beanParams, "ethnicity.name", commandOptions.ethnicityName, true); + putNestedIfNotEmpty(beanParams, "ethnicity.description", commandOptions.ethnicityDescription, true); + putNestedIfNotEmpty(beanParams, "ethnicity.source", commandOptions.ethnicitySource, true); + putNestedIfNotEmpty(beanParams, "ethnicity.url", commandOptions.ethnicityUrl, true); + putNestedMapIfNotEmpty(beanParams, "ethnicity.attributes", commandOptions.ethnicityAttributes, true); + putNestedIfNotNull(beanParams, "parentalConsanguinity", commandOptions.parentalConsanguinity, true); + putNestedIfNotEmpty(beanParams, "population.name", commandOptions.populationName, true); + putNestedIfNotEmpty(beanParams, "population.subpopulation", commandOptions.populationSubpopulation, true); + putNestedIfNotEmpty(beanParams, "population.description", commandOptions.populationDescription, true); + putNestedIfNotEmpty(beanParams, "dateOfBirth", commandOptions.dateOfBirth, true); + putNestedIfNotNull(beanParams, "karyotypicSex", commandOptions.karyotypicSex, true); + putNestedIfNotNull(beanParams, "lifeStatus", commandOptions.lifeStatus, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); individualCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -411,43 +411,43 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "father.id",commandOptions.fatherId, true); - putNestedIfNotEmpty(beanParams, "father.uuid",commandOptions.fatherUuid, true); - putNestedIfNotEmpty(beanParams, "mother.id",commandOptions.motherId, true); - putNestedIfNotEmpty(beanParams, "mother.uuid",commandOptions.motherUuid, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "parentalConsanguinity",commandOptions.parentalConsanguinity, true); - putNestedIfNotEmpty(beanParams, "location.address",commandOptions.locationAddress, true); - putNestedIfNotEmpty(beanParams, "location.postalCode",commandOptions.locationPostalCode, true); - putNestedIfNotEmpty(beanParams, "location.city",commandOptions.locationCity, true); - putNestedIfNotEmpty(beanParams, "location.state",commandOptions.locationState, true); - putNestedIfNotEmpty(beanParams, "location.country",commandOptions.locationCountry, true); - putNestedIfNotEmpty(beanParams, "sex.id",commandOptions.sexId, true); - putNestedIfNotEmpty(beanParams, "sex.name",commandOptions.sexName, true); - putNestedIfNotEmpty(beanParams, "sex.description",commandOptions.sexDescription, true); - putNestedIfNotEmpty(beanParams, "sex.source",commandOptions.sexSource, true); - putNestedIfNotEmpty(beanParams, "sex.url",commandOptions.sexUrl, true); - putNestedIfNotNull(beanParams, "sex.attributes",commandOptions.sexAttributes, true); - putNestedIfNotEmpty(beanParams, "ethnicity.id",commandOptions.ethnicityId, true); - putNestedIfNotEmpty(beanParams, "ethnicity.name",commandOptions.ethnicityName, true); - putNestedIfNotEmpty(beanParams, "ethnicity.description",commandOptions.ethnicityDescription, true); - putNestedIfNotEmpty(beanParams, "ethnicity.source",commandOptions.ethnicitySource, true); - putNestedIfNotEmpty(beanParams, "ethnicity.url",commandOptions.ethnicityUrl, true); - putNestedIfNotNull(beanParams, "ethnicity.attributes",commandOptions.ethnicityAttributes, true); - putNestedIfNotEmpty(beanParams, "population.name",commandOptions.populationName, true); - putNestedIfNotEmpty(beanParams, "population.subpopulation",commandOptions.populationSubpopulation, true); - putNestedIfNotEmpty(beanParams, "population.description",commandOptions.populationDescription, true); - putNestedIfNotEmpty(beanParams, "dateOfBirth",commandOptions.dateOfBirth, true); - putNestedIfNotNull(beanParams, "karyotypicSex",commandOptions.karyotypicSex, true); - putNestedIfNotNull(beanParams, "lifeStatus",commandOptions.lifeStatus, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "father.id", commandOptions.fatherId, true); + putNestedIfNotEmpty(beanParams, "father.uuid", commandOptions.fatherUuid, true); + putNestedIfNotEmpty(beanParams, "mother.id", commandOptions.motherId, true); + putNestedIfNotEmpty(beanParams, "mother.uuid", commandOptions.motherUuid, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "parentalConsanguinity", commandOptions.parentalConsanguinity, true); + putNestedIfNotEmpty(beanParams, "location.address", commandOptions.locationAddress, true); + putNestedIfNotEmpty(beanParams, "location.postalCode", commandOptions.locationPostalCode, true); + putNestedIfNotEmpty(beanParams, "location.city", commandOptions.locationCity, true); + putNestedIfNotEmpty(beanParams, "location.state", commandOptions.locationState, true); + putNestedIfNotEmpty(beanParams, "location.country", commandOptions.locationCountry, true); + putNestedIfNotEmpty(beanParams, "sex.id", commandOptions.sexId, true); + putNestedIfNotEmpty(beanParams, "sex.name", commandOptions.sexName, true); + putNestedIfNotEmpty(beanParams, "sex.description", commandOptions.sexDescription, true); + putNestedIfNotEmpty(beanParams, "sex.source", commandOptions.sexSource, true); + putNestedIfNotEmpty(beanParams, "sex.url", commandOptions.sexUrl, true); + putNestedMapIfNotEmpty(beanParams, "sex.attributes", commandOptions.sexAttributes, true); + putNestedIfNotEmpty(beanParams, "ethnicity.id", commandOptions.ethnicityId, true); + putNestedIfNotEmpty(beanParams, "ethnicity.name", commandOptions.ethnicityName, true); + putNestedIfNotEmpty(beanParams, "ethnicity.description", commandOptions.ethnicityDescription, true); + putNestedIfNotEmpty(beanParams, "ethnicity.source", commandOptions.ethnicitySource, true); + putNestedIfNotEmpty(beanParams, "ethnicity.url", commandOptions.ethnicityUrl, true); + putNestedMapIfNotEmpty(beanParams, "ethnicity.attributes", commandOptions.ethnicityAttributes, true); + putNestedIfNotEmpty(beanParams, "population.name", commandOptions.populationName, true); + putNestedIfNotEmpty(beanParams, "population.subpopulation", commandOptions.populationSubpopulation, true); + putNestedIfNotEmpty(beanParams, "population.description", commandOptions.populationDescription, true); + putNestedIfNotEmpty(beanParams, "dateOfBirth", commandOptions.dateOfBirth, true); + putNestedIfNotNull(beanParams, "karyotypicSex", commandOptions.karyotypicSex, true); + putNestedIfNotNull(beanParams, "lifeStatus", commandOptions.lifeStatus, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); individualUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index cf28e7983bf..ffc4c7a61a1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -137,8 +137,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "job",commandOptions.job, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "job", commandOptions.job, true); jobAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -170,25 +170,25 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "tool.id",commandOptions.toolId, true); - putNestedIfNotEmpty(beanParams, "tool.description",commandOptions.toolDescription, true); - putNestedIfNotNull(beanParams, "tool.scope",commandOptions.toolScope, true); - putNestedIfNotNull(beanParams, "tool.type",commandOptions.toolType, true); - putNestedIfNotNull(beanParams, "tool.resource",commandOptions.toolResource, true); - putNestedIfNotNull(beanParams, "priority",commandOptions.priority, true); - putNestedIfNotEmpty(beanParams, "commandLine",commandOptions.commandLine, true); - putNestedIfNotNull(beanParams, "params",commandOptions.params, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "outDir.path",commandOptions.outDirPath, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "result.id",commandOptions.resultId, true); - putNestedIfNotNull(beanParams, "result.attributes",commandOptions.resultAttributes, true); - putNestedIfNotEmpty(beanParams, "stdout.path",commandOptions.stdoutPath, true); - putNestedIfNotEmpty(beanParams, "stderr.path",commandOptions.stderrPath, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "tool.id", commandOptions.toolId, true); + putNestedIfNotEmpty(beanParams, "tool.description", commandOptions.toolDescription, true); + putNestedIfNotNull(beanParams, "tool.scope", commandOptions.toolScope, true); + putNestedIfNotNull(beanParams, "tool.type", commandOptions.toolType, true); + putNestedIfNotNull(beanParams, "tool.resource", commandOptions.toolResource, true); + putNestedIfNotNull(beanParams, "priority", commandOptions.priority, true); + putNestedIfNotEmpty(beanParams, "commandLine", commandOptions.commandLine, true); + putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "outDir.path", commandOptions.outDirPath, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "result.id", commandOptions.resultId, true); + putNestedMapIfNotEmpty(beanParams, "result.attributes", commandOptions.resultAttributes, true); + putNestedIfNotEmpty(beanParams, "stdout.path", commandOptions.stdoutPath, true); + putNestedIfNotEmpty(beanParams, "stderr.path", commandOptions.stderrPath, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); jobCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -257,9 +257,9 @@ private RestResponse retry() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobRetryParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "job",commandOptions.job, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); - putNestedIfNotNull(beanParams, "params",commandOptions.params, true); + putNestedIfNotEmpty(beanParams, "job", commandOptions.job, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); + putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); jobRetryParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -392,10 +392,10 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visited",commandOptions.visited, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visited", commandOptions.visited, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); jobUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 6135886595d..6f351a6b38a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -189,10 +189,10 @@ private RestResponse configureCellbase() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CellBaseConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "url",commandOptions.url, true); - putNestedIfNotEmpty(beanParams, "version",commandOptions.version, true); - putNestedIfNotEmpty(beanParams, "dataRelease",commandOptions.dataRelease, true); - putNestedIfNotEmpty(beanParams, "apiKey",commandOptions.apiKey, true); + putNestedIfNotEmpty(beanParams, "url", commandOptions.url, true); + putNestedIfNotEmpty(beanParams, "version", commandOptions.version, true); + putNestedIfNotEmpty(beanParams, "dataRelease", commandOptions.dataRelease, true); + putNestedIfNotEmpty(beanParams, "apiKey", commandOptions.apiKey, true); cellBaseConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -231,8 +231,8 @@ private RestResponse aggregateVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAggregateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantAggregateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -291,15 +291,15 @@ private RestResponse indexVariantAnnotation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "create",commandOptions.create, true); - putNestedIfNotEmpty(beanParams, "load",commandOptions.load, true); - putNestedIfNotEmpty(beanParams, "customName",commandOptions.customName, true); - putNestedIfNotNull(beanParams, "sampleIndexAnnotation",commandOptions.sampleIndexAnnotation, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "create", commandOptions.create, true); + putNestedIfNotEmpty(beanParams, "load", commandOptions.load, true); + putNestedIfNotEmpty(beanParams, "customName", commandOptions.customName, true); + putNestedIfNotNull(beanParams, "sampleIndexAnnotation", commandOptions.sampleIndexAnnotation, true); variantAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -335,7 +335,7 @@ private RestResponse saveVariantAnnotation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAnnotationSaveParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "annotationId",commandOptions.annotationId, true); + putNestedIfNotEmpty(beanParams, "annotationId", commandOptions.annotationId, true); variantAnnotationSaveParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -368,7 +368,7 @@ private RestResponse configureVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantConfigureParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "configuration",commandOptions.configuration, true); + putNestedMapIfNotEmpty(beanParams, "configuration", commandOptions.configuration, true); variantConfigureParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -407,9 +407,9 @@ private RestResponse deleteVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFileDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotNull(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); variantFileDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -448,9 +448,9 @@ private RestResponse aggregateVariantFamily() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAggregateFamilyParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "gapsGenotype",commandOptions.gapsGenotype, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "gapsGenotype", commandOptions.gapsGenotype, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantAggregateFamilyParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -489,10 +489,10 @@ private RestResponse indexVariantFamily() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFamilyIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotNull(beanParams, "updateIndex",commandOptions.updateIndex, true); - putNestedIfNotNull(beanParams, "skipIncompleteFamilies",commandOptions.skipIncompleteFamilies, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "updateIndex", commandOptions.updateIndex, true); + putNestedIfNotNull(beanParams, "skipIncompleteFamilies", commandOptions.skipIncompleteFamilies, true); variantFamilyIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -531,36 +531,36 @@ private RestResponse indexVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "transform",commandOptions.transform, true); - putNestedIfNotNull(beanParams, "gvcf",commandOptions.gvcf, true); - putNestedIfNotNull(beanParams, "normalizationSkip",commandOptions.normalizationSkip, true); - putNestedIfNotEmpty(beanParams, "referenceGenome",commandOptions.referenceGenome, true); - putNestedIfNotEmpty(beanParams, "failOnMalformedLines",commandOptions.failOnMalformedLines, true); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "load",commandOptions.load, true); - putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); - putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); - putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "loadArchive",commandOptions.loadArchive, true); - putNestedIfNotEmpty(beanParams, "loadHomRef",commandOptions.loadHomRef, true); - putNestedIfNotEmpty(beanParams, "postLoadCheck",commandOptions.postLoadCheck, true); - putNestedIfNotEmpty(beanParams, "includeGenotypes",commandOptions.includeGenotypes, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotEmpty(beanParams, "merge",commandOptions.merge, true); - putNestedIfNotEmpty(beanParams, "deduplicationPolicy",commandOptions.deduplicationPolicy, true); - putNestedIfNotNull(beanParams, "calculateStats",commandOptions.calculateStats, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexSearch",commandOptions.indexSearch, true); - putNestedIfNotNull(beanParams, "skipIndexedFiles",commandOptions.skipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "transform", commandOptions.transform, true); + putNestedIfNotNull(beanParams, "gvcf", commandOptions.gvcf, true); + putNestedIfNotNull(beanParams, "normalizationSkip", commandOptions.normalizationSkip, true); + putNestedIfNotEmpty(beanParams, "referenceGenome", commandOptions.referenceGenome, true); + putNestedIfNotEmpty(beanParams, "failOnMalformedLines", commandOptions.failOnMalformedLines, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotNull(beanParams, "load", commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload", commandOptions.forceReload, true); + putNestedIfNotEmpty(beanParams, "loadSplitData", commandOptions.loadSplitData, true); + putNestedIfNotNull(beanParams, "loadMultiFileData", commandOptions.loadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "loadSampleIndex", commandOptions.loadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "loadArchive", commandOptions.loadArchive, true); + putNestedIfNotEmpty(beanParams, "loadHomRef", commandOptions.loadHomRef, true); + putNestedIfNotEmpty(beanParams, "postLoadCheck", commandOptions.postLoadCheck, true); + putNestedIfNotEmpty(beanParams, "includeGenotypes", commandOptions.includeGenotypes, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotEmpty(beanParams, "merge", commandOptions.merge, true); + putNestedIfNotEmpty(beanParams, "deduplicationPolicy", commandOptions.deduplicationPolicy, true); + putNestedIfNotNull(beanParams, "calculateStats", commandOptions.calculateStats, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexSearch", commandOptions.indexSearch, true); + putNestedIfNotNull(beanParams, "skipIndexedFiles", commandOptions.skipIndexedFiles, true); variantIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -599,41 +599,41 @@ private RestResponse launcherVariantIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFileIndexJobLauncherParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "directory",commandOptions.directory, true); - putNestedIfNotNull(beanParams, "resumeFailed",commandOptions.resumeFailed, true); - putNestedIfNotNull(beanParams, "ignoreFailed",commandOptions.ignoreFailed, true); - putNestedIfNotNull(beanParams, "maxJobs",commandOptions.maxJobs, true); - putNestedIfNotEmpty(beanParams, "indexParams.file",commandOptions.indexParamsFile, true); - putNestedIfNotNull(beanParams, "indexParams.resume",commandOptions.indexParamsResume, true); - putNestedIfNotEmpty(beanParams, "indexParams.outdir",commandOptions.indexParamsOutdir, true); - putNestedIfNotNull(beanParams, "indexParams.transform",commandOptions.indexParamsTransform, true); - putNestedIfNotNull(beanParams, "indexParams.gvcf",commandOptions.indexParamsGvcf, true); - putNestedIfNotNull(beanParams, "indexParams.normalizationSkip",commandOptions.indexParamsNormalizationSkip, true); - putNestedIfNotEmpty(beanParams, "indexParams.referenceGenome",commandOptions.indexParamsReferenceGenome, true); - putNestedIfNotEmpty(beanParams, "indexParams.failOnMalformedLines",commandOptions.indexParamsFailOnMalformedLines, true); - putNestedIfNotNull(beanParams, "indexParams.family",commandOptions.indexParamsFamily, true); - putNestedIfNotNull(beanParams, "indexParams.somatic",commandOptions.indexParamsSomatic, true); - putNestedIfNotNull(beanParams, "indexParams.load",commandOptions.indexParamsLoad, true); - putNestedIfNotNull(beanParams, "indexParams.forceReload",commandOptions.indexParamsForceReload, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData",commandOptions.indexParamsLoadSplitData, true); - putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData",commandOptions.indexParamsLoadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex",commandOptions.indexParamsLoadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadArchive",commandOptions.indexParamsLoadArchive, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadHomRef",commandOptions.indexParamsLoadHomRef, true); - putNestedIfNotEmpty(beanParams, "indexParams.postLoadCheck",commandOptions.indexParamsPostLoadCheck, true); - putNestedIfNotEmpty(beanParams, "indexParams.includeGenotypes",commandOptions.indexParamsIncludeGenotypes, true); - putNestedIfNotEmpty(beanParams, "indexParams.includeSampleData",commandOptions.indexParamsIncludeSampleData, true); - putNestedIfNotEmpty(beanParams, "indexParams.merge",commandOptions.indexParamsMerge, true); - putNestedIfNotEmpty(beanParams, "indexParams.deduplicationPolicy",commandOptions.indexParamsDeduplicationPolicy, true); - putNestedIfNotNull(beanParams, "indexParams.calculateStats",commandOptions.indexParamsCalculateStats, true); - putNestedIfNotNull(beanParams, "indexParams.aggregated",commandOptions.indexParamsAggregated, true); - putNestedIfNotEmpty(beanParams, "indexParams.aggregationMappingFile",commandOptions.indexParamsAggregationMappingFile, true); - putNestedIfNotNull(beanParams, "indexParams.annotate",commandOptions.indexParamsAnnotate, true); - putNestedIfNotEmpty(beanParams, "indexParams.annotator",commandOptions.indexParamsAnnotator, true); - putNestedIfNotNull(beanParams, "indexParams.overwriteAnnotations",commandOptions.indexParamsOverwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexParams.indexSearch",commandOptions.indexParamsIndexSearch, true); - putNestedIfNotNull(beanParams, "indexParams.skipIndexedFiles",commandOptions.indexParamsSkipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "directory", commandOptions.directory, true); + putNestedIfNotNull(beanParams, "resumeFailed", commandOptions.resumeFailed, true); + putNestedIfNotNull(beanParams, "ignoreFailed", commandOptions.ignoreFailed, true); + putNestedIfNotNull(beanParams, "maxJobs", commandOptions.maxJobs, true); + putNestedIfNotEmpty(beanParams, "indexParams.file", commandOptions.indexParamsFile, true); + putNestedIfNotNull(beanParams, "indexParams.resume", commandOptions.indexParamsResume, true); + putNestedIfNotEmpty(beanParams, "indexParams.outdir", commandOptions.indexParamsOutdir, true); + putNestedIfNotNull(beanParams, "indexParams.transform", commandOptions.indexParamsTransform, true); + putNestedIfNotNull(beanParams, "indexParams.gvcf", commandOptions.indexParamsGvcf, true); + putNestedIfNotNull(beanParams, "indexParams.normalizationSkip", commandOptions.indexParamsNormalizationSkip, true); + putNestedIfNotEmpty(beanParams, "indexParams.referenceGenome", commandOptions.indexParamsReferenceGenome, true); + putNestedIfNotEmpty(beanParams, "indexParams.failOnMalformedLines", commandOptions.indexParamsFailOnMalformedLines, true); + putNestedIfNotNull(beanParams, "indexParams.family", commandOptions.indexParamsFamily, true); + putNestedIfNotNull(beanParams, "indexParams.somatic", commandOptions.indexParamsSomatic, true); + putNestedIfNotNull(beanParams, "indexParams.load", commandOptions.indexParamsLoad, true); + putNestedIfNotNull(beanParams, "indexParams.forceReload", commandOptions.indexParamsForceReload, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData", commandOptions.indexParamsLoadSplitData, true); + putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData", commandOptions.indexParamsLoadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex", commandOptions.indexParamsLoadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadArchive", commandOptions.indexParamsLoadArchive, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadHomRef", commandOptions.indexParamsLoadHomRef, true); + putNestedIfNotEmpty(beanParams, "indexParams.postLoadCheck", commandOptions.indexParamsPostLoadCheck, true); + putNestedIfNotEmpty(beanParams, "indexParams.includeGenotypes", commandOptions.indexParamsIncludeGenotypes, true); + putNestedIfNotEmpty(beanParams, "indexParams.includeSampleData", commandOptions.indexParamsIncludeSampleData, true); + putNestedIfNotEmpty(beanParams, "indexParams.merge", commandOptions.indexParamsMerge, true); + putNestedIfNotEmpty(beanParams, "indexParams.deduplicationPolicy", commandOptions.indexParamsDeduplicationPolicy, true); + putNestedIfNotNull(beanParams, "indexParams.calculateStats", commandOptions.indexParamsCalculateStats, true); + putNestedIfNotNull(beanParams, "indexParams.aggregated", commandOptions.indexParamsAggregated, true); + putNestedIfNotEmpty(beanParams, "indexParams.aggregationMappingFile", commandOptions.indexParamsAggregationMappingFile, true); + putNestedIfNotNull(beanParams, "indexParams.annotate", commandOptions.indexParamsAnnotate, true); + putNestedIfNotEmpty(beanParams, "indexParams.annotator", commandOptions.indexParamsAnnotator, true); + putNestedIfNotNull(beanParams, "indexParams.overwriteAnnotations", commandOptions.indexParamsOverwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexParams.indexSearch", commandOptions.indexParamsIndexSearch, true); + putNestedIfNotNull(beanParams, "indexParams.skipIndexedFiles", commandOptions.indexParamsSkipIndexedFiles, true); variantFileIndexJobLauncherParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -669,9 +669,9 @@ private RestResponse runVariantJulie() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JulieParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohorts",commandOptions.cohorts, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "cohorts", commandOptions.cohorts, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); julieParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -706,8 +706,8 @@ private RestResponse repairVariantMetadata() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStorageMetadataRepairToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "studies",commandOptions.studies, true); - putNestedIfNotNull(beanParams, "samplesBatchSize",commandOptions.samplesBatchSize, true); + putNestedIfNotNull(beanParams, "studies", commandOptions.studies, true); + putNestedIfNotNull(beanParams, "samplesBatchSize", commandOptions.samplesBatchSize, true); variantStorageMetadataRepairToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -746,8 +746,8 @@ private RestResponse synchronizeVariantMetadata() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStorageMetadataSynchronizeParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotNull(beanParams, "files",commandOptions.files, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotNull(beanParams, "files", commandOptions.files, true); variantStorageMetadataSynchronizeParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -782,9 +782,9 @@ private RestResponse pruneVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantPruneParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "project",commandOptions.project, true); - putNestedIfNotNull(beanParams, "dryRun",commandOptions.dryRun, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "project", commandOptions.project, true); + putNestedIfNotNull(beanParams, "dryRun", commandOptions.dryRun, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantPruneParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -823,9 +823,9 @@ private RestResponse deleteVariantSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSampleDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantSampleDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -864,11 +864,11 @@ private RestResponse indexVariantSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondarySampleIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "buildIndex",commandOptions.buildIndex, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotNull(beanParams, "familyIndex",commandOptions.familyIndex, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "buildIndex", commandOptions.buildIndex, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotNull(beanParams, "familyIndex", commandOptions.familyIndex, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondarySampleIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -957,12 +957,12 @@ private RestResponse indexVariantScore() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantScoreIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "scoreName",commandOptions.scoreName, true); - putNestedIfNotEmpty(beanParams, "cohort1",commandOptions.cohort1, true); - putNestedIfNotEmpty(beanParams, "cohort2",commandOptions.cohort2, true); - putNestedIfNotEmpty(beanParams, "input",commandOptions.input, true); - putNestedIfNotEmpty(beanParams, "inputColumns",commandOptions.inputColumns, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "scoreName", commandOptions.scoreName, true); + putNestedIfNotEmpty(beanParams, "cohort1", commandOptions.cohort1, true); + putNestedIfNotEmpty(beanParams, "cohort2", commandOptions.cohort2, true); + putNestedIfNotEmpty(beanParams, "input", commandOptions.input, true); + putNestedIfNotEmpty(beanParams, "inputColumns", commandOptions.inputColumns, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantScoreIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1002,9 +1002,9 @@ private RestResponse variantSecondaryAnnotationIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondaryAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondaryAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1043,11 +1043,11 @@ private RestResponse variantSecondarySampleIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondarySampleIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "buildIndex",commandOptions.buildIndex, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotNull(beanParams, "familyIndex",commandOptions.familyIndex, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "buildIndex", commandOptions.buildIndex, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotNull(beanParams, "familyIndex", commandOptions.familyIndex, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondarySampleIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1113,9 +1113,9 @@ private RestResponse secondaryIndexVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondaryAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondaryAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1176,8 +1176,8 @@ private RestResponse deleteVariantStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); variantStatsDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1216,12 +1216,12 @@ private RestResponse indexVariantStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "overwriteStats",commandOptions.overwriteStats, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "overwriteStats", commandOptions.overwriteStats, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); variantStatsIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1260,7 +1260,7 @@ private RestResponse deleteVariantStudy() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStudyDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantStudyDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index c0ce6156d34..ec848be2ac2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -126,12 +126,12 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), OrganizationCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "configuration.defaultUserExpirationDate",commandOptions.configurationDefaultUserExpirationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "configuration.defaultUserExpirationDate", commandOptions.configurationDefaultUserExpirationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -162,10 +162,10 @@ private RestResponse createNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); - putNestedIfNotNull(beanParams, "valueType",commandOptions.valueType, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); noteCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -228,8 +228,8 @@ private RestResponse updateNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); noteUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -261,7 +261,7 @@ private RestResponse userUpdateStatus() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserStatusUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "status",commandOptions.status, true); + putNestedIfNotEmpty(beanParams, "status", commandOptions.status, true); userStatusUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -293,14 +293,14 @@ private RestResponse updateUser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), OrganizationUserUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotNull(beanParams, "quota.diskUsage",commandOptions.quotaDiskUsage, true); - putNestedIfNotNull(beanParams, "quota.cpuUsage",commandOptions.quotaCpuUsage, true); - putNestedIfNotNull(beanParams, "quota.maxDisk",commandOptions.quotaMaxDisk, true); - putNestedIfNotNull(beanParams, "quota.maxCpu",commandOptions.quotaMaxCpu, true); - putNestedIfNotEmpty(beanParams, "account.expirationDate",commandOptions.accountExpirationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotNull(beanParams, "quota.diskUsage", commandOptions.quotaDiskUsage, true); + putNestedIfNotNull(beanParams, "quota.cpuUsage", commandOptions.quotaCpuUsage, true); + putNestedIfNotNull(beanParams, "quota.maxDisk", commandOptions.quotaMaxDisk, true); + putNestedIfNotNull(beanParams, "quota.maxCpu", commandOptions.quotaMaxCpu, true); + putNestedIfNotEmpty(beanParams, "account.expirationDate", commandOptions.accountExpirationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -332,11 +332,11 @@ private RestResponse updateConfiguration() throws Exc .readValue(new java.io.File(commandOptions.jsonFile), OrganizationConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "defaultUserExpirationDate",commandOptions.defaultUserExpirationDate, true); - putNestedIfNotNull(beanParams, "optimizations.simplifyPermissions",commandOptions.optimizationsSimplifyPermissions, true); - putNestedIfNotEmpty(beanParams, "token.algorithm",commandOptions.tokenAlgorithm, true); - putNestedIfNotEmpty(beanParams, "token.secretKey",commandOptions.tokenSecretKey, true); - putNestedIfNotNull(beanParams, "token.expiration",commandOptions.tokenExpiration, true); + putNestedIfNotEmpty(beanParams, "defaultUserExpirationDate", commandOptions.defaultUserExpirationDate, true); + putNestedIfNotNull(beanParams, "optimizations.simplifyPermissions", commandOptions.optimizationsSimplifyPermissions, true); + putNestedIfNotEmpty(beanParams, "token.algorithm", commandOptions.tokenAlgorithm, true); + putNestedIfNotEmpty(beanParams, "token.secretKey", commandOptions.tokenSecretKey, true); + putNestedIfNotNull(beanParams, "token.expiration", commandOptions.tokenExpiration, true); organizationConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -380,12 +380,12 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), OrganizationUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "owner",commandOptions.owner, true); - putNestedIfNotNull(beanParams, "admins",commandOptions.admins, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "owner", commandOptions.owner, true); + putNestedIfNotNull(beanParams, "admins", commandOptions.admins, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java index df4a22211a3..929d89e1e4e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java @@ -104,19 +104,19 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ProjectCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "organism.scientificName",commandOptions.organismScientificName, true); - putNestedIfNotEmpty(beanParams, "organism.commonName",commandOptions.organismCommonName, true); - putNestedIfNotEmpty(beanParams, "organism.assembly",commandOptions.organismAssembly, true); - putNestedIfNotEmpty(beanParams, "cellbase.url",commandOptions.cellbaseUrl, true); - putNestedIfNotEmpty(beanParams, "cellbase.version",commandOptions.cellbaseVersion, true); - putNestedIfNotEmpty(beanParams, "cellbase.dataRelease",commandOptions.cellbaseDataRelease, true); - putNestedIfNotEmpty(beanParams, "cellbase.apiKey",commandOptions.cellbaseApiKey, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "organism.scientificName", commandOptions.organismScientificName, true); + putNestedIfNotEmpty(beanParams, "organism.commonName", commandOptions.organismCommonName, true); + putNestedIfNotEmpty(beanParams, "organism.assembly", commandOptions.organismAssembly, true); + putNestedIfNotEmpty(beanParams, "cellbase.url", commandOptions.cellbaseUrl, true); + putNestedIfNotEmpty(beanParams, "cellbase.version", commandOptions.cellbaseVersion, true); + putNestedIfNotEmpty(beanParams, "cellbase.dataRelease", commandOptions.cellbaseDataRelease, true); + putNestedIfNotEmpty(beanParams, "cellbase.apiKey", commandOptions.cellbaseApiKey, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); projectCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -208,14 +208,14 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ProjectUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "organism.scientificName",commandOptions.organismScientificName, true); - putNestedIfNotEmpty(beanParams, "organism.commonName",commandOptions.organismCommonName, true); - putNestedIfNotEmpty(beanParams, "organism.assembly",commandOptions.organismAssembly, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "organism.scientificName", commandOptions.organismScientificName, true); + putNestedIfNotEmpty(beanParams, "organism.commonName", commandOptions.organismCommonName, true); + putNestedIfNotEmpty(beanParams, "organism.assembly", commandOptions.organismAssembly, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); projectUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java index 2aaf0238931..c8e02456a4a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java @@ -133,12 +133,12 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); sampleAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -172,7 +172,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -207,32 +207,32 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "individualId",commandOptions.individualId, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.description",commandOptions.sourceDescription, true); - putNestedIfNotEmpty(beanParams, "source.source",commandOptions.sourceSource, true); - putNestedIfNotEmpty(beanParams, "source.url",commandOptions.sourceUrl, true); - putNestedIfNotEmpty(beanParams, "processing.preparationMethod",commandOptions.processingPreparationMethod, true); - putNestedIfNotEmpty(beanParams, "processing.extractionMethod",commandOptions.processingExtractionMethod, true); - putNestedIfNotEmpty(beanParams, "processing.labSampleId",commandOptions.processingLabSampleId, true); - putNestedIfNotEmpty(beanParams, "processing.quantity",commandOptions.processingQuantity, true); - putNestedIfNotEmpty(beanParams, "processing.date",commandOptions.processingDate, true); - putNestedIfNotNull(beanParams, "processing.attributes",commandOptions.processingAttributes, true); - putNestedIfNotEmpty(beanParams, "collection.type",commandOptions.collectionType, true); - putNestedIfNotEmpty(beanParams, "collection.quantity",commandOptions.collectionQuantity, true); - putNestedIfNotEmpty(beanParams, "collection.method",commandOptions.collectionMethod, true); - putNestedIfNotEmpty(beanParams, "collection.date",commandOptions.collectionDate, true); - putNestedIfNotNull(beanParams, "collection.attributes",commandOptions.collectionAttributes, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "individualId", commandOptions.individualId, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.description", commandOptions.sourceDescription, true); + putNestedIfNotEmpty(beanParams, "source.source", commandOptions.sourceSource, true); + putNestedIfNotEmpty(beanParams, "source.url", commandOptions.sourceUrl, true); + putNestedIfNotEmpty(beanParams, "processing.preparationMethod", commandOptions.processingPreparationMethod, true); + putNestedIfNotEmpty(beanParams, "processing.extractionMethod", commandOptions.processingExtractionMethod, true); + putNestedIfNotEmpty(beanParams, "processing.labSampleId", commandOptions.processingLabSampleId, true); + putNestedIfNotEmpty(beanParams, "processing.quantity", commandOptions.processingQuantity, true); + putNestedIfNotEmpty(beanParams, "processing.date", commandOptions.processingDate, true); + putNestedMapIfNotEmpty(beanParams, "processing.attributes", commandOptions.processingAttributes, true); + putNestedIfNotEmpty(beanParams, "collection.type", commandOptions.collectionType, true); + putNestedIfNotEmpty(beanParams, "collection.quantity", commandOptions.collectionQuantity, true); + putNestedIfNotEmpty(beanParams, "collection.method", commandOptions.collectionMethod, true); + putNestedIfNotEmpty(beanParams, "collection.date", commandOptions.collectionDate, true); + putNestedMapIfNotEmpty(beanParams, "collection.attributes", commandOptions.collectionAttributes, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); sampleCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -444,33 +444,33 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "individualId",commandOptions.individualId, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.description",commandOptions.sourceDescription, true); - putNestedIfNotEmpty(beanParams, "source.source",commandOptions.sourceSource, true); - putNestedIfNotEmpty(beanParams, "source.url",commandOptions.sourceUrl, true); - putNestedIfNotEmpty(beanParams, "processing.preparationMethod",commandOptions.processingPreparationMethod, true); - putNestedIfNotEmpty(beanParams, "processing.extractionMethod",commandOptions.processingExtractionMethod, true); - putNestedIfNotEmpty(beanParams, "processing.labSampleId",commandOptions.processingLabSampleId, true); - putNestedIfNotEmpty(beanParams, "processing.quantity",commandOptions.processingQuantity, true); - putNestedIfNotEmpty(beanParams, "processing.date",commandOptions.processingDate, true); - putNestedIfNotNull(beanParams, "processing.attributes",commandOptions.processingAttributes, true); - putNestedIfNotEmpty(beanParams, "collection.type",commandOptions.collectionType, true); - putNestedIfNotEmpty(beanParams, "collection.quantity",commandOptions.collectionQuantity, true); - putNestedIfNotEmpty(beanParams, "collection.method",commandOptions.collectionMethod, true); - putNestedIfNotEmpty(beanParams, "collection.date",commandOptions.collectionDate, true); - putNestedIfNotNull(beanParams, "collection.attributes",commandOptions.collectionAttributes, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "individualId", commandOptions.individualId, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.description", commandOptions.sourceDescription, true); + putNestedIfNotEmpty(beanParams, "source.source", commandOptions.sourceSource, true); + putNestedIfNotEmpty(beanParams, "source.url", commandOptions.sourceUrl, true); + putNestedIfNotEmpty(beanParams, "processing.preparationMethod", commandOptions.processingPreparationMethod, true); + putNestedIfNotEmpty(beanParams, "processing.extractionMethod", commandOptions.processingExtractionMethod, true); + putNestedIfNotEmpty(beanParams, "processing.labSampleId", commandOptions.processingLabSampleId, true); + putNestedIfNotEmpty(beanParams, "processing.quantity", commandOptions.processingQuantity, true); + putNestedIfNotEmpty(beanParams, "processing.date", commandOptions.processingDate, true); + putNestedMapIfNotEmpty(beanParams, "processing.attributes", commandOptions.processingAttributes, true); + putNestedIfNotEmpty(beanParams, "collection.type", commandOptions.collectionType, true); + putNestedIfNotEmpty(beanParams, "collection.quantity", commandOptions.collectionQuantity, true); + putNestedIfNotEmpty(beanParams, "collection.method", commandOptions.collectionMethod, true); + putNestedIfNotEmpty(beanParams, "collection.date", commandOptions.collectionDate, true); + putNestedMapIfNotEmpty(beanParams, "collection.attributes", commandOptions.collectionAttributes, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); sampleUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 2e8bd7b0040..29fc25873f3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -178,9 +178,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotEmpty(beanParams, "template",commandOptions.template, true); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotEmpty(beanParams, "template", commandOptions.template, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); studyAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -212,18 +212,18 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "alias",commandOptions.alias, true); - putNestedIfNotEmpty(beanParams, "type.id",commandOptions.typeId, true); - putNestedIfNotEmpty(beanParams, "type.description",commandOptions.typeDescription, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "alias", commandOptions.alias, true); + putNestedIfNotEmpty(beanParams, "type.id", commandOptions.typeId, true); + putNestedIfNotEmpty(beanParams, "type.description", commandOptions.typeDescription, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); studyCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -336,8 +336,8 @@ private RestResponse updateGroups() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "users",commandOptions.users, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "users", commandOptions.users, true); groupCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -366,7 +366,7 @@ private RestResponse updateGroupsUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "users",commandOptions.users, true); + putNestedIfNotNull(beanParams, "users", commandOptions.users, true); groupUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -397,10 +397,10 @@ private RestResponse createNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); - putNestedIfNotNull(beanParams, "valueType",commandOptions.valueType, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); noteCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -462,8 +462,8 @@ private RestResponse updateNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); noteUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -499,10 +499,10 @@ private RestResponse updatePermissionRules() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PermissionRule.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "query",commandOptions.query, true); - putNestedIfNotNull(beanParams, "members",commandOptions.members, true); - putNestedIfNotNull(beanParams, "permissions",commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedMapIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotNull(beanParams, "members", commandOptions.members, true); + putNestedIfNotNull(beanParams, "permissions", commandOptions.permissions, true); permissionRule = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -577,17 +577,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "alias",commandOptions.alias, true); - putNestedIfNotEmpty(beanParams, "type.id",commandOptions.typeId, true); - putNestedIfNotEmpty(beanParams, "type.description",commandOptions.typeDescription, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "alias", commandOptions.alias, true); + putNestedIfNotEmpty(beanParams, "type.id", commandOptions.typeId, true); + putNestedIfNotEmpty(beanParams, "type.description", commandOptions.typeDescription, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); studyUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -627,11 +627,11 @@ private RestResponse updateVariableSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariableSetCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "unique",commandOptions.unique, true); - putNestedIfNotNull(beanParams, "confidential",commandOptions.confidential, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "unique", commandOptions.unique, true); + putNestedIfNotNull(beanParams, "confidential", commandOptions.confidential, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); variableSetCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -660,18 +660,18 @@ private RestResponse updateVariableSetsVariables() throws Exception .readValue(new java.io.File(commandOptions.jsonFile), Variable.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "category",commandOptions.category, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotNull(beanParams, "required",commandOptions.required, true); - putNestedIfNotNull(beanParams, "multiValue",commandOptions.multiValue, true); - putNestedIfNotNull(beanParams, "allowedValues",commandOptions.allowedValues, true); - putNestedIfNotNull(beanParams, "allowedKeys",commandOptions.allowedKeys, true); - putNestedIfNotNull(beanParams, "rank",commandOptions.rank, true); - putNestedIfNotEmpty(beanParams, "dependsOn",commandOptions.dependsOn, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "category", commandOptions.category, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "required", commandOptions.required, true); + putNestedIfNotNull(beanParams, "multiValue", commandOptions.multiValue, true); + putNestedIfNotNull(beanParams, "allowedValues", commandOptions.allowedValues, true); + putNestedIfNotNull(beanParams, "allowedKeys", commandOptions.allowedKeys, true); + putNestedIfNotNull(beanParams, "rank", commandOptions.rank, true); + putNestedIfNotEmpty(beanParams, "dependsOn", commandOptions.dependsOn, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); variable = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java index 4883cedef5c..5f8dfce148c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java @@ -136,11 +136,11 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "organization", commandOptions.organization, true); userCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -178,11 +178,11 @@ private RestResponse password() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PasswordChangeParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "organizationId",commandOptions.organizationId, true); - putNestedIfNotEmpty(beanParams, "user",commandOptions.user, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "newPassword",commandOptions.newPassword, true); - putNestedIfNotEmpty(beanParams, "reset",commandOptions.reset, true); + putNestedIfNotEmpty(beanParams, "organizationId", commandOptions.organizationId, true); + putNestedIfNotEmpty(beanParams, "user", commandOptions.user, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "newPassword", commandOptions.newPassword, true); + putNestedIfNotEmpty(beanParams, "reset", commandOptions.reset, true); passwordChangeParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -253,8 +253,8 @@ private RestResponse updateConfigs() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ConfigUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "configuration",commandOptions.configuration, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedMapIfNotEmpty(beanParams, "configuration", commandOptions.configuration, true); configUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -303,8 +303,8 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); userUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) From f35ed460069549a6917b7ebbdee5ea3bed5ba731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 24 Jul 2024 12:54:55 +0100 Subject: [PATCH 084/128] Prepare next release 2.12.6.1-SNAPSHOT --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index e6cd5eabffb..d3036d76d4f 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index c17445c0570..d1973d3109e 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 986afb42ef4..3e1b2d619b0 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 4ba23d9eb1f..46b8cfd6f76 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 917c115a102..856af7335e3 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index d1a843a369a..fb62d45951d 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index a0b09511331..ae48d80da2e 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 85ac42f2654..f02d2f42c4b 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 726d8456aa3..3ccfe32b3ba 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index ac49b0167ac..f4b18899665 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 290972cc572..f5b784112fa 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index ddcd73ee837..c6f99ad3ea4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index 64269c04696..b0143b64ede 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index 575b5ef7b73..af8d695e22e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 7583d28aa4f..40481f27836 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index 1198b5718ab..7d2c4b3feb5 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index f6cba2e9980..dc0ed33fa02 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 31cf6d9531f..a3708473206 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 669b237ff25..5fcf5ffccf5 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 7e4d5f81c7f..9550459bafe 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 3dfc10baa5a..3e367630b6a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6 + 2.12.6.1-SNAPSHOT pom OpenCGA From 9a7dd156807cc9d16dea68b396d76156b54d9de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 24 Jul 2024 12:59:25 +0100 Subject: [PATCH 085/128] storage: Hash large vairants into solr collections. Add unhashed attr_id #TASK-6596 --- .../VariantStorageMetadataManager.java | 23 ++++++++++++ .../core/variant/VariantStorageEngine.java | 13 +++++++ .../core/variant/VariantStoragePipeline.java | 5 +-- .../SearchIndexVariantQueryExecutor.java | 14 ++++---- .../variant/search/VariantSearchModel.java | 28 +++++++++++++++ .../VariantSearchToVariantConverter.java | 29 +++++++++++---- .../variant/search/solr/SolrQueryParser.java | 27 +++++++------- .../variant/VariantStorageEngineSVTest.java | 36 +++++++++++++++---- .../VariantSearchToVariantConverterTest.java | 2 +- .../search/solr/SolrQueryParserTest.java | 4 +-- .../search/HadoopVariantSearchDataWriter.java | 2 +- 11 files changed, 143 insertions(+), 40 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index 6c539e4b077..a65ea678479 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -573,11 +573,15 @@ public ProjectMetadata getProjectMetadata() { public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws StorageEngineException { ProjectMetadata projectMetadata = getProjectMetadata(); + + checkSameSpeciesAndAssembly(options, projectMetadata); if (options != null && (projectMetadata == null || StringUtils.isEmpty(projectMetadata.getSpecies()) && options.containsKey(SPECIES.key()) || StringUtils.isEmpty(projectMetadata.getAssembly()) && options.containsKey(ASSEMBLY.key()))) { projectMetadata = updateProjectMetadata(pm -> { + // Check again, in case it was updated by another thread + checkSameSpeciesAndAssembly(options, pm); if (pm == null) { pm = new ProjectMetadata(); } @@ -598,6 +602,25 @@ public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws Sto return projectMetadata; } + private static void checkSameSpeciesAndAssembly(ObjectMap options, ProjectMetadata projectMetadata) throws StorageEngineException { + if (options != null && projectMetadata != null) { + if (options.containsKey(ASSEMBLY.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getAssembly()) && !projectMetadata.getAssembly() + .equalsIgnoreCase(options.getString(ASSEMBLY.key()))) { + throw new StorageEngineException("Incompatible assembly change from '" + projectMetadata.getAssembly() + "' to '" + + options.getString(ASSEMBLY.key()) + "'"); + } + } + if (options.containsKey(SPECIES.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getSpecies()) && !projectMetadata.getSpecies() + .equalsIgnoreCase(toCellBaseSpeciesName(options.getString(SPECIES.key())))) { + throw new StorageEngineException("Incompatible species change from '" + projectMetadata.getSpecies() + "' to '" + + options.getString(SPECIES.key()) + "'"); + } + } + } + } + public DataResult getVariantFileMetadata(int studyId, int fileId, QueryOptions options) throws StorageEngineException { return fileDBAdaptor.getVariantFileMetadata(studyId, fileId, options); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index 0f51fbe39eb..b541b2f4ae4 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -1362,6 +1362,19 @@ public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQu throw new VariantQueryException("No VariantQueryExecutor found to run the query!"); } + public final VariantQueryExecutor getVariantQueryExecutor(Class clazz) + throws StorageEngineException { + Optional first = getVariantQueryExecutors() + .stream() + .filter(e -> e instanceof SearchIndexVariantQueryExecutor) + .findFirst(); + if (first.isPresent()) { + return first.get(); + } else { + throw new StorageEngineException("VariantQueryExecutor " + clazz + " not found"); + } + } + public Query preProcessQuery(Query originalQuery, QueryOptions options) { try { return getVariantQueryParser().preProcessQuery(originalQuery, options); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 5b37d2512b1..722d79e59fd 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -60,10 +60,7 @@ import org.opencb.opencga.storage.core.io.plain.StringDataReader; import org.opencb.opencga.storage.core.io.plain.StringDataWriter; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index bd6b8e6437f..899f71a9b72 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -183,7 +183,9 @@ public VariantQueryResult approximateCount(ParsedVariantQuery variantQuery DataResult nativeResult = searchManager .nativeQuery(dbName, searchEngineQuery, queryOptions); - List variantIds = nativeResult.getResults().stream().map(VariantSearchModel::getId).collect(Collectors.toList()); + List variantIds = nativeResult.getResults().stream() + .map(VariantSearchModel::toVariantSimple) + .collect(Collectors.toList()); // Adjust numSamples if the results from SearchManager is smaller than numSamples // If this happens, the count is not approximated if (variantIds.size() < sampling) { @@ -283,12 +285,12 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) { return intersect; } - protected Iterator variantIdIteratorFromSearch(Query query) { + protected Iterator variantIdIteratorFromSearch(Query query) { return variantIdIteratorFromSearch(query, Integer.MAX_VALUE, 0, null); } - protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { - Iterator variantsIterator; + protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { + Iterator variantsIterator; QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, limit) .append(QueryOptions.SKIP, skip) @@ -302,14 +304,14 @@ protected Iterator variantIdIteratorFromSearch(Query query, int limit, i } variantsIterator = nativeResult.getResults() .stream() - .map(VariantSearchModel::getId) + .map(VariantSearchModel::toVariantSimple) .iterator(); } else { SolrNativeIterator nativeIterator = searchManager.nativeIterator(dbName, query, queryOptions); if (numTotalResults != null) { numTotalResults.set(nativeIterator.getNumFound()); } - variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::getId); + variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::toVariantSimple); } } catch (VariantSearchException | IOException e) { throw new VariantQueryException("Error querying " + VariantSearchManager.SEARCH_ENGINE_ID, e); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java index 9b0bb69792c..835af18a0a0 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java @@ -17,12 +17,15 @@ package org.opencb.opencga.storage.core.variant.search; import org.apache.solr.client.solrj.beans.Field; +import org.opencb.biodata.models.variant.Variant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter.HASH_PREFIX; + /** * Created by wasim on 09/11/16. */ @@ -140,6 +143,9 @@ public class VariantSearchModel { @Field("fileInfo_*") private Map fileInfo; + @Field("attr_*") + private Map attr; + public static final double MISSING_VALUE = -100.0; @@ -171,6 +177,7 @@ public VariantSearchModel() { this.qual = new HashMap<>(); this.filter = new HashMap<>(); this.fileInfo = new HashMap<>(); + this.attr = new HashMap<>(); } public VariantSearchModel(VariantSearchModel init) { @@ -210,6 +217,7 @@ public VariantSearchModel(VariantSearchModel init) { this.qual = init.getQual(); this.filter = init.getFilter(); this.fileInfo = init.getFileInfo(); + this.attr = init.getAttr(); } @Override @@ -251,6 +259,7 @@ public String toString() { sb.append(", qual=").append(qual); sb.append(", filter=").append(filter); sb.append(", fileInfo=").append(fileInfo); + sb.append(", attr=").append(attr); sb.append('}'); return sb.toString(); } @@ -259,6 +268,17 @@ public String getId() { return id; } + public Variant toVariantSimple() { + String variantId = getId(); + if (variantId.startsWith(HASH_PREFIX)) { + Object o = getAttr().get("attr_id"); + variantId = o instanceof String ? (String) o : ((List) o).get(0); + } + Variant variant = new Variant(variantId); + variant.setId(variantId); + return variant; + } + public VariantSearchModel setId(String id) { this.id = id; return this; @@ -579,4 +599,12 @@ public VariantSearchModel setFileInfo(Map fileInfo) { return this; } + public Map getAttr() { + return attr; + } + + public VariantSearchModel setAttr(Map attr) { + this.attr = attr; + return this; + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java index 62841c0a3b9..10f3b58ff3f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java @@ -56,6 +56,7 @@ public class VariantSearchToVariantConverter implements ComplexTypeConverter includeFields; @@ -79,10 +80,9 @@ public VariantSearchToVariantConverter(Set includeFields) { @Override public Variant convertToDataModelType(VariantSearchModel variantSearchModel) { // set chromosome, start, end, ref, alt from ID - Variant variant = new Variant(variantSearchModel.getId()); + Variant variant = variantSearchModel.toVariantSimple(); - // set ID, chromosome, start, end, ref, alt, type - variant.setId(variantSearchModel.getVariantId()); + // set chromosome, start, end, ref, alt, type // set variant type if (StringUtils.isNotEmpty(variantSearchModel.getType())) { @@ -662,8 +662,10 @@ public VariantSearchModel convertToStorageType(Variant variant) { List other = new ArrayList<>(); // Set general Variant attributes: id, dbSNP, chromosome, start, end, type - variantSearchModel.setId(variant.toString()); // Internal unique ID e.g. 3:1000:AT:- - variantSearchModel.setVariantId(variant.getId()); + String variantId = getVariantId(variant); + variantSearchModel.setId(variantId); // Internal unique ID e.g. 3:1000:AT:- + variantSearchModel.setVariantId(variantId); + variantSearchModel.getAttr().put("attr_id", variant.toString()); variantSearchModel.setChromosome(variant.getChromosome()); variantSearchModel.setStart(variant.getStart()); variantSearchModel.setEnd(variant.getEnd()); @@ -1019,8 +1021,7 @@ public VariantSearchModel convertToStorageType(Variant variant) { // This field contains all possible IDs: id, dbSNP, names, genes, transcripts, protein, clinvar, hpo, ... // This will help when searching by variant id. This is added at the end of the method after collecting all IDs Set xrefs = variantAnnotationModelUtils.extractXRefs(variant.getAnnotation()); - xrefs.add(variantSearchModel.getId()); - xrefs.add(variantSearchModel.getVariantId()); + xrefs.add(variantId); if (variant.getNames() != null && !variant.getNames().isEmpty()) { variant.getNames().forEach(name -> { if (name != null) { @@ -1032,6 +1033,20 @@ public VariantSearchModel convertToStorageType(Variant variant) { return variantSearchModel; } + public static String getVariantId(Variant variant) { + String variantString = variant.toString(); + if (variantString.length() > 32766) { + // variantString.length() >= Short.MAX_VALUE + return hashVariantId(variant, variantString); + } else { + return variantString; + } + } + + public static String hashVariantId(Variant variant, String variantString) { + return HASH_PREFIX + variant.getChromosome() + ":" + variant.getStart() + ":" + Integer.toString(variantString.hashCode()); + } + private void convertStudies(Variant variant, VariantSearchModel variantSearchModel, List other) { // Sanity check if (CollectionUtils.isEmpty(variant.getStudies())) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java index 0cf045ada4c..a618fa22af4 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java @@ -25,7 +25,6 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.common.SolrException; import org.opencb.biodata.models.core.Region; -import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.solr.FacetQueryParser; @@ -35,7 +34,10 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.*; +import org.opencb.opencga.storage.core.variant.query.KeyOpValue; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.Values; +import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; import org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter; import org.slf4j.Logger; @@ -79,7 +81,7 @@ public class SolrQueryParser { static { includeMap = new HashMap<>(); - includeMap.put("id", "id,variantId"); + includeMap.put("id", "id,variantId,attr_id"); includeMap.put("chromosome", "chromosome"); includeMap.put("start", "start"); includeMap.put("end", "end"); @@ -477,7 +479,9 @@ private String parseGenomicFilter(Query query) { genes.addAll(variantQueryXref.getGenes()); xrefs.addAll(variantQueryXref.getIds()); xrefs.addAll(variantQueryXref.getOtherXrefs()); - xrefs.addAll(variantQueryXref.getVariants().stream().map(Variant::toString).collect(Collectors.toList())); + xrefs.addAll(variantQueryXref.getVariants().stream() + .map(VariantSearchToVariantConverter::getVariantId) + .collect(Collectors.toList())); // Regions if (StringUtils.isNotEmpty(query.getString(REGION.key()))) { @@ -1616,15 +1620,12 @@ private String[] includeFieldsWithMandatory(String[] includes) { return new String[0]; } - String[] mandatoryIncludeFields = new String[]{"id", "chromosome", "start", "end", "type"}; - String[] includeWithMandatory = new String[includes.length + mandatoryIncludeFields.length]; - for (int i = 0; i < includes.length; i++) { - includeWithMandatory[i] = includes[i]; - } - for (int i = 0; i < mandatoryIncludeFields.length; i++) { - includeWithMandatory[includes.length + i] = mandatoryIncludeFields[i]; - } - return includeWithMandatory; + Set mandatoryIncludeFields = new HashSet<>(Arrays.asList("id", "attr_id", "chromosome", "start", "end", "type")); + Set includeWithMandatory = new LinkedHashSet<>(includes.length + mandatoryIncludeFields.size()); + + includeWithMandatory.addAll(Arrays.asList(includes)); + includeWithMandatory.addAll(mandatoryIncludeFields); + return includeWithMandatory.toArray(new String[0]); } /** diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 64fc14c4c6a..05f090a162f 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -1,6 +1,7 @@ package org.opencb.opencga.storage.core.variant; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Ignore; import org.junit.Test; import org.opencb.biodata.formats.variant.io.VariantReader; @@ -20,6 +21,11 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutor; +import org.opencb.opencga.storage.core.variant.search.SearchIndexVariantQueryExecutor; +import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; import java.net.URI; import java.nio.file.Paths; @@ -49,22 +55,27 @@ public abstract class VariantStorageEngineSVTest extends VariantStorageBaseTest protected static URI input2; protected static URI input3; + @ClassRule + public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); + @Before public void before() throws Exception { if (!loaded) { clearDB(DB_NAME); + } + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.reloadCellbaseConfiguration(); + solr.configure(variantStorageEngine); + if (!loaded) { loadFiles(); loaded = true; } } protected void loadFiles() throws Exception { - variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); - variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); - variantStorageEngine.reloadCellbaseConfiguration(); - input1 = getResourceUri("variant-test-sv.vcf"); studyMetadata = new StudyMetadata(1, "s1"); variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); @@ -86,6 +97,7 @@ protected void loadFiles() throws Exception { .append(VariantStorageOptions.STATS_CALCULATE.key(), true) .append(VariantStorageOptions.ASSEMBLY.key(), "grch38")); + variantStorageEngine.secondaryIndex(); } @Test @@ -106,6 +118,18 @@ public void checkCount() throws Exception { assertEquals(expected, count); } + @Test + public void checkSecondaryAnnotationIndex() throws Exception { + VariantQueryExecutor variantQueryExecutor = variantStorageEngine.getVariantQueryExecutor(SearchIndexVariantQueryExecutor.class); + for (Variant variant : variantStorageEngine) { + ParsedVariantQuery query = variantStorageEngine + .parseQuery(new Query(VariantQueryParam.ID.key(), variant.toString()), new QueryOptions()); + VariantQueryResult result = variantQueryExecutor.get(query); + assertEquals(1, result.getNumResults()); + assertEquals(variant.toString(), result.first().toString()); + } + } + @Test public void checkCorrectnessFile1() throws Exception { checkCorrectness(VariantStorageEngineSVTest.input1); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java index 5839db1745b..539ea8c22d6 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java @@ -49,7 +49,7 @@ public void test() throws Exception { expectedVariant.addStudyEntry(aux.getStudy("2")); VariantSearchModel variantSearchModel = converter.convertToStorageType(expectedVariant); - assertNull(variantSearchModel.getVariantId()); + assertNotNull(variantSearchModel.getVariantId()); assertEquals(variantId, variantSearchModel.getId()); Variant actualVariant = converter.convertToDataModelType(variantSearchModel); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java index a74bcd8f8ed..6890ad7534d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java @@ -43,8 +43,8 @@ public class SolrQueryParserTest { private String studyName = "platinum"; private String flBase = "fl=other,geneToSoAcc,traits,type,soAcc,score_*,sift,passStats_*,caddRaw,biotypes,polyphenDesc,studies,end,id,variantId," - + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,siftDesc," - + "phastCons,phylop,altStats_*,id,chromosome,start,end,type"; + + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,attr_id,siftDesc," + + "phastCons,phylop,altStats_*"; private String flDefault1 = flBase + ",fileInfo__*,qual__*,filter__*,sampleFormat__*"; private String flDefaultStudy = flBase + ",fileInfo__" + studyName + "__*,qual__" + studyName + "__*," + "filter__" + studyName + "__*,sampleFormat__" + studyName + "__*"; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java index 17c61739496..39c63923c02 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java @@ -68,7 +68,7 @@ protected void add(List batch) throws Exception { return PhoenixHelper.toBytes(studyIds, PIntegerArray.INSTANCE); }); - byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("id").toString())); + byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("attr_id").toString())); variantRows.add(row); mutations.add(new Put(row) .addColumn(family, VariantPhoenixSchema.VariantColumn.INDEX_STUDIES.bytes(), bytes)); From 5d3c7f461aa20e898bbd8e56b28e61bdc3cb65fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 24 Jul 2024 17:01:14 +0100 Subject: [PATCH 086/128] storage: Fix some tests. #TASK-6596 --- .../core/variant/VariantStorageEngineBNDTest.java | 2 ++ .../variant/dummy/DummyProjectMetadataAdaptor.java | 5 ++--- .../variant/HadoopVariantStorageEngineBNDTest.java | 2 +- .../variant/index/family/FamilyIndexTest.java | 14 +++++++------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index 08dabda7562..0e90e5a174a 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -48,6 +48,7 @@ public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); if (!loaded) { clearDB(DB_NAME); loadFiles(); @@ -59,6 +60,7 @@ protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); studyMetadata = new StudyMetadata(1, "s1"); // variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); input1 = getResourceUri("variant-test-bnd.vcf"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java index d223180d9d1..3ba92ed7f1c 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java @@ -45,10 +45,9 @@ public void refresh() { @Override public synchronized DataResult getProjectMetadata() { final DataResult result = new DataResult<>(); - if (projectMetadata == null) { - projectMetadata = new ProjectMetadata("hsapiens", "grch37", 1); + if (projectMetadata != null) { + result.setResults(Collections.singletonList(projectMetadata.copy())); } - result.setResults(Collections.singletonList(projectMetadata.copy())); return result; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java index aea720d356a..b613df935ba 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java @@ -19,7 +19,7 @@ public class HadoopVariantStorageEngineBNDTest extends VariantStorageEngineBNDTe @Override protected void loadFiles() throws Exception { super.loadFiles(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); + VariantHbaseTestUtils.printVariants(((HadoopVariantStorageEngine) variantStorageEngine).getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index f67d5734d7c..a6aeba91145 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -63,13 +63,13 @@ public class FamilyIndexTest extends VariantStorageBaseTest implements HadoopVar @Before public void before() throws Exception { + HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.reloadCellbaseConfiguration(); if (!loaded) { - HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); - variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); - variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); - variantStorageEngine.reloadCellbaseConfiguration(); URI outputUri = newOutputUri(); ObjectMap params = new ObjectMap(VariantStorageOptions.ANNOTATE.key(), false) @@ -91,7 +91,7 @@ public void before() throws Exception { variantStorageEngine.annotate(outputUri, new ObjectMap()); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(variantStorageEngine.getDBAdaptor(), newOutputUri(getTestName().getMethodName())); mendelianErrorVariants = new HashSet<>(); deNovoVariants = new HashSet<>(); From 84af7c166e5270391394cd995108b08c6c774ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 24 Jul 2024 18:51:37 +0100 Subject: [PATCH 087/128] storage: Fix NPE at some tests. #TASK-6596 --- .../core/metadata/VariantMetadataConverterTest.java | 2 ++ .../storage/core/variant/io/VariantWriterFactoryTest.java | 8 ++++++-- .../storage/hadoop/variant/gaps/FillGapsTaskTest.java | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java index 2aacde2e3ce..30e4bba2e5f 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java @@ -9,6 +9,7 @@ import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.metadata.VariantMetadata; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.io.managers.LocalIOConnector; @@ -45,6 +46,7 @@ public class VariantMetadataConverterTest { @Before public void setUp() throws Exception { metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + projectMetadata = metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); URI uri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); variantReaderUtils = new VariantReaderUtils(new IOConnectorProvider(LocalIOConnector.class)); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java index 32ef120f888..776abd3874d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java @@ -23,11 +23,13 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.metadata.VariantFileHeader; import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.io.DataWriter; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantDBAdaptor; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageEngine; @@ -86,8 +88,10 @@ public void testContigLengthNull() throws IOException, StorageEngineException { new VariantFileHeaderComplexLine("contig", "chr3", null, null, null, Collections.singletonMap("length", ".")), new VariantFileHeaderComplexLine("contig", "chr4", null, null, null, Collections.singletonMap("length", "1234")) )); - StudyMetadata study = dbAdaptor.getMetadataManager().createStudy("study"); - dbAdaptor.getMetadataManager().unsecureUpdateStudyMetadata(study.setVariantHeader(header)); + VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager(); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); + StudyMetadata study = metadataManager.createStudy("study"); + metadataManager.unsecureUpdateStudyMetadata(study.setVariantHeader(header)); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(10000); DataWriter writer = new VariantWriterFactory(dbAdaptor).newDataWriter( VariantWriterFactory.VariantOutputFormat.VCF, diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java index 3d9db73719e..63f579cb99e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java @@ -13,6 +13,7 @@ import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; import org.opencb.biodata.models.variant.protobuf.VcfSliceProtos; import org.opencb.biodata.tools.variant.converters.proto.VariantToVcfSliceConverter; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -44,6 +45,7 @@ public class FillGapsTaskTest { public void setUp() throws Exception { DummyVariantStorageMetadataDBAdaptorFactory.clear(); metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); studyMetadata = metadataManager.createStudy("S"); metadataManager.updateStudyMetadata("S", sm -> { sm.getAttributes().put(VariantStorageOptions.EXTRA_FORMAT_FIELDS.key(), "DP"); From f7b4b52883ab25868e61bdc6ba6b6a4f07f7548f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 25 Jul 2024 08:43:52 +0100 Subject: [PATCH 088/128] storage: Fix VariantDBAdaptorTest. #TASK-6596 --- .../hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java index ca0d268d4f5..b2339a3cd7e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java @@ -119,6 +119,7 @@ public void before() throws Exception { e.printStackTrace(); } } + variantStorageEngine.getOptions().append(VariantStorageOptions.ASSEMBLY.key(), "GRCH38"); cellBaseUtils = variantStorageEngine.getCellBaseUtils(); expectedConnections = GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getTotalSum(); } From 8cdc46cb0f4eaecee5d5534881b52bcccf8e26de Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 26 Jul 2024 11:19:16 +0200 Subject: [PATCH 089/128] Prepare release 2.12.6.1 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 6 +++--- 21 files changed, 23 insertions(+), 23 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index d3036d76d4f..936c61eadeb 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index d1973d3109e..bc0aa70b4d3 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 3e1b2d619b0..633264eb919 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 46b8cfd6f76..56067742d78 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 856af7335e3..ecd91094c72 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index fb62d45951d..fec3eae4696 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index ae48d80da2e..83e92525e37 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index f02d2f42c4b..6967cfa50f1 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 3ccfe32b3ba..b95f3959f09 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index f4b18899665..ecffe332c36 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index f5b784112fa..a649118bc8b 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index c6f99ad3ea4..cc7eaacba34 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index b0143b64ede..8f1b560364a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index af8d695e22e..b217db7ac21 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 40481f27836..954c3b945e1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index 7d2c4b3feb5..8612e1a6661 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index dc0ed33fa02..b8cff73a473 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index a3708473206..8379af29027 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 5fcf5ffccf5..c43e60d7a91 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 9550459bafe..27934f088aa 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 3e367630b6a..3b2c6ea239f 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1-SNAPSHOT + 2.12.6.1 pom OpenCGA @@ -43,8 +43,8 @@ - 2.12.6 - 2.12.6 + + 2.12.6.1 5.8.4 2.12.2 4.12.0 From 9a38f03cd201af965fb3f3a54b43a9370e527467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 29 Jul 2024 11:13:11 +0200 Subject: [PATCH 090/128] storage-core: call the function parseConsequenceTypes from VariantQueryUtils to replace the aliases "loss_of_function" and "protein_altering" with their corresponding list of values, #TASK-6492 --- .../storage/core/variant/search/solr/SolrQueryParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java index 0cf045ada4c..461b6e498ba 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java @@ -492,7 +492,7 @@ private String parseGenomicFilter(Query query) { // Consequence types (cts) String ctLogicalOperator = " OR "; if (StringUtils.isNotEmpty(query.getString(ANNOT_CONSEQUENCE_TYPE.key(), ""))) { - consequenceTypes = Arrays.asList(query.getString(ANNOT_CONSEQUENCE_TYPE.key()).split("[,;]")); + consequenceTypes = parseConsequenceTypes(Arrays.asList(query.getString(ANNOT_CONSEQUENCE_TYPE.key()).split("[,;]"))); if (query.getString(ANNOT_CONSEQUENCE_TYPE.key()).contains(";")) { ctLogicalOperator = " AND "; // TODO This must be removed as soon as we have the Query procesing in use From 9dbf1f873e21a6d7f0881faeca3fd06fd0ec9a0e Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 30 Jul 2024 09:58:46 +0200 Subject: [PATCH 091/128] Prepare Port Patch 1.10.6.1 -> 2.2.1 #TASK-6515 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 936c61eadeb..7c32e987cdc 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index bc0aa70b4d3..d0ab2466da5 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 633264eb919..4cbfa454138 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 56067742d78..043c1c5788d 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index ecd91094c72..02a59d91cf6 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index fec3eae4696..c59da1f754d 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 83e92525e37..8c8ab880796 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 6967cfa50f1..2b5d462abff 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index b95f3959f09..2f760ec9164 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index ecffe332c36..7b26545a09d 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index a649118bc8b..06b0412be9d 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index cc7eaacba34..6d7a6649ad7 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index 8f1b560364a..84c787463ec 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index b217db7ac21..f9c584917b1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 954c3b945e1..bc999c1656f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index 8612e1a6661..b35c5350d64 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index b8cff73a473..52de24ee0b9 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 8379af29027..2a91d2753e4 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index c43e60d7a91..4a47e44df35 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 27934f088aa..2e7253d9889 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 3b2c6ea239f..5dd8f303bed 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.6.1 + 3.2.1-SNAPSHOT pom OpenCGA From 68a80b806d3e730e1e76b1408f77d082bb7df5b1 Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 5 Aug 2024 11:41:00 +0200 Subject: [PATCH 092/128] catalog: query case by version, #TASK-5964 --- .../managers/ClinicalAnalysisManagerTest.java | 26 +++++++++++++++++++ .../rest/analysis/ClinicalWebService.java | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index 8243077223b..3e11037847a 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -586,6 +586,32 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { } @Test + public void queryByVersionTest() throws CatalogException { + Individual individual = new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2"))); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); + + ClinicalComment comment = new ClinicalComment(orgOwnerUserId, "my comment", new ArrayList<>(), ""); + ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() + .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) + .setDescription("My description").setType(ClinicalAnalysis.Type.SINGLE) + .setQualityControl(new ClinicalAnalysisQualityControl(ClinicalAnalysisQualityControl.QualityControlSummary.LOW, + Collections.singletonList(comment), Collections.emptyList())) + .setProband(individual); + + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, true, INCLUDE_RESULT, ownerToken).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams().setDescription("blabla"), null, ownerToken); + + ClinicalAnalysis case1 = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), null, ownerToken).first(); + assertEquals(2, case1.getVersion()); + assertEquals("blabla", case1.getDescription()); + + Query query = new Query(ParamConstants.CLINICAL_VERSION_PARAM, 1); + case1 = catalogManager.getClinicalAnalysisManager().get(studyFqn, Collections.singletonList(clinicalAnalysis.getId()), query, null, false, ownerToken).first(); + assertEquals(1, case1.getVersion()); + assertEquals(clinicalAnalysis.getDescription(), case1.getDescription()); + } + + @Test public void createAndUpdateClinicalAnalysisWithQualityControl() throws CatalogException, InterruptedException { Individual individual = new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2"))); catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java index 79f94aa9544..3dd6fa53aa4 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java @@ -408,7 +408,7 @@ public Response info( query.remove("clinicalAnalysis"); List analysisList = getIdList(clinicalAnalysisStr); - DataResult analysisResult = clinicalManager.get(studyStr, analysisList, queryOptions, true, token); + DataResult analysisResult = clinicalManager.get(studyStr, analysisList, query, queryOptions, true, token); return createOkResponse(analysisResult); } catch (Exception e) { return createErrorResponse(e); From 4d2dba5e36b609f5f23e8e72e6723e04fe0fc4e9 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 6 Aug 2024 13:34:58 +0200 Subject: [PATCH 093/128] core: update configuration.yml, #TASK-6494 --- opencga-core/src/main/resources/configuration.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 4d16f4aa68d..3d283d3ed24 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -7,7 +7,9 @@ workspace: ${OPENCGA.USER.WORKSPACE} jobDir: ${OPENCGA.USER.WORKSPACE}/jobs # Maximum number of login attempts before banning a user account -maxLoginAttempts: ${OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS} +account: + maxLoginAttempts: ${OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS} + passwordExpirationDays: 0 panel: host: "http://resources.opencb.org/opencb/opencga/disease-panels" From 86512aa9fe8cb57b7f17ded5d6533cc032a470f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 11 Jul 2024 14:29:29 +0100 Subject: [PATCH 094/128] analysis: Fix ClinicalInterpretationAnalysisTest tests. #TASK-6703 Cherry-pick from 8495dd23fb4c7b99001cbc8613182075e5e17a12 --- .../clinical/ClinicalInterpretationAnalysisTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java index 98ae847087c..8a8de40962b 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java @@ -50,6 +50,7 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.core.tools.result.ExecutionResult; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import java.io.File; @@ -140,8 +141,11 @@ public void customAnalysisFromClinicalAnalysisTest() throws Exception { .setConfig(config); ExecutionResult result = customAnalysis.start(); - - checkInterpretation(238, result); + int expected = opencga.getVariantStorageManager().get(new VariantQuery(query), + new QueryOptions(QueryOptions.LIMIT, 500), clinicalTest.token) + .getResults().size(); + assertNotEquals(0, expected); + checkInterpretation(expected, result); } @Test From d84e860b0b0512751a5ee3666d8d9bfb7befbbef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 7 Aug 2024 21:47:26 +0100 Subject: [PATCH 095/128] storage: Fix VariantLocalConflictResolverTest tests. #TASK-6703 --- .../variant/archive/mr/VariantLocalConflictResolverTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java index f716adb0110..c1dc22a6bb4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java @@ -169,7 +169,7 @@ public void resolveConflictIndelCase1() throws Exception { se.setSampleDataKeys(Arrays.asList(GENOTYPE_KEY, GENOTYPE_FILTER_KEY)); se.setSamplesPosition(asMap("S1", 0)); se.addSampleData("S1", Arrays.asList("1/2", "LowGQXHetDel")); - se.getSecondaryAlternates().add(new AlternateCoordinate(null, null, 328, "CTT", "CTTTC", INDEL)); + se.getSecondaryAlternates().add(new AlternateCoordinate(null, 328, null, "CTT", "CTTTC", INDEL)); addAttribute(v1, FILTER, "LowGQXHetDel"); From fab3cab4698baa723d52b8c7bd4e3547df227df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 12 Jul 2024 09:13:22 +0100 Subject: [PATCH 096/128] analysis: Fix test VariantOperationsTest#testCellbaseConfigure #TASK-6703 Cherry-pick fdd88bc2d73b5093676b1a270e3d0cdd --- .../variant/manager/VariantOperationsTest.java | 11 +++++++++-- .../opencga/catalog/managers/ProjectManager.java | 6 +++++- .../org/opencb/opencga/core/api/ParamConstants.java | 2 +- .../opencga/core/cellbase/CellBaseValidator.java | 2 +- .../core/config/storage/CellBaseConfiguration.java | 2 +- .../core/variant/VariantStorageEngineBNDTest.java | 4 ++-- .../core/variant/VariantStorageEngineSVTest.java | 2 +- .../variant/VariantStorageSearchIntersectTest.java | 2 +- 8 files changed, 21 insertions(+), 10 deletions(-) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index f9a698b2fc9..406103ce71e 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -515,13 +515,20 @@ public void testCellbaseConfigure() throws Exception { assertEquals("GRCh38", cellBaseUtils.getAssembly()); String newCellbase = "https://uk.ws.zettagenomics.com/cellbase/"; - String newCellbaseVersion = "v5.8"; + String newCellbaseVersion = "v5.2"; + String newCellbaseDataRelease = "1"; assertNotEquals(newCellbase, cellBaseUtils.getURL()); assertNotEquals(newCellbaseVersion, cellBaseUtils.getVersion()); + assertNotEquals(newCellbaseDataRelease, cellBaseUtils.getDataRelease()); - variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, "1", ""), false, null, token); + variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, newCellbaseDataRelease, ""), false, null, token); CellBaseConfiguration cellbaseConfiguration = catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getCellbase(); + + assertEquals(newCellbase, cellbaseConfiguration.getUrl()); + assertEquals(newCellbaseVersion, cellbaseConfiguration.getVersion()); + assertEquals(newCellbaseDataRelease, cellbaseConfiguration.getDataRelease()); + // assertTrue(family.getPedigreeGraph() != null); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index ca0ddeb625c..ae99b2e10ee 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -319,9 +319,13 @@ private void validateProjectForCreation(Project project, User user) throws Catal throw new CatalogParameterException("Missing mandatory organism information"); } try { + //TODO: Should the datarelease be undefined? When undefined, it'd be read from cellbase meta endpoints. + String defaultDataRelease = project.getOrganism().getAssembly().equalsIgnoreCase("grch38") + ? ParamConstants.CELLBASE_DATA_RELEASE_GRCH38 + : null; CellBaseConfiguration cellBaseConfiguration = ParamUtils.defaultObject(project.getCellbase(), new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, - ParamConstants.CELLBASE_DATA_RELEASE, ParamConstants.CELLBASE_APIKEY)); + defaultDataRelease, ParamConstants.CELLBASE_APIKEY)); cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, project.getOrganism().getScientificName(), project.getOrganism().getAssembly(), true); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 086f90132ec..13f02ad0bae 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -77,7 +77,7 @@ public class ParamConstants { public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase"; public static final String CELLBASE_VERSION = "v5.8"; - public static final String CELLBASE_DATA_RELEASE = "7"; + public static final String CELLBASE_DATA_RELEASE_GRCH38 = "7"; public static final String CELLBASE_APIKEY = null; public static final String POP_FREQ_1000G_CB_V4 = "1kG_phase3"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index 5ed331143db..4dd5543e9b8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -171,7 +171,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException logger.warn("DataRelease not supported on version '" + serverVersion + ".x'"); } else { String dataRelease = getDataRelease(); - if (dataRelease == null) { + if (StringUtils.isEmpty(dataRelease)) { if (autoComplete) { cellBaseConfiguration.setDataRelease(getDefaultDataRelease()); } else { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java index 101a822562c..be0b4c366a8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java @@ -46,7 +46,7 @@ public class CellBaseConfiguration { private String apiKey; public CellBaseConfiguration() { - this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, ParamConstants.CELLBASE_DATA_RELEASE, + this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, ParamConstants.CELLBASE_DATA_RELEASE_GRCH38, ParamConstants.CELLBASE_APIKEY); } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index 08dabda7562..32e1657fcb8 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -47,7 +47,7 @@ public abstract class VariantStorageEngineBNDTest extends VariantStorageBaseTest public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); if (!loaded) { clearDB(DB_NAME); loadFiles(); @@ -58,7 +58,7 @@ public void before() throws Exception { protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); studyMetadata = new StudyMetadata(1, "s1"); // variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); input1 = getResourceUri("variant-test-bnd.vcf"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index eb9d54d608f..8585b396ab4 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -60,7 +60,7 @@ public void before() throws Exception { protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); variantStorageEngine.reloadCellbaseConfiguration(); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java index 3827a56da35..58bae56a733 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java @@ -81,7 +81,7 @@ public void before() throws Exception { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); dbAdaptor = variantStorageEngine.getDBAdaptor(); if (!loaded) { loadFile(); From ed3efc871d607889109a2b825a468541cbac5c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 7 Aug 2024 21:57:42 +0100 Subject: [PATCH 097/128] storage: Fix tests VariantQueryExecutorTest. #TASK-6703 --- .../variant/query/executors/VariantQueryExecutorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java index 3d0b2524681..b2f6407b2c5 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java @@ -139,8 +139,8 @@ public void testXRefRs() throws StorageEngineException { with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem( with("GeneName", ConsequenceType::getGeneName, is("TEX13B")))))); - matchers.put("COSV60260399", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem( - with("Cosmic", EvidenceEntry::getId, is("COSV60260399")))))); + matchers.put("RCV000155534", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem( + with("Clinvar", EvidenceEntry::getId, is("RCV000155534")))))); matchers.put("ENST00000341832.11(ENSG00000248333):c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem( is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G"))))); matchers.put("ENST00000341832.11:c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem( From ebaef56fcbe65a0b75b57226a80126f00a5882ba Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 8 Aug 2024 16:41:09 +0200 Subject: [PATCH 098/128] catalog: add salt to passwords, #TASK-6494 --- .../OrganizationsCommandExecutor.java | 1 - .../options/OrganizationsCommandOptions.java | 3 - ....java => AddPasswordHistoryMigration.java} | 31 +-- .../authorization/AuthorizationDBAdaptor.java | 24 +-- .../opencga/catalog/db/DBAdaptorFactory.java | 5 +- .../db/api/AnnotationSetDBAdaptor.java | 17 +- .../db/api/ClinicalAnalysisDBAdaptor.java | 3 +- .../catalog/db/api/CohortDBAdaptor.java | 3 +- .../catalog/db/api/FamilyDBAdaptor.java | 2 +- .../opencga/catalog/db/api/FileDBAdaptor.java | 13 +- .../catalog/db/api/IndividualDBAdaptor.java | 2 +- .../db/api/InterpretationDBAdaptor.java | 4 +- .../opencga/catalog/db/api/JobDBAdaptor.java | 3 +- .../opencga/catalog/db/api/NoteDBAdaptor.java | 3 +- .../catalog/db/api/OrganizationDBAdaptor.java | 4 +- .../catalog/db/api/PanelDBAdaptor.java | 6 +- .../catalog/db/api/ProjectDBAdaptor.java | 4 +- .../catalog/db/api/SampleDBAdaptor.java | 8 +- .../catalog/db/api/StudyDBAdaptor.java | 27 +-- .../opencga/catalog/db/api/UserDBAdaptor.java | 12 +- .../db/mongodb/AnnotationMongoDBAdaptor.java | 11 +- .../mongodb/AuthorizationMongoDBAdaptor.java | 17 +- .../ClinicalAnalysisMongoDBAdaptor.java | 9 +- .../db/mongodb/CohortMongoDBAdaptor.java | 8 +- .../db/mongodb/FamilyMongoDBAdaptor.java | 11 +- .../db/mongodb/FileMongoDBAdaptor.java | 13 +- .../db/mongodb/IndividualMongoDBAdaptor.java | 10 +- .../mongodb/InterpretationMongoDBAdaptor.java | 12 +- .../catalog/db/mongodb/JobMongoDBAdaptor.java | 11 +- .../catalog/db/mongodb/MongoDBAdaptor.java | 15 +- .../db/mongodb/MongoDBAdaptorFactory.java | 4 +- .../db/mongodb/NoteMongoDBAdaptor.java | 8 +- .../mongodb/OrganizationMongoDBAdaptor.java | 6 +- .../db/mongodb/PanelMongoDBAdaptor.java | 15 +- .../db/mongodb/ProjectMongoDBAdaptor.java | 12 +- .../db/mongodb/SampleMongoDBAdaptor.java | 14 +- .../db/mongodb/StudyMongoDBAdaptor.java | 27 +-- .../db/mongodb/UserMongoDBAdaptor.java | 195 ++++++++++++------ .../opencga/catalog/managers/NoteManager.java | 2 +- .../catalog/managers/OrganizationManager.java | 9 +- .../mongodb/IndividualMongoDBAdaptorTest.java | 5 +- .../db/mongodb/JobMongoDBAdaptorTest.java | 8 +- .../db/mongodb/SampleMongoDBAdaptorTest.java | 6 +- .../db/mongodb/StudyMongoDBAdaptorTest.java | 14 +- .../db/mongodb/UserMongoDBAdaptorTest.java | 2 +- .../catalog/managers/UserManagerTest.java | 4 +- .../user/OrganizationUserUpdateParams.java | 51 ++++- 47 files changed, 362 insertions(+), 312 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/{AddPasswordHistory.java => AddPasswordHistoryMigration.java} (62%) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index 68a4eb5dc0d..1294b1397e7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -301,7 +301,6 @@ private RestResponse updateUser() throws Exception { putNestedIfNotNull(beanParams, "quota.cpuUsage", commandOptions.quotaCpuUsage, true); putNestedIfNotNull(beanParams, "quota.maxDisk", commandOptions.quotaMaxDisk, true); putNestedIfNotNull(beanParams, "quota.maxCpu", commandOptions.quotaMaxCpu, true); - putNestedIfNotEmpty(beanParams, "account.expirationDate", commandOptions.accountExpirationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java index 7b60a3059b2..0f9f2a18c49 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java @@ -305,9 +305,6 @@ public class UpdateUserCommandOptions { @Parameter(names = {"--quota-max-cpu"}, description = "The body web service maxCpu parameter", required = false, arity = 1) public Integer quotaMaxCpu; - @Parameter(names = {"--account-expiration-date"}, description = "The body web service expirationDate parameter", required = false, arity = 1) - public String accountExpirationDate; - @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistory.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java similarity index 62% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistory.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java index d0fb6af2d12..f016e564c77 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistory.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java @@ -3,37 +3,40 @@ import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.lang3.StringUtils; +import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; +import java.util.Collections; + +import static org.opencb.opencga.catalog.db.mongodb.UserMongoDBAdaptor.*; @Migration(id = "add_archivePasswords_array", description = "Add password history #6494", version = "3.2.1", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) -public class AddPasswordHistory extends MigrationTool { +public class AddPasswordHistoryMigration extends MigrationTool { @Override protected void run() throws Exception { - Bson query = Filters.exists("_archivePasswords", false); - Bson projection = Projections.include("_password"); + Bson query = Filters.exists(PRIVATE_PASSWORD_ARCHIVE, false); + Bson projection = Projections.include(PRIVATE_PASSWORD); migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), query, projection, (document, bulk) -> { - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - - // Add _archivePasswords String currentPassword = document.getString("_password"); - List archivePasswords = new ArrayList<>(); - if (StringUtils.isNotEmpty(currentPassword)) { - archivePasswords.add(currentPassword); - } - updateDocument.getSet().put("_archivePasswords", archivePasswords); + + Document passwordDoc = new Document() + .append(HASH, currentPassword) + .append(SALT, ""); + Document privatePassword = new Document(); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + updateDocument.getSet().put(PRIVATE_PASSWORD, privatePassword); bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); }); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java index c86787e84df..09c142e68e9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java @@ -16,10 +16,8 @@ package org.opencb.opencga.catalog.auth.authorization; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.common.Enums; @@ -80,27 +78,21 @@ > OpenCGAResult> get(List resourceIds, L */ OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resource entry) throws CatalogException; - OpenCGAResult setToMembers(long studyId, List members, - List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setToMembers(long studyId, List members, List aclParams) + throws CatalogException; // Special method only to set acls in study - OpenCGAResult setToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException; - OpenCGAResult addToMembers(long studyId, List members, - List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addToMembers(long studyId, List members, List aclParams) + throws CatalogException; // Special method only to add acls in study - OpenCGAResult addToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException; - OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeFromMembers(List members, List aclParams) throws CatalogException; - OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException; OpenCGAResult setAcls(List resourceIds, AclEntryList aclEntryList, Enums.Resource resource) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java index 5f3367d4d1c..883943f1a4b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java @@ -19,10 +19,8 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.organizations.Organization; @@ -83,8 +81,7 @@ default String getCatalogDatabase(String prefix, String organization) { MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException; - OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) throws CatalogException; void deleteOrganization(Organization organization) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java index a06f3762b4d..ccca18225c9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java @@ -21,6 +21,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.study.Variable; @@ -62,12 +63,10 @@ OpenCGAResult update(Query query, ObjectMap parameters, List variab * @param variableSetId variable set id to identify the annotations that will add a new annotation. * @param variable new variable that will be added. * @return a OpenCGAResult object. - * @throws CatalogDBException if the variable could not be added to an existing annotationSet. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if the variable could not be added to an existing annotationSet, there is any unexpected parameter or + * the operation is not authorized. */ - OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException; // /** // * This method will rename the id of all the annotations corresponding to the variableSetId changing oldName per newName. @@ -88,12 +87,10 @@ OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variab * @param variableSetId variable set id for which the annotationSets have to delete the annotation. * @param annotationName Annotation name. * @return a OpenCGAResult object. - * @throws CatalogDBException when there is an error in the database. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException when there is an error in the database, there is any unexpected parameter or the operation is not + * authorized. */ - OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) throws CatalogException; /** * Makes a groupBy to obtain the different values that every annotation has and the total number of each. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 04f1aeafbe4..4fd3c2f9a3b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -235,8 +235,7 @@ default void checkId(long clinicalAnalysisId) throws CatalogDBException, Catalog OpenCGAResult nativeInsert(Map clinicalAnalysis, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList, QueryOptions options) throws CatalogException; OpenCGAResult update(long id, ObjectMap parameters, List variableSetList, List clinicalAuditList, QueryOptions queryOptions) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java index d8b6f82182e..c95d006161d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java @@ -133,8 +133,7 @@ default void checkId(long cohortId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map cohort, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) throws CatalogException; OpenCGAResult get(long cohortId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java index da321435f29..fe9fd976e64 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java @@ -149,7 +149,7 @@ default void checkId(long familyId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map family, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + QueryOptions options) throws CatalogException; OpenCGAResult get(long familyId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 2ffe6fd1ee9..a341840a6bf 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -233,13 +233,10 @@ default void checkId(long fileId) throws CatalogDBException, CatalogParameterExc * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due to any formatting error or the user is not authorised. */ OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List variableSetList, QueryOptions options) throws CatalogException; /*** * Inserts the passed file in the database. @@ -252,13 +249,11 @@ OpenCGAResult insert(long studyId, File file, List existingSamples, List * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due any formatting error or the user is not authorised. */ OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; /*** * Retrieves the file from the database containing the fileId given. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java index d55e1019e0e..b6dbc94b861 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java @@ -163,7 +163,7 @@ default void checkId(long individualId) throws CatalogDBException, CatalogParame OpenCGAResult nativeInsert(Map individual, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(long individualId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java index 8c95202e0de..519e25fa3eb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java @@ -24,6 +24,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -135,8 +136,7 @@ default void checkId(long interpretationId) throws CatalogDBException, CatalogPa OpenCGAResult nativeInsert(Map interpretation, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList) throws CatalogException; OpenCGAResult update(long uid, ObjectMap parameters, List clinicalAuditList, ParamUtils.SaveInterpretationAs action, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index a6016da7441..ebbc7b62143 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -54,8 +54,7 @@ default void checkId(long jobId) throws CatalogDBException, CatalogParameterExce OpenCGAResult nativeInsert(Map job, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException; default OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException { //return updateStatus(query, new Job.JobStatus(Job.JobStatus.PREPARED)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java index 34d3495407d..eeecd6d55df 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java @@ -6,6 +6,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.response.OpenCGAResult; @@ -16,7 +17,7 @@ public interface NoteDBAdaptor extends DBAdaptor { - OpenCGAResult insert(Note note) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Note note) throws CatalogException; default OpenCGAResult get(long noteUid, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java index d8d8e8a0a51..93e65c63121 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java @@ -5,6 +5,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.response.OpenCGAResult; @@ -98,8 +99,7 @@ public static QueryParams getParam(String key) { // // OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; // - OpenCGAResult insert(Organization organization, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException; OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java index a6ce18d5a2f..63bdd0d5e18 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java @@ -153,11 +153,9 @@ default void checkUid(long panelUid) throws CatalogDBException, CatalogParameter } } - OpenCGAResult insert(long studyUid, List panelList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyUid, List panelList) throws CatalogException; - OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) throws CatalogException; OpenCGAResult get(long panelId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java index fcdfba9034d..1ff667bdbac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java @@ -23,6 +23,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.StudyPermissions; @@ -133,8 +134,7 @@ default void checkId(long projectId) throws CatalogDBException { OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; - OpenCGAResult insert(Project project, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException; OpenCGAResult get(long project, QueryOptions options) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java index b9f0b5b1b36..485ab7b10dd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java @@ -63,7 +63,7 @@ default void checkId(long sampleId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map sample, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(long sampleId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -86,13 +86,11 @@ OpenCGAResult updateProjectRelease(long studyId, int release) */ OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) throws CatalogException; - default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) throws CatalogException { return setRgaIndexes(studyUid, Collections.emptyList(), rgaIndex); } - OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException; enum QueryParams implements QueryParam { ID("id", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index b4e0aea1d2e..0afd3f255d4 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.common.Enums; @@ -249,8 +250,7 @@ OpenCGAResult setUsersToGroup(long studyId, String groupId, List */ OpenCGAResult removeUsersFromGroup(long studyId, String groupId, List members) throws CatalogDBException; - OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException; /** * Delete a group. @@ -273,12 +273,9 @@ OpenCGAResult removeUsersFromAllGroups(long studyId, List users) * @param groupList List containing possible groups that are synced and where the user should be added to. * @param authOrigin Authentication origin of the synced groups. * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException; /** * ADD or REMOVE user to list of provided groups. @@ -288,13 +285,10 @@ OpenCGAResult resyncUserWithSyncedGroups(String user, List groupL * @param groupList List of group ids. * @param action Update action [ADD, REMOVE] * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, ParamUtils.AddRemoveAction action) + throws CatalogException; /** * Create the permission rule to the list of permission rules defined for the entry in the studyId. @@ -381,13 +375,13 @@ default void checkVariableSetExists(String variableSetId, long studyId) throws C OpenCGAResult createVariableSet(long studyId, VariableSet variableSet) throws CatalogDBException; OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult renameFieldVariableSet(long variableSetId, String oldName, String newName, String user) throws CatalogDBException, CatalogAuthorizationException; OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult getVariableSet(long variableSetUid, QueryOptions options) throws CatalogDBException; @@ -411,8 +405,7 @@ OpenCGAResult getVariableSet(long variableSetId, QueryOptions optio OpenCGAResult getVariableSets(Query query, QueryOptions queryOptions, String user) throws CatalogDBException, CatalogAuthorizationException; - OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException; void updateDiskUsage(ClientSession clientSession, long studyId, long size) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index 8c67639017c..52b0022f533 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -22,10 +22,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.response.OpenCGAResult; @@ -75,7 +72,7 @@ default void checkIds(List userIds) throws CatalogDBException, CatalogPa void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException; OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -97,9 +94,10 @@ OpenCGAResult delete(String userId, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException; + throws CatalogException; - OpenCGAResult resetPassword(String userId, String email, String newCryptPass) throws CatalogDBException; + OpenCGAResult resetPassword(String userId, String email, String newCryptPass) + throws CatalogException; // Config operations OpenCGAResult setConfig(String userId, String name, Map config) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java index d1e48be9331..02ce5649729 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java @@ -31,6 +31,7 @@ import org.opencb.opencga.catalog.db.mongodb.converters.AnnotationConverter; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.AnnotationUtils; import org.opencb.opencga.catalog.utils.Constants; @@ -687,8 +688,7 @@ private List getNewAnnotationList(List annotationSetLis return annotationList; } - public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException { long startTime = startQuery(); // We generate the generic document that should be inserted @@ -764,12 +764,9 @@ public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, * @param variableSetId Variable set id. * @param fieldId Field id corresponds with the variable name whose annotations have to be removed. * @return A OpenCGAResult object. - * @throws CatalogDBException if there is any unexpected error. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if there is any unexpected error or parameter, or if the operation is not authorized. */ - public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) throws CatalogException { long startTime = startQuery(); UpdateDocument updateDocument = new UpdateDocument(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index 445c1c6047a..ac6dd7af9fe 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -34,10 +34,8 @@ import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; @@ -578,7 +576,7 @@ public OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resou @Override public OpenCGAResult setToMembers(long studyId, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -598,8 +596,7 @@ public OpenCGAResult setToMembers(long studyId, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -652,7 +649,7 @@ private void setToMembers(List resourceIds, List members, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); addToMembersGroupInStudy(studyId, members, clientSession); @@ -696,8 +693,7 @@ private void addToMembers(List resourceIds, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction((clientSession) -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -724,7 +720,7 @@ private void addToMembersGroupInStudy(long studyId, List members, Client @Override public OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -766,8 +762,7 @@ private void removeFromMembers(ClientSession clientSession, List resourceI } @Override - public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException { if (members == null || members.isEmpty()) { throw new CatalogDBException("Missing 'members' array."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index ec066ba4fc6..08216439ba5 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -256,7 +256,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List va try { return runTransaction(clientSession -> transactionalUpdate(clientSession, result.first(), parameters, variableSetList, clinicalAuditList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update clinical analysis {}: {}", clinicalAnalysisId, e.getMessage(), e); throw new CatalogDBException("Could not update clinical analysis " + clinicalAnalysisId + ": " + e.getMessage(), e.getCause()); } @@ -684,7 +684,7 @@ public OpenCGAResult delete(ClinicalAnalysis clinicalAnalysis, List privateDelete(clientSession, clinicalAnalysis, clinicalAuditList)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete Clinical Analysis " + clinicalAnalysis.getId() + ": " + e.getMessage(), e.getCause()); @@ -702,7 +702,7 @@ public OpenCGAResult delete(Query query, List c try { result.append(runTransaction(clientSession -> privateDelete(clientSession, clinicalAnalysis, clinicalAuditList))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, clinicalAnalysis.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1048,8 +1048,7 @@ public OpenCGAResult nativeInsert(Map clinicalAnalysis, String u @Override public OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java index 7f25b1ee7b9..6b948a722ba 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java @@ -96,7 +96,7 @@ public OpenCGAResult nativeInsert(Map cohort, String userId) thr @Override public OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -275,7 +275,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, cohort, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update cohort {}: {}", cohort.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohort.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -545,7 +545,7 @@ public OpenCGAResult delete(Cohort cohort) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find cohort " + cohort.getId() + " with uid " + cohort.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohort.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete cohort '" + cohort.getId() + "': " + e.getMessage(), e.getCause()); } @@ -561,7 +561,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String cohortId = cohort.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, cohort))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohortId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohortId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index 9e8c35d00ab..97bfd92a671 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -115,8 +115,7 @@ public OpenCGAResult nativeInsert(Map family, String userId) thr @Override public OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + QueryOptions options) throws CatalogException { try { AtomicReference familyCopy = new AtomicReference<>(); OpenCGAResult result = runTransaction(clientSession -> { @@ -333,7 +332,7 @@ public OpenCGAResult update(long familyUid, ObjectMap parameters, List transactionalUpdate(clientSession, familyDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", familyDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update family " + familyDataResult.first().getId() + ": " + e.getMessage(), e.getCause()); @@ -368,7 +367,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, family, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", family.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, family.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -784,7 +783,7 @@ public OpenCGAResult delete(Family family) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find family " + family.getId() + " with uid " + family.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", family.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete family " + family.getId() + ": " + e.getMessage(), e.getCause()); } @@ -801,7 +800,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String familyId = family.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, family))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", familyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, familyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index f214620bf39..0f8383a9905 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -127,8 +127,7 @@ public OpenCGAResult nativeInsert(Map file, String userId) throw @Override public OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List variableSetList, QueryOptions options) throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -144,7 +143,7 @@ public OpenCGAResult insert(long studyId, File file, List existingSample @Override public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -344,7 +343,7 @@ public OpenCGAResult update(long fileUid, ObjectMap parameters, List transactionalUpdate(clientSession, fileDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", fileDataResult.first().getPath(), e.getMessage(), e); throw new CatalogDBException("Could not update file " + fileDataResult.first().getPath() + ": " + e.getMessage(), e.getCause()); } @@ -370,7 +369,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, file, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", file.getPath(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, file.getPath(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -931,7 +930,7 @@ public OpenCGAResult delete(File file, String status) try { return runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", file.getPath(), e.getMessage(), e); throw new CatalogDBException("Could not delete file " + file.getPath() + ": " + e.getMessage(), e.getCause()); } @@ -962,7 +961,7 @@ public OpenCGAResult delete(Query query, String status) Document fileDocument = iterator.next(); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", fileDocument.getString(QueryParams.PATH.key()), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, fileDocument.getString(QueryParams.ID.key()), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 13dbb40741c..03a6704af17 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -119,7 +119,7 @@ public OpenCGAResult nativeInsert(Map individual, String userId) @Override public OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -329,7 +329,7 @@ public OpenCGAResult update(long individualUid, ObjectMap parameters, List transactionalUpdate(clientSession, individualUid, parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { throw new CatalogDBException("Could not update individual: " + e.getMessage(), e.getCause()); } } @@ -363,7 +363,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, individual, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update individual {}: {}", individual.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individual.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1051,7 +1051,7 @@ public OpenCGAResult delete(Individual individual) throws CatalogDBException, Ca throw new CatalogDBException("Could not find individual " + individual.getId() + " with uid " + individual.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individual.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e); } @@ -1068,7 +1068,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String individualId = individual.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, individual))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individualId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individualId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java index 3d78516dd55..f15a2ac5739 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java @@ -40,6 +40,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.InterpretationCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.ClinicalAnalysisManager; import org.opencb.opencga.catalog.utils.Constants; @@ -120,8 +121,7 @@ public OpenCGAResult nativeInsert(Map interpretation, String use @Override public OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting interpretation insert transaction for interpretation id '{}'", interpretation.getId()); @@ -635,7 +635,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List try { return runTransaction(clientSession -> update(clientSession, interpretation.first(), parameters, clinicalAuditList, action, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not update interpretation " + interpretationId + ": " + e.getMessage(), e.getCause()); } @@ -654,7 +654,7 @@ public OpenCGAResult revert(long id, int previousVersion, List c return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); }); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not delete interpretation " + interpretation.getId() + ": " + e.getMessage(), e.getCause()); } @@ -852,7 +852,7 @@ public OpenCGAResult delete(Query query, List cli return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); })); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, interpretationId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 53c9a5f157e..d89e0020f93 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -100,8 +100,7 @@ public OpenCGAResult nativeInsert(Map job, String userId) throws } @Override - public OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -228,7 +227,7 @@ public OpenCGAResult update(long jobUid, ObjectMap parameters, QueryOptions quer try { return runTransaction(session -> privateUpdate(session, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update job " + dataResult.first().getId() + ": " + e.getMessage(), e.getCause()); } @@ -254,7 +253,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Job job = iterator.next(); try { result.append(runTransaction(session -> privateUpdate(session, job, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", job.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, job.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -306,7 +305,7 @@ public OpenCGAResult delete(Job job) throws CatalogDBException, CatalogParameter throw new CatalogDBException("Could not find job " + job.getId() + " with uid " + job.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", job.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete job " + job.getId() + ": " + e.getMessage(), e.getCause()); } @@ -322,7 +321,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String jobId = job.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, job))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", jobId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, jobId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java index 7f41e38411d..10e84541343 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java @@ -93,16 +93,15 @@ public MongoDBAdaptor(Configuration configuration, Logger logger) { } public interface TransactionBodyWithException { - T execute(ClientSession session) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + T execute(ClientSession session) throws CatalogException; } - protected T runTransaction(TransactionBodyWithException body) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + protected T runTransaction(TransactionBodyWithException body) throws CatalogException { return runTransaction(body, null); } protected T runTransaction(TransactionBodyWithException inputBody, Consumer onException) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { ClientSession session = dbAdaptorFactory.getMongoDataStore().startSession(); try { TransactionBodyWithException body; @@ -124,7 +123,7 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum return session.withTransaction(() -> { try { return body.execute(session); - } catch (CatalogDBException | CatalogAuthorizationException | CatalogParameterException e) { + } catch (CatalogException e) { throw new CatalogDBRuntimeException(e); } }); @@ -147,6 +146,12 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum onException.accept(cause); } throw cause; + } else if (e.getCause() instanceof CatalogAuthenticationException) { + CatalogAuthenticationException cause = (CatalogAuthenticationException) e.getCause(); + if (onException != null) { + onException.accept(cause); + } + throw cause; } else { throw e; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index 621c537cddd..e262c3d3595 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -27,10 +27,8 @@ import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.managers.NoteManager; import org.opencb.opencga.core.api.ParamConstants; @@ -261,7 +259,7 @@ public MetaDBAdaptor getCatalogMetaDBAdaptor(String organizationId) throws Catal @Override public OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { OrganizationMongoDBAdaptorFactory orgFactory = getOrganizationMongoDBAdaptorFactory(organization.getId(), false); if (orgFactory != null && orgFactory.isCatalogDBReady()) { throw new CatalogDBException("Organization '" + organization.getId() + "' already exists."); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java index 820f15ed2f7..0b953919713 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java @@ -14,6 +14,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -51,8 +52,7 @@ public NoteMongoDBAdaptor(MongoDBCollection noteCollection, MongoDBCollection ar } @Override - public OpenCGAResult insert(Note note) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Note note) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting note insert transaction for note id '{}'", note.getId()); @@ -142,7 +142,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, QueryOptions queryOp throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateUpdate(clientSession, uid, parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update note: {}", e.getMessage(), e); throw new CatalogDBException("Could not update note: " + e.getMessage(), e.getCause()); } @@ -262,7 +262,7 @@ public OpenCGAResult delete(Note note) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateDelete(clientSession, note)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { throw new CatalogDBException("Could not delete note " + note.getId() + ": " + e.getMessage(), e); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java index aa090f498d1..b971f5a5740 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java @@ -17,6 +17,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.OrganizationCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.OrganizationManager; import org.opencb.opencga.catalog.utils.Constants; @@ -52,8 +53,7 @@ public OrganizationMongoDBAdaptor(MongoDBCollection organizationCollection, Conf } @Override - public OpenCGAResult insert(Organization organization, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting organization insert transaction for organization id '{}'", organization.getId()); @@ -124,7 +124,7 @@ public OpenCGAResult update(String organizationId, ObjectMap param try { QueryOptions options = queryOptions != null ? new QueryOptions(queryOptions) : QueryOptions.empty(); return runTransaction(clientSession -> privateUpdate(clientSession, organizationId, parameters, options)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update organization {}: {}", organizationId, e.getMessage(), e); throw new CatalogDBException("Could not update organization " + organizationId + ": " + e.getMessage(), e.getCause()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java index d82535b9351..fcc8f32fd6f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java @@ -33,6 +33,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -87,8 +88,7 @@ public MongoDBCollection getPanelArchiveCollection() { } @Override - public OpenCGAResult insert(long studyUid, List panelList) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, List panelList) throws CatalogException { if (panelList == null || panelList.isEmpty()) { throw new CatalogDBException("Missing panel list"); } @@ -108,8 +108,7 @@ public OpenCGAResult insert(long studyUid, List panelList) throws Catalog } @Override - public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting insert transaction of panel id '{}'", panel.getId()); @@ -291,7 +290,7 @@ public OpenCGAResult update(long panelUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update panel '" + dataResult.first().getId() + "': " + e.getMessage(), e.getCause()); } @@ -317,7 +316,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Panel panel = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, panel, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", panel.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panel.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -464,7 +463,7 @@ public OpenCGAResult delete(Panel panel) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find panel " + panel.getId() + " with uid " + panel.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panel.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete panel '" + panel.getId() + "': " + e.getMessage(), e.getCause()); } @@ -480,7 +479,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String panelId = panel.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, panel))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panelId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panelId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index 89d90aab932..3a91bc26284 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -32,6 +32,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.ProjectCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -82,8 +83,7 @@ public OpenCGAResult nativeInsert(Map project, String userId) th } @Override - public OpenCGAResult insert(Project project, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting project insert transaction for project id '{}'", project.getId()); @@ -201,7 +201,7 @@ public OpenCGAResult update(long projectUid, ObjectMap parameters, QueryOptions try { return runTransaction(clientSession -> privateUpdate(clientSession, projectDataResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", projectDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update project '" + projectDataResult.first().getId() + "': " + e.getMessage(), e.getCause()); @@ -220,7 +220,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Project project = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, project, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -383,7 +383,7 @@ public OpenCGAResult delete(long id, QueryOptions queryOptions) throws CatalogDB public OpenCGAResult delete(Project project) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateDelete(clientSession, project)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete project '" + project.getId() + "': " + e.getMessage(), e.getCause()); } @@ -412,7 +412,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, project))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index 198c9fce456..d9058193e7d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -39,6 +39,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.SampleCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.IndividualManager; import org.opencb.opencga.catalog.managers.SampleManager; @@ -199,7 +200,7 @@ Sample insert(ClientSession clientSession, long studyUid, Sample sample, List insert(long studyId, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting sample insert transaction for sample id '{}'", sample.getId()); @@ -262,7 +263,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List va try { return runTransaction(clientSession -> privateUpdate(clientSession, documentResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); throw new CatalogDBException("Could not update sample " + sampleId + ": " + e.getMessage(), e.getCause()); } @@ -297,7 +298,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, sampleDocument, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -857,8 +858,7 @@ public OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) } @Override - public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException { ObjectMap params; try { params = new ObjectMap(getDefaultObjectMapper().writeValueAsString(rgaIndex)); @@ -943,7 +943,7 @@ public OpenCGAResult delete(Sample sample) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find sample " + sample.getId() + " with uid " + sample.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sample.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e); } @@ -960,7 +960,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, sample))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index 5c6e0ff2035..c91fd4c3665 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -38,6 +38,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.StudyCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.FqnUtils; @@ -490,8 +491,7 @@ OpenCGAResult removeUsersFromAdminsGroup(ClientSession clientSession, Lis } @Override - public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException { if (users == null || users.size() == 0) { throw new CatalogDBException("Unable to remove users from groups. List of users is empty"); } @@ -546,8 +546,7 @@ public OpenCGAResult syncGroup(long studyId, String groupId, Group.Sync s } @Override - public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogDBException("Missing user field"); } @@ -600,8 +599,7 @@ public OpenCGAResult resyncUserWithSyncedGroups(String user, List @Override public OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + ParamUtils.AddRemoveAction action) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogParameterException("Missing user parameter"); @@ -820,7 +818,7 @@ public OpenCGAResult createVariableSet(long studyId, VariableSet va @Override public OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableNotInVariableSet(variableSet.first(), variable.getId()); @@ -897,9 +895,7 @@ public OpenCGAResult renameFieldVariableSet(long variableSetId, Str @Override public OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - long startTime = startQuery(); - + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableInVariableSet(variableSet.first(), name); @@ -1167,8 +1163,7 @@ public OpenCGAResult getVariableSets(Query query, QueryOptions quer } @Override - public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException { try { return runTransaction(clientSession -> { if (force) { @@ -1341,7 +1336,7 @@ public OpenCGAResult update(long studyUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, studyResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", studyId, e.getMessage(), e); throw new CatalogDBException("Could not update study '" + studyId + "': " + e.getMessage(), e.getCause()); } @@ -1364,7 +1359,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Study study = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, study, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", study.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, study.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1478,7 +1473,7 @@ public OpenCGAResult delete(Study study) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find study " + study.getId() + " with uid " + study.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", study.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete study " + study.getId() + ": " + e.getMessage(), e.getCause()); } @@ -1494,7 +1489,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { String studyId = study.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, study))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", studyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, studyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 76bc3bd9926..775b6cd3b2c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -18,6 +18,7 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.Updates; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; @@ -37,13 +38,12 @@ import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.UserConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.project.Project; @@ -73,8 +73,22 @@ public class UserMongoDBAdaptor extends CatalogMongoDBAdaptor implements UserDBA private final MongoDBCollection deletedUserCollection; private UserConverter userConverter; - private static final String PRIVATE_PASSWORD = "_password"; - private static final String ARCHIVE_PASSWORD = "_archivePasswords"; + // --- Password constants --- + public static final String HASH = "hash"; + public static final String SALT = "salt"; + + public static final String PRIVATE_PASSWORD = "_password"; + + public static final String CURRENT = "current"; + private static final String PRIVATE_PASSWORD_CURRENT = "_password." + CURRENT; + private static final String PRIVATE_PASSWORD_CURRENT_HASH = PRIVATE_PASSWORD_CURRENT + "." + HASH; + private static final String PRIVATE_PASSWORD_CURRENT_SALT = PRIVATE_PASSWORD_CURRENT + "." + SALT; + + public static final String ARCHIVE = "archive"; + public static final String PRIVATE_PASSWORD_ARCHIVE = "_password." + ARCHIVE; + private static final String PRIVATE_PASSWORD_ARCHIVE_HASH = PRIVATE_PASSWORD_ARCHIVE + "." + HASH; + private static final String PRIVATE_PASSWORD_ARCHIVE_SALT = PRIVATE_PASSWORD_ARCHIVE + "." + SALT; + // -------------------------- public UserMongoDBAdaptor(MongoDBCollection userCollection, MongoDBCollection deletedUserCollection, Configuration configuration, OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { @@ -96,8 +110,7 @@ boolean exists(ClientSession clientSession, String userId) throws CatalogDBExcep } @Override - public OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(User user, String password, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -120,8 +133,18 @@ private void insert(ClientSession clientSession, User user, String password) thr Document userDocument = userConverter.convertToStorageType(user); userDocument.append(ID, user.getId()); - userDocument.append(PRIVATE_PASSWORD, encryptPassword(password)); - userDocument.append(ARCHIVE_PASSWORD, Collections.singletonList(encryptPassword(password))); + + Document privatePassword = new Document(); + if (StringUtils.isNotEmpty(password)) { + String salt = PasswordUtils.getStrongRandomPassword(); + String hash = encryptPassword(password + salt); + Document passwordDoc = new Document() + .append(HASH, hash) + .append(SALT, salt); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + } + userDocument.put(PRIVATE_PASSWORD, privatePassword); userCollection.insert(clientSession, userDocument, null); } @@ -134,78 +157,118 @@ public OpenCGAResult get(String userId, QueryOptions options) } @Override - public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException { + public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { return setPassword(userId, oldPassword, newPassword); } @Override - public void authenticate(String userId, String password) throws CatalogAuthenticationException { - Document bson; - try { - bson = new Document() - .append(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(password)); - } catch (CatalogDBException e) { - throw new CatalogAuthenticationException("Could not encrypt password: " + e.getMessage(), e); + public void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException { + Bson query = Filters.and( + Filters.eq(QueryParams.ID.key(), userId), + Filters.eq(INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA) + ); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult dataResult = userCollection.find(query, projection, QueryOptions.empty()); + if (dataResult.getNumResults() == 0) { + throw new CatalogDBException("User " + userId + " not found"); + } + Document userDocument = dataResult.first(); + Document rootPasswordDoc = userDocument.get(PRIVATE_PASSWORD, Document.class); + if (rootPasswordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); } - if (userCollection.count(bson).getNumMatches() == 0) { - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); + Document passwordDoc = rootPasswordDoc.get(CURRENT, Document.class); + if (passwordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); + } + + String salt = passwordDoc.getString(SALT); + String hash = encryptPassword(password + salt); + if (!hash.equals(passwordDoc.getString(HASH))) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); } } @Override - public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogDBException { + public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogException { return setPassword(userId, null, newPassword); } - public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, String newPassword) throws CatalogDBException { - List queryFilter = new ArrayList<>(); - queryFilter.add(Filters.eq(QueryParams.ID.key(), userId)); - queryFilter.add(Filters.ne(ARCHIVE_PASSWORD, encryptPassword(newPassword))); - if (StringUtils.isNotEmpty(oldPassword)) { - queryFilter.add(Filters.eq(PRIVATE_PASSWORD, encryptPassword(oldPassword))); - } - Bson query = Filters.and(queryFilter); + public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, String newPassword) throws CatalogException { + String prefixErrorMsg = "Could not update the password. "; + return runTransaction(clientSession -> { + // 1. Obtain archived passwords + Bson query = Filters.eq(QueryParams.ID.key(), userId); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult userQueryResult = userCollection.find(clientSession, query, projection, QueryOptions.empty()); + if (userQueryResult.getNumResults() == 0) { + throw new CatalogDBException(prefixErrorMsg + "User " + userId + " not found."); + } + Document userDoc = userQueryResult.first(); + Document passwordDoc = userDoc.get(PRIVATE_PASSWORD, Document.class); + + // 1.1. Check oldPassword + if (StringUtils.isNotEmpty(oldPassword)) { + Document currentPasswordDoc = passwordDoc.get(CURRENT, Document.class); + String currentSalt = currentPasswordDoc.getString(SALT); + String currentHash = encryptPassword(oldPassword + currentSalt); + if (!currentHash.equals(currentPasswordDoc.getString(HASH))) { + throw new CatalogAuthenticationException(prefixErrorMsg + "Please, verify that the current password is correct."); + } + } - UpdateDocument updateDocument = new UpdateDocument(); - String encryptedPassword = encryptPassword(newPassword); - updateDocument.getSet().put(PRIVATE_PASSWORD, encryptedPassword); - updateDocument.getPush().put(ARCHIVE_PASSWORD, encryptedPassword); - updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED.key(), TimeUtils.getTime()); - if (configuration.getAccount().getPasswordExpirationDays() > 0) { - Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); - String stringDate = TimeUtils.getTime(date); - updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE.key(), stringDate); - } - Document update = updateDocument.toFinalUpdateDocument(); + // 2. Calculate all possible hashValues with new password + Set hashValues = new HashSet<>(); + List saltValues = passwordDoc.getList(ARCHIVE, Document.class).stream() + .map(document -> document.getString(SALT)) + .collect(Collectors.toList()); + for (String saltValue : saltValues) { + hashValues.add(encryptPassword(newPassword + saltValue)); + } - logger.debug("Change password: query '{}'; update: '{}'", query.toBsonDocument(), update); - DataResult result = userCollection.update(query, update, null); - if (result.getNumUpdated() == 0) { - if (result.getNumMatches() == 0) { - Query userQuery = new Query(QueryParams.ID.key(), userId); - OpenCGAResult queryResult = nativeGet(userQuery, - new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(PRIVATE_PASSWORD, ARCHIVE_PASSWORD))); - if (queryResult.getNumResults() == 0) { - throw new CatalogDBException("Could not update the password. User not found."); - } - Document userDocument = queryResult.first(); - if (StringUtils.isNotEmpty(oldPassword)) { - String dbPassword = userDocument.getString(PRIVATE_PASSWORD); - if (!encryptPassword(oldPassword).equals(dbPassword)) { - throw new CatalogDBException("Could not update the password. Please, verify that the current password is correct."); - } - } - List archivePassword = userDocument.getList(ARCHIVE_PASSWORD, String.class); - if (archivePassword.contains(encryptedPassword)) { - throw new CatalogDBException("Could not update the password. The new password has already been used. Please, use" - + " a different one."); + // 3. Check new password has not been used before + for (Document document : passwordDoc.getList(ARCHIVE, Document.class)) { + String hashValue = document.getString(HASH); + if (hashValues.contains(hashValue)) { + throw new CatalogAuthenticationException(prefixErrorMsg + "The new password has already been used." + + " Please, use a different one."); } } - throw new CatalogDBException("Could not update the password. Please, verify that the current password is correct."); - } - return new OpenCGAResult(result); + + // 4. Generate new salt for current password + String newSalt = PasswordUtils.getStrongRandomPassword(); + String newHash = encryptPassword(newPassword + newSalt); + + // 5. Generate update document + UpdateDocument updateDocument = new UpdateDocument(); + // add to current + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_HASH, newHash); + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_SALT, newSalt); + + // add to archive + Document document = new Document() + .append(HASH, newHash) + .append(SALT, newSalt); + updateDocument.getPush().put(PRIVATE_PASSWORD_ARCHIVE, document); + + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED.key(), TimeUtils.getTime()); + if (configuration.getAccount().getPasswordExpirationDays() > 0) { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + String stringDate = TimeUtils.getTime(date); + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE.key(), stringDate); + } + Document update = updateDocument.toFinalUpdateDocument(); + + logger.debug("Change password: query '{}'; update: '{}'", query.toBsonDocument(), update); + DataResult result = userCollection.update(clientSession, query, update, null); + if (result.getNumUpdated() == 0) { + throw new CatalogAuthenticationException("Could not update the password. Please, verify that the current password is" + + " correct."); + } + return new OpenCGAResult(result); + }, e -> logger.error("User {}: {}", userId, e.getMessage())); } @Override diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java index 4628126dc51..f32c8422dec 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java @@ -303,7 +303,7 @@ private OpenCGAResult update(long noteUid, NoteUpdateParams noteUpdatePara updateMap.put(NoteDBAdaptor.QueryParams.USER_ID.key(), tokenPayload.getUserId()); OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, - QueryOptions.empty()); + options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated note OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java index 2612ea7f633..20ec12a7f2b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java @@ -327,9 +327,6 @@ public OpenCGAResult updateUser(@Nullable String organizationId, String us throw new CatalogException("Max CPU cannot be negative"); } } - if (updateParams.getAccount() != null && StringUtils.isNotEmpty(updateParams.getAccount().getExpirationDate())) { - ParamUtils.checkDateIsNotExpired(updateParams.getAccount().getExpirationDate(), "expirationDate"); - } ObjectMap updateMap; try { @@ -337,6 +334,12 @@ public OpenCGAResult updateUser(@Nullable String organizationId, String us } catch (JsonProcessingException e) { throw new CatalogException("Could not parse OrganizationUserUpdateParams object: " + e.getMessage(), e); } + + if (updateParams.getInternal() != null && updateParams.getInternal().getAccount() != null + && StringUtils.isNotEmpty(updateParams.getInternal().getAccount().getExpirationDate())) { + ParamUtils.checkDateIsNotExpired(updateParams.getInternal().getAccount().getExpirationDate(), "expirationDate"); + } + OpenCGAResult updateResult = getUserDBAdaptor(myOrganizationId).update(userId, updateMap); auditManager.auditUpdate(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, userId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java index 4cae5cd565b..8882260fabc 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java @@ -26,9 +26,8 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.IndividualPopulation; @@ -187,7 +186,7 @@ public void testModifyIndividualNegativeFatherId() throws Exception { } @Test - public void testAvoidDuplicatedSamples() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void testAvoidDuplicatedSamples() throws CatalogException { dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), Collections.emptyList(), QueryOptions.empty()); Sample sample1 = getSample(studyUid, "sample1"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java index 9c6d82a4c79..a382282764d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java @@ -60,7 +60,7 @@ private Job getNewJob(String id) { } @Test - public void createJobTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void createJobTest() throws CatalogException { Job job = getNewJob("jobName1"); System.out.println(catalogJobDBAdaptor.insert(studyUid, job, null)); @@ -120,7 +120,7 @@ public void getJobTest() throws CatalogException { } @Test - public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void testSortResultsPriorityAndCreationDate() throws CatalogException { Date startDate = TimeUtils.getDate(); // Create 100 jobs @@ -176,7 +176,7 @@ public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, // } @Test - public void getJobsOrderedByDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void getJobsOrderedByDate() throws CatalogException { // Job with current date Job job1 = getNewJob("job1"); @@ -250,7 +250,7 @@ public void updateInputAndOutputFiles() throws Exception { } @Test - public void groupByStatus() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void groupByStatus() throws CatalogException { for (int i = 0; i < 10; i++) { Enums.ExecutionStatus status = new Enums.ExecutionStatus(); if (i < 5) { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java index 5488bc6a545..fa908630690 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java @@ -27,10 +27,8 @@ import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.SampleManager; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.cohort.Cohort; @@ -175,7 +173,7 @@ public class SampleMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { // } @Test - public void searchByOntology() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void searchByOntology() throws CatalogException { List ontologyList = Arrays.asList( new Phenotype("hpo:123", "One hpo term", "hpo", Phenotype.Status.UNKNOWN), new Phenotype("hpo:456", "Another hpo term", "hpo", Phenotype.Status.UNKNOWN), @@ -387,7 +385,7 @@ public void caseInsensitiveSearchTest() throws Exception { // Test if we can search for samples of an individual @Test - public void getSampleWithIndividual() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void getSampleWithIndividual() throws CatalogException { QueryOptions queryOptions = new QueryOptions(); // We create a new sample with the individual diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index 697a77d894f..f81f28b37a6 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -20,10 +20,8 @@ import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -112,7 +110,7 @@ public void createVariableSetTest() throws CatalogDBException { } @Test - public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void testRemoveFieldFromVariableSet() throws CatalogException { DataResult variableSetDataResult = createExampleVariableSet("VARSET_1", false); DataResult result = catalogStudyDBAdaptor.removeFieldFromVariableSet(5L, variableSetDataResult.first().getUid(), "NAME", orgAdminUserId1); @@ -165,7 +163,7 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void addFieldToVariableSetTest1() throws CatalogException { DataResult varset1 = createExampleVariableSet("VARSET_1", false); createExampleVariableSet("VARSET_2", true); Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, @@ -191,7 +189,7 @@ public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAutho * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest2() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void addFieldToVariableSetTest2() throws CatalogException { Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap()); thrown.expect(CatalogDBException.class); @@ -208,7 +206,7 @@ public void createGroup() throws CatalogDBException { } @Test - public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void removeUsersFromAllGroups() throws CatalogException { catalogStudyDBAdaptor.createGroup(studyUid, new Group("name1", Arrays.asList(normalUserId1, normalUserId2))); catalogStudyDBAdaptor.createGroup(studyUid, new Group("name2", Arrays.asList(normalUserId1, normalUserId2, normalUserId3))); catalogStudyDBAdaptor.createGroup(studyUid, new Group("name3", Arrays.asList(normalUserId1, normalUserId3))); @@ -221,7 +219,7 @@ public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParamet } @Test - public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void resyncUserWithSyncedGroups() throws CatalogException { // We create synced groups and not synced groups in study studyUid Group group = new Group("@notSyncedGroup", Arrays.asList(normalUserId1, normalUserId2, normalUserId3)); catalogStudyDBAdaptor.createGroup(studyUid, group); @@ -293,7 +291,7 @@ public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParam } @Test - public void updateUserToGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void updateUserToGroups() throws CatalogException { // We create synced groups and not synced groups in study studyUid Group group = new Group("@notSyncedGroup", Collections.emptyList()); catalogStudyDBAdaptor.createGroup(studyUid, group); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java index a007c100b0f..8e4571d86ea 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java @@ -100,7 +100,7 @@ public void getUserTest() throws CatalogDBException, CatalogParameterException, } @Test - public void changePasswordTest() throws CatalogDBException, CatalogAuthenticationException { + public void changePasswordTest() throws CatalogException { DataResult result = catalogUserDBAdaptor.changePassword(normalUserId1, TestParamConstants.PASSWORD, "1234"); assertEquals(1, result.getNumUpdated()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java index 7bfda8c5261..f23295fbd60 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java @@ -438,7 +438,7 @@ public void testModifyUser() throws CatalogException, InterruptedException, IOEx assertEquals(userPost.getEmail(), newEmail); catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); - CatalogDBException exception = assertThrows(CatalogDBException.class, + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD)); assertTrue(exception.getMessage().contains("The new password has already been used")); @@ -513,7 +513,7 @@ public void changePasswordTest() throws CatalogException { catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword); catalogManager.getUserManager().login(organizationId, normalUserId1, newPassword); - CatalogDBException exception = assertThrows(CatalogDBException.class, + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword)); assertTrue(exception.getMessage().contains("verify that the current password is correct")); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java index ce30fd63c41..938ebd6dc7f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java @@ -11,16 +11,16 @@ public class OrganizationUserUpdateParams extends UserUpdateParams { private UserQuota quota; - private Account account; + private Internal internal; private Map attributes; public OrganizationUserUpdateParams() { } - public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Account account, Map attributes) { + public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Internal internal, Map attributes) { super(name, email); this.quota = quota; - this.account = account; + this.internal = internal; this.attributes = attributes; } @@ -33,7 +33,7 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { public String toString() { final StringBuilder sb = new StringBuilder("OrganizationUserUpdateParams{"); sb.append("quota=").append(quota); - sb.append(", account=").append(account); + sb.append(", internal=").append(internal); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); @@ -48,12 +48,27 @@ public OrganizationUserUpdateParams setQuota(UserQuota quota) { return this; } + @Deprecated + @JsonIgnore public Account getAccount() { - return account; + return getInternal().getAccount(); } + @Deprecated public OrganizationUserUpdateParams setAccount(Account account) { - this.account = account; + if (internal == null) { + internal = new Internal(); + } + internal.setAccount(account); + return this; + } + + public Internal getInternal() { + return internal; + } + + public OrganizationUserUpdateParams setInternal(Internal internal) { + this.internal = internal; return this; } @@ -78,6 +93,30 @@ public OrganizationUserUpdateParams setEmail(String email) { return this; } + public static class Internal { + private Account account; + + public Internal() { + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Internal{"); + sb.append("account=").append(account); + sb.append('}'); + return sb.toString(); + } + + public Account getAccount() { + return account; + } + + public Internal setAccount(Account account) { + this.account = account; + return this; + } + } + public static class Account { private String expirationDate; From 5c8c2142c3aa3e7fdf9953663fce84aab21eedc9 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 9 Aug 2024 11:23:38 +0200 Subject: [PATCH 099/128] Fix Kill job method #TASK-6636 --- .github/workflows/pull-request-approved.yml | 2 +- opencga-app/app/misc/clients/javascript_client_generator.py | 2 +- opencga-client/src/main/javascript/ClinicalAnalysis.js | 4 ++-- opencga-client/src/main/javascript/File.js | 2 +- opencga-client/src/main/javascript/GA4GH.js | 4 ++-- opencga-client/src/main/javascript/Job.js | 2 +- opencga-client/src/main/javascript/Project.js | 2 +- opencga-client/src/main/javascript/Study.js | 2 +- opencga-client/src/main/javascript/User.js | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 753466dd67a..d3eaf4eaf0c 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -15,5 +15,5 @@ jobs: uses: ./.github/workflows/test-analysis.yml needs: build with: - test_profile: runShortTests,runMediumTests,runLongTests + test_profile: runShortTests,runMediumTests secrets: inherit diff --git a/opencga-app/app/misc/clients/javascript_client_generator.py b/opencga-app/app/misc/clients/javascript_client_generator.py index 2ac2fb88667..e471d09ac48 100644 --- a/opencga-app/app/misc/clients/javascript_client_generator.py +++ b/opencga-app/app/misc/clients/javascript_client_generator.py @@ -116,7 +116,7 @@ def get_method_definition(self, category, endpoint): f'"{self.get_endpoint_subcategory()}"' if self.subcategory else "null", self.get_endpoint_id2() if self.get_endpoint_id2() else "null", f'"{self.get_endpoint_action()}"' if self.get_endpoint_action() else "null", - "data" if self.has_body() else False, + "data" if self.has_body() else ("null" if self.get_endpoint_method(endpoint).lower() == "post" else False), query_string_params ] if s) return (f' {self.get_method_doc(endpoint)}' diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index f2e4031455c..d5796f0b596 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -834,7 +834,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ clearInterpretation(clinicalAnalysis, interpretations, params) { - return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "clear", params); + return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "clear", null, params); } /** Delete interpretation @@ -859,7 +859,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ revertInterpretation(clinicalAnalysis, interpretation, version, params) { - return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "revert", {version, ...params}); + return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "revert", null, {version, ...params}); } /** Update interpretation fields diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index 3e1abab6f85..cfedee4d933 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -266,7 +266,7 @@ export default class File extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ upload(params) { - return this._post("files", null, null, null, "upload", params); + return this._post("files", null, null, null, "upload", null, params); } /** Return the acl defined for the file or folder. If member is provided, it will only return the acl for the member. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index 02af95b1363..222eaa7c67c 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -39,7 +39,7 @@ export default class GA4GH extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ searchReads() { - return this._post("ga4gh", null, "reads", null, "search"); + return this._post("ga4gh", null, "reads", null, "search", null); } /** Fetch alignment files using HTSget protocol @@ -78,7 +78,7 @@ export default class GA4GH extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ searchVariants() { - return this._post("ga4gh", null, "variants", null, "search"); + return this._post("ga4gh", null, "variants", null, "search", null); } } \ No newline at end of file diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index f0ebb640147..bb88f18dd7a 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -213,7 +213,7 @@ export default class Job extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ kill(job, params) { - return this._post("jobs", job, null, null, "kill", params); + return this._post("jobs", job, null, null, "kill", null, params); } /** Show the first lines of a log file (up to a limit) diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index 524221df852..041fa7ed691 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -85,7 +85,7 @@ export default class Project extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ incRelease(project) { - return this._post("projects", project, null, null, "incRelease"); + return this._post("projects", project, null, null, "incRelease", null); } /** Fetch all the studies contained in the project diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 1fa1c7a1c1b..bbe53e93df9 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -281,7 +281,7 @@ export default class Study extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ uploadTemplates(study, params) { - return this._post("studies", study, "templates", null, "upload", params); + return this._post("studies", study, "templates", null, "upload", null, params); } /** Delete template diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index 843416dd14e..32b28182c79 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -39,7 +39,7 @@ export default class User extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ anonymous(organization) { - return this._post("users", null, null, null, "anonymous", organization); + return this._post("users", null, null, null, "anonymous", null, organization); } /** Create a new user From f5c19954585b9a1fef164fd29b468a51882921a0 Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 9 Aug 2024 12:11:14 +0200 Subject: [PATCH 100/128] catalog: backwards compatibility, #TASK-6494 --- .../opencga/catalog/db/api/UserDBAdaptor.java | 1 + .../db/mongodb/UserMongoDBAdaptor.java | 27 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index 52b0022f533..84efa64aea2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -119,6 +119,7 @@ enum QueryParams implements QueryParam { CREATION_DATE("creationDate", TEXT_ARRAY, ""), MODIFICATION_DATE("modificationDate", TEXT_ARRAY, ""), DEPRECATED_ACCOUNT("account", OBJECT, ""), // Deprecated since 3.2.1 #TASK-6494 TODO: Remove in future releases + DEPRECATED_ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), // Deprecated since 3.2.1 #TASK-6494 INTERNAL("internal", OBJECT, ""), INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 775b6cd3b2c..940e5d264f9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -41,6 +41,7 @@ import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; @@ -165,7 +166,11 @@ public OpenCGAResult changePassword(String userId, String oldPassword, String ne public void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException { Bson query = Filters.and( Filters.eq(QueryParams.ID.key(), userId), - Filters.eq(INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA) + // TODO: Deprecated. Remove Filters.or using the deprecated account authentication id + Filters.or( + Filters.eq(DEPRECATED_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA), + Filters.eq(INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA) + ) ); Bson projection = Projections.include(PRIVATE_PASSWORD); DataResult dataResult = userCollection.find(query, projection, QueryOptions.empty()); @@ -173,7 +178,25 @@ public void authenticate(String userId, String password) throws CatalogDBExcepti throw new CatalogDBException("User " + userId + " not found"); } Document userDocument = dataResult.first(); - Document rootPasswordDoc = userDocument.get(PRIVATE_PASSWORD, Document.class); + Object rootPasswordObject = userDocument.get(PRIVATE_PASSWORD); + Document rootPasswordDoc; + // TODO: Remove this block of code in the future when all users have been migrated + if (rootPasswordObject instanceof String) { + if (ParamConstants.OPENCGA_USER_ID.equals(userId)) { + logger.warn("User {} is using the deprecated password format. Please, migrate your code as soon as possible.", userId); + if (!encryptPassword(password).equals(rootPasswordObject)) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); + } + return; + } else { + throw new CatalogDBException("User '" + userId + "' is using the deprecated password format. Please, ask your" + + " administrator to run the pending migrations to fix this issue."); + } + } else { + rootPasswordDoc = (Document) rootPasswordObject; + } + // TODO: End of block of code to remove (and replace using commented code below) +// Document rootPasswordDoc = userDocument.get(PRIVATE_PASSWORD, Document.class); if (rootPasswordDoc == null) { throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + " with the developers."); From 85f58f37d6575506915bb1bf203a58de056789d6 Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 9 Aug 2024 12:25:24 +0200 Subject: [PATCH 101/128] catalog: increase random salt length to 32, #TASK-6494 --- .../opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java | 4 ++-- .../java/org/opencb/opencga/core/common/PasswordUtils.java | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 940e5d264f9..2375ddc03f2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -137,7 +137,7 @@ private void insert(ClientSession clientSession, User user, String password) thr Document privatePassword = new Document(); if (StringUtils.isNotEmpty(password)) { - String salt = PasswordUtils.getStrongRandomPassword(); + String salt = PasswordUtils.getStrongRandomSalt(); String hash = encryptPassword(password + salt); Document passwordDoc = new Document() .append(HASH, hash) @@ -261,7 +261,7 @@ public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, St } // 4. Generate new salt for current password - String newSalt = PasswordUtils.getStrongRandomPassword(); + String newSalt = PasswordUtils.getStrongRandomSalt(); String newHash = encryptPassword(newPassword + newSalt); // 5. Generate update document diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java index d0e102cebed..e6e99ed2768 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java @@ -29,6 +29,10 @@ public static String getStrongRandomPassword() { return getStrongRandomPassword(DEFAULT_PASSWORD_LENGTH); } + public static String getStrongRandomSalt() { + return getStrongRandomPassword(32); + } + public static String getStrongRandomPassword(int length) { CharacterRule upper = new CharacterRule(EnglishCharacterData.UpperCase); CharacterRule lower = new CharacterRule(EnglishCharacterData.LowerCase); From 893787ca74c58fc23107dea10ac80c96706f3cfe Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 9 Aug 2024 12:28:31 +0200 Subject: [PATCH 102/128] core: add constant variable for the salt lenght, #TASK-6494 --- .../java/org/opencb/opencga/core/common/PasswordUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java index e6e99ed2768..91fb20b8455 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java @@ -10,6 +10,7 @@ public class PasswordUtils { public static final int MIN_STRONG_PASSWORD_LENGTH = 8; public static final int DEFAULT_PASSWORD_LENGTH = 10; + public static final int DEFAULT_SALT_LENGTH = 32; public static final String PASSWORD_REQUIREMENT = "Password must contain at least " + MIN_STRONG_PASSWORD_LENGTH + " characters, including at least one uppercase letter, one lowercase letter, one digit and one special character."; @@ -30,7 +31,7 @@ public static String getStrongRandomPassword() { } public static String getStrongRandomSalt() { - return getStrongRandomPassword(32); + return getStrongRandomPassword(DEFAULT_SALT_LENGTH); } public static String getStrongRandomPassword(int length) { From 00ddc0a0d6be471aaf41715595242c5b341c1fa2 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 9 Aug 2024 12:43:17 +0200 Subject: [PATCH 103/128] Fix compile error CELLBASE_DATA_RELEASE_GRCH38 #TASK-6515 --- .../storage/core/variant/VariantStorageEngineSVTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 05f090a162f..1573b27fd1d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -65,7 +65,7 @@ public void before() throws Exception { } variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); variantStorageEngine.reloadCellbaseConfiguration(); solr.configure(variantStorageEngine); From 759a7c64a4f4d6115bfb27d94729b979bfa60659 Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 9 Aug 2024 16:41:45 +0200 Subject: [PATCH 104/128] catalog: add salt to encryptPassword method, #TASK-6494 --- .../db/mongodb/UserMongoDBAdaptor.java | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 2375ddc03f2..62d83109679 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -138,7 +138,7 @@ private void insert(ClientSession clientSession, User user, String password) thr Document privatePassword = new Document(); if (StringUtils.isNotEmpty(password)) { String salt = PasswordUtils.getStrongRandomSalt(); - String hash = encryptPassword(password + salt); + String hash = encryptPassword(password, salt); Document passwordDoc = new Document() .append(HASH, hash) .append(SALT, salt); @@ -184,7 +184,7 @@ public void authenticate(String userId, String password) throws CatalogDBExcepti if (rootPasswordObject instanceof String) { if (ParamConstants.OPENCGA_USER_ID.equals(userId)) { logger.warn("User {} is using the deprecated password format. Please, migrate your code as soon as possible.", userId); - if (!encryptPassword(password).equals(rootPasswordObject)) { + if (!encryptPassword(password, "").equals(rootPasswordObject)) { throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); } return; @@ -208,7 +208,7 @@ public void authenticate(String userId, String password) throws CatalogDBExcepti } String salt = passwordDoc.getString(SALT); - String hash = encryptPassword(password + salt); + String hash = encryptPassword(password, salt); if (!hash.equals(passwordDoc.getString(HASH))) { throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); } @@ -236,7 +236,7 @@ public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, St if (StringUtils.isNotEmpty(oldPassword)) { Document currentPasswordDoc = passwordDoc.get(CURRENT, Document.class); String currentSalt = currentPasswordDoc.getString(SALT); - String currentHash = encryptPassword(oldPassword + currentSalt); + String currentHash = encryptPassword(oldPassword, currentSalt); if (!currentHash.equals(currentPasswordDoc.getString(HASH))) { throw new CatalogAuthenticationException(prefixErrorMsg + "Please, verify that the current password is correct."); } @@ -248,7 +248,7 @@ public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, St .map(document -> document.getString(SALT)) .collect(Collectors.toList()); for (String saltValue : saltValues) { - hashValues.add(encryptPassword(newPassword + saltValue)); + hashValues.add(encryptPassword(newPassword, saltValue)); } // 3. Check new password has not been used before @@ -262,7 +262,7 @@ public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, St // 4. Generate new salt for current password String newSalt = PasswordUtils.getStrongRandomSalt(); - String newHash = encryptPassword(newPassword + newSalt); + String newHash = encryptPassword(newPassword, newSalt); // 5. Generate update document UpdateDocument updateDocument = new UpdateDocument(); @@ -739,18 +739,10 @@ public void forEach(Query query, Consumer action, QueryOptions o } } - public static void main(String[] args) throws CatalogDBException { - System.out.println(encryptPassword("admin")); - } - - private static String encryptPassword(String password) throws CatalogDBException { + private static String encryptPassword(String password, String salt) throws CatalogDBException { if (StringUtils.isNotEmpty(password)) { - if (password.matches("^[a-fA-F0-9]{40}$")) { - // Password already cyphered - return password; - } try { - return CryptoUtils.sha1(password); + return CryptoUtils.sha1(password + salt); } catch (NoSuchAlgorithmException e) { throw new CatalogDBException("Could not encrypt password", e); } From d32104498c863eeed159d9ff4494e7aa8466f013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 13 Aug 2024 10:42:24 +0100 Subject: [PATCH 105/128] storage: Fix merge issue with ASSEMBLY. #TASK-6515 --- .../storage/core/variant/VariantStorageEngineBNDTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index cd8e37b6444..a675e5304b0 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -48,6 +48,7 @@ public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); if (!loaded) { clearDB(DB_NAME); @@ -60,7 +61,7 @@ protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); - + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); studyMetadata = new StudyMetadata(1, "s1"); // variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); input1 = getResourceUri("variant-test-bnd.vcf"); From 8c1cde14620673bba3cd6c25552abce2fb7ee757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 13 Aug 2024 16:35:48 +0100 Subject: [PATCH 106/128] storage: Embedded solr should be optional in tests. #TASK-6515 --- .../variant/VariantStorageEngineSVTest.java | 10 ++--- .../HadoopVariantStorageEngineSVTest.java | 45 +++++++++++++++++-- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 1573b27fd1d..2523fad5ba2 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -1,7 +1,7 @@ package org.opencb.opencga.storage.core.variant; +import org.junit.Assume; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Ignore; import org.junit.Test; import org.opencb.biodata.formats.variant.io.VariantReader; @@ -25,7 +25,6 @@ import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutor; import org.opencb.opencga.storage.core.variant.search.SearchIndexVariantQueryExecutor; -import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; import java.net.URI; import java.nio.file.Paths; @@ -55,9 +54,6 @@ public abstract class VariantStorageEngineSVTest extends VariantStorageBaseTest protected static URI input2; protected static URI input3; - @ClassRule - public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); - @Before public void before() throws Exception { if (!loaded) { @@ -68,7 +64,7 @@ public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); variantStorageEngine.reloadCellbaseConfiguration(); - solr.configure(variantStorageEngine); + if (!loaded) { loadFiles(); loaded = true; @@ -97,7 +93,6 @@ protected void loadFiles() throws Exception { .append(VariantStorageOptions.STATS_CALCULATE.key(), true) .append(VariantStorageOptions.ASSEMBLY.key(), "grch38")); - variantStorageEngine.secondaryIndex(); } @Test @@ -120,6 +115,7 @@ public void checkCount() throws Exception { @Test public void checkSecondaryAnnotationIndex() throws Exception { + Assume.assumeTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); VariantQueryExecutor variantQueryExecutor = variantStorageEngine.getVariantQueryExecutor(SearchIndexVariantQueryExecutor.class); for (Variant variant : variantStorageEngine) { ParsedVariantQuery query = variantStorageEngine diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java index fbcbd773477..26bcb49bb0f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java @@ -1,8 +1,6 @@ package org.opencb.opencga.storage.hadoop.variant; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.*; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; @@ -12,13 +10,15 @@ import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import static org.opencb.opencga.core.api.ParamConstants.OVERWRITE; /** @@ -33,9 +33,46 @@ public class HadoopVariantStorageEngineSVTest extends VariantStorageEngineSVTest public static HadoopExternalResource externalResource = new HadoopExternalResource(); private Logger logger = LoggerFactory.getLogger(getClass()); + public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); + + @BeforeClass + public static void beforeClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.before(); + solr.configure(externalResource.getVariantStorageEngine()); + System.out.println("Start embedded solr"); + } else { + System.out.println("Skip embedded solr tests"); + } + } + + @AfterClass + public static void afterClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.after(); + } + } + + @Override + public void before() throws Exception { + super.before(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } + } + @Override protected void loadFiles() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } super.loadFiles(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + variantStorageEngine.secondaryIndex(); + assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } else { + assertFalse(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); } From fdc4d89207e86ebf123584696d09b863285cac9b Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 19 Aug 2024 09:39:35 +0200 Subject: [PATCH 107/128] catalog: simplify code, #TASK-6494 --- .../db/mongodb/UserMongoDBAdaptor.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 62d83109679..24c71463e32 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -242,29 +242,22 @@ public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, St } } - // 2. Calculate all possible hashValues with new password - Set hashValues = new HashSet<>(); - List saltValues = passwordDoc.getList(ARCHIVE, Document.class).stream() - .map(document -> document.getString(SALT)) - .collect(Collectors.toList()); - for (String saltValue : saltValues) { - hashValues.add(encryptPassword(newPassword, saltValue)); - } - - // 3. Check new password has not been used before + // 2. Check new password has not been used before for (Document document : passwordDoc.getList(ARCHIVE, Document.class)) { String hashValue = document.getString(HASH); - if (hashValues.contains(hashValue)) { + String saltValue = document.getString(SALT); + String encryptedPassword = encryptPassword(newPassword, saltValue); + if (encryptedPassword.equals(hashValue)) { throw new CatalogAuthenticationException(prefixErrorMsg + "The new password has already been used." + " Please, use a different one."); } } - // 4. Generate new salt for current password + // 3. Generate new salt for current password String newSalt = PasswordUtils.getStrongRandomSalt(); String newHash = encryptPassword(newPassword, newSalt); - // 5. Generate update document + // 4. Generate update document UpdateDocument updateDocument = new UpdateDocument(); // add to current updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_HASH, newHash); From a2579af876597da62e94ed6c2ee5fac924ff56d0 Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 19 Aug 2024 14:52:46 +0200 Subject: [PATCH 108/128] catalog: add missing snapshot to parse query method, #TASK-5964 --- .../catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index 2b9d84cd876..e4fb4672afb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -1374,6 +1374,9 @@ private Bson parseQuery(Query query, Document extraQuery, String user) queryCopy.get(Constants.PRIVATE_ANNOTATION_PARAM_TYPES, ObjectMap.class)); } break; + case SNAPSHOT: + addAutoOrQuery(RELEASE_FROM_VERSION, queryParam.key(), queryCopy, queryParam.type(), andBsonList); + break; // Other parameter that can be queried. case ID: case UUID: From 06ad286eabaa93c6530a0c33e04a55e360cdeec1 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 20 Aug 2024 16:39:00 +0200 Subject: [PATCH 109/128] catalog: allow users with ADMIN to update Interpretations, #TASK-5964 --- .../managers/InterpretationManager.java | 189 +++++++++--------- .../managers/InterpretationManagerTest.java | 83 +++++--- 2 files changed, 147 insertions(+), 125 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 7f7be104d95..3ff808034cd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -641,89 +641,89 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly // } // } - public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, - ParamUtils.SaveInterpretationAs as, QueryOptions options, String token) - throws CatalogException { - return update(studyStr, query, updateParams, as, false, options, token); - } - - public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, - ParamUtils.SaveInterpretationAs as, boolean ignoreException, QueryOptions options, - String token) throws CatalogException { - options = ParamUtils.defaultObject(options, QueryOptions::new); - - JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); - CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); - String organizationId = studyFqn.getOrganizationId(); - String userId = tokenPayload.getUserId(organizationId); - Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); - - String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); - - ObjectMap updateMap; - try { - updateMap = updateParams != null ? updateParams.getUpdateMap() : null; - } catch (JsonProcessingException e) { - throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); - } - - ObjectMap auditParams = new ObjectMap() - .append("study", studyStr) - .append("query", query) - .append("updateParams", updateMap) - .append("as", as) - .append("ignoreException", ignoreException) - .append("options", options) - .append("token", token); - - Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); - fixQueryObject(organizationId, study, finalQuery, userId); - - DBIterator iterator; - try { - finalQuery.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = getInterpretationDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_INTERPRETATION_FINDING_IDS, - userId); - } catch (CatalogException e) { - auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, "", "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - throw e; - } - - auditManager.initAuditBatch(operationId); - OpenCGAResult result = OpenCGAResult.empty(); - while (iterator.hasNext()) { - Interpretation interpretation = iterator.next(); - try { - List clinicalAuditList = new ArrayList<>(); - clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_INTERPRETATION, - "Update interpretation '" + interpretation.getId() + "'", TimeUtils.getTime())); - if (as != null) { - clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, - "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); - } - OpenCGAResult writeResult = update(organizationId, study, interpretation, updateParams, clinicalAuditList, as, options, - userId); - auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), - interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - - result.append(writeResult); - } catch (CatalogException e) { - Event event = new Event(Event.Type.ERROR, interpretation.getId(), e.getMessage()); - result.getEvents().add(event); - result.setNumErrors(result.getNumErrors() + 1); - - logger.error("Cannot update interpretation {}: {}", interpretation.getId(), e.getMessage(), e); - auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), - interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - } - } - auditManager.finishAuditBatch(organizationId, operationId); - - return endResult(result, ignoreException); - } +// public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, +// ParamUtils.SaveInterpretationAs as, QueryOptions options, String token) +// throws CatalogException { +// return update(studyStr, query, updateParams, as, false, options, token); +// } +// +// public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, +// ParamUtils.SaveInterpretationAs as, boolean ignoreException, QueryOptions options, +// String token) throws CatalogException { +// options = ParamUtils.defaultObject(options, QueryOptions::new); +// +// JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); +// CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); +// String organizationId = studyFqn.getOrganizationId(); +// String userId = tokenPayload.getUserId(organizationId); +// Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); +// +// String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); +// +// ObjectMap updateMap; +// try { +// updateMap = updateParams != null ? updateParams.getUpdateMap() : null; +// } catch (JsonProcessingException e) { +// throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); +// } +// +// ObjectMap auditParams = new ObjectMap() +// .append("study", studyStr) +// .append("query", query) +// .append("updateParams", updateMap) +// .append("as", as) +// .append("ignoreException", ignoreException) +// .append("options", options) +// .append("token", token); +// +// Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); +// fixQueryObject(organizationId, study, finalQuery, userId); +// +// DBIterator iterator; +// try { +// finalQuery.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); +// iterator = getInterpretationDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_INTERPRETATION_FINDING_IDS, +// userId); +// } catch (CatalogException e) { +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, "", "", study.getId(), +// study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); +// throw e; +// } +// +// auditManager.initAuditBatch(operationId); +// OpenCGAResult result = OpenCGAResult.empty(); +// while (iterator.hasNext()) { +// Interpretation interpretation = iterator.next(); +// try { +// List clinicalAuditList = new ArrayList<>(); +// clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_INTERPRETATION, +// "Update interpretation '" + interpretation.getId() + "'", TimeUtils.getTime())); +// if (as != null) { +// clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, +// "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); +// } +// OpenCGAResult writeResult = update(organizationId, study, interpretation, updateParams, clinicalAuditList, as, options, +// userId); +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), +// interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, +// new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); +// +// result.append(writeResult); +// } catch (CatalogException e) { +// Event event = new Event(Event.Type.ERROR, interpretation.getId(), e.getMessage()); +// result.getEvents().add(event); +// result.setNumErrors(result.getNumErrors() + 1); +// +// logger.error("Cannot update interpretation {}: {}", interpretation.getId(), e.getMessage(), e); +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), +// interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, +// new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); +// } +// } +// auditManager.finishAuditBatch(organizationId, operationId); +// +// return endResult(result, ignoreException); +// } public OpenCGAResult update(String studyStr, String clinicalAnalysisId, String intepretationId, InterpretationUpdateParams updateParams, ParamUtils.SaveInterpretationAs as, @@ -927,10 +927,10 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation // Check if user has permissions to write clinical analysis ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().internalGet(organizationId, study.getUid(), interpretation.getClinicalAnalysisId(), INCLUDE_CLINICAL_ANALYSIS, userId).first(); - if (clinicalAnalysis.isLocked()) { - throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" - + " the Interpretation."); - } +// if (clinicalAnalysis.isLocked()) { +// throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" +// + " the Interpretation."); +// } if (interpretation.isLocked() || updateParams.getLocked() != null) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, @@ -940,6 +940,11 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation ClinicalAnalysisPermissions.WRITE); } + if (clinicalAnalysis.isLocked() && updateParams.getLocked() != null && !updateParams.getLocked()) { + throw new CatalogException("Could not unlock the Interpretation. Case is locked so unlocking the Interpretation is not" + + " allowed."); + } + List events = new ArrayList<>(); if (updateParams != null && StringUtils.isNotEmpty(updateParams.getCreationDate())) { @@ -958,11 +963,11 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation } } - if (!parameters.isEmpty() && interpretation.isLocked() - && parameters.getBoolean(InterpretationDBAdaptor.QueryParams.LOCKED.key(), true)) { - throw new CatalogException("Could not update the Interpretation. Interpretation '" + interpretation.getId() - + " is locked. Please, unlock it first."); - } +// if (!parameters.isEmpty() && interpretation.isLocked() +// && parameters.getBoolean(InterpretationDBAdaptor.QueryParams.LOCKED.key(), true)) { +// throw new CatalogException("Could not update the Interpretation. Interpretation '" + interpretation.getId() +// + " is locked. Please, unlock it first."); +// } if (updateParams != null && updateParams.getComments() != null && !updateParams.getComments().isEmpty()) { List comments = new ArrayList<>(updateParams.getComments().size()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java index 99232c2e11d..217aa3f4899 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java @@ -12,15 +12,18 @@ import org.opencb.opencga.core.models.clinical.*; import org.opencb.opencga.core.models.common.StatusParam; import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.panel.PanelReferenceParam; import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.study.StudyAclParams; +import org.opencb.opencga.core.models.study.StudyPermissions; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import static org.junit.Assert.*; @@ -58,25 +61,6 @@ private DataResult createDummyEnvironment(boolean createFamily INCLUDE_RESULT, ownerToken); } - private List registerDummyFiles() throws CatalogException { - List files = new LinkedList<>(); - - String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - vcfFile = getClass().getResource("/biofiles/family.vcf").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - String bamFile = getClass().getResource("/biofiles/HG00096.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - - return files; - } - @Test public void deleteLockedInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); @@ -149,15 +133,34 @@ public void interpretationLockedTest() throws CatalogException { assertFalse(secondaryInterpretation.isLocked()); } + // Add ADMIN permissions to the user2 + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, + new StudyAclParams(StudyPermissions.Permissions.ADMIN_CLINICAL_ANALYSIS.name(), null), ParamUtils.AclAction.SET, ownerToken); + // Try to update interpretation 1 try { catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), - new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), ownerToken); + new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), normalToken1); fail("Interpretation is locked so it should not allow this"); } catch (CatalogException e) { - assertTrue(e.getMessage().contains("locked")); + assertTrue(e.getMessage().contains(ClinicalAnalysisPermissions.ADMIN.name())); } + // Try to update interpretation 1 + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), normalToken2); + Interpretation interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), + QueryOptions.empty(), ownerToken).first(); + assertEquals("blabla", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + + // Try to update interpretation 1 + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("blabla2"), null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("blabla2", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + // Update interpretation 2 catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), ownerToken); @@ -173,15 +176,6 @@ public void interpretationLockedTest() throws CatalogException { assertEquals("bloblo", interpretation2.getDescription()); assertTrue(interpretation2.isLocked()); - // Try to lock again and update interpretation 2 - try { - catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), - new InterpretationUpdateParams().setDescription("blabla").setLocked(true), null, QueryOptions.empty(), ownerToken); - fail("Interpretation was already locked so it should not allow this"); - } catch (CatalogException e) { - assertTrue(e.getMessage().contains("locked")); - } - // Unlock and update interpretation 2 catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), new InterpretationUpdateParams().setDescription("blabla").setLocked(false), null, QueryOptions.empty(), ownerToken); @@ -208,6 +202,29 @@ public void interpretationLockedTest() throws CatalogException { assertTrue(secondaryInterpretation.isLocked()); } + // Try to update the interpretation 1 + try { + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description"), null, QueryOptions.empty(), normalToken1); + fail("Case and Interpretation are locked so it should not allow this"); + } catch (CatalogException e) { + assertTrue(e.getMessage().contains(ClinicalAnalysisPermissions.ADMIN.name()) && e.getMessage().toLowerCase().contains("permission denied")); + } + + // Try to update the interpretation 1 (ADMIN permission) + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description"), null, QueryOptions.empty(), normalToken2); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("new description", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + + // Try to update the interpretation 1 (owner user) + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description2"), null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("new description2", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + // Try to unlock interpretation 1 try { catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), From 7c3f885734fce7d6e3183d8a9324495432b8f466 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 21 Aug 2024 11:57:44 +0200 Subject: [PATCH 110/128] catalog: check Interpretations can't be touched when CLOSED, #TASK-5964 --- .../managers/InterpretationManager.java | 60 +++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 3ff808034cd..5ace84fca9b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -69,18 +69,16 @@ public class InterpretationManager extends ResourceManager { public static final QueryOptions INCLUDE_CLINICAL_ANALYSIS = keepFieldsInQueryOptions(ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, - Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key())); + Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key(), + ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key())); public static final QueryOptions INCLUDE_INTERPRETATION_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( InterpretationDBAdaptor.QueryParams.ID.key(), InterpretationDBAdaptor.QueryParams.UID.key(), InterpretationDBAdaptor.QueryParams.UUID.key(), InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), - InterpretationDBAdaptor.QueryParams.LOCKED.key(), + InterpretationDBAdaptor.QueryParams.LOCKED.key(), InterpretationDBAdaptor.QueryParams.STATUS.key(), InterpretationDBAdaptor.QueryParams.VERSION.key(), InterpretationDBAdaptor.QueryParams.STUDY_UID.key())); - public static final QueryOptions INCLUDE_INTERPRETATION_FINDING_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - InterpretationDBAdaptor.QueryParams.ID.key(), InterpretationDBAdaptor.QueryParams.UID.key(), - InterpretationDBAdaptor.QueryParams.UUID.key(), InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), - InterpretationDBAdaptor.QueryParams.VERSION.key(), InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), - InterpretationDBAdaptor.QueryParams.LOCKED.key(), InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS_ID.key(), - InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS_ID.key())); + public static final QueryOptions INCLUDE_INTERPRETATION_FINDING_IDS = keepFieldsInQueryOptions(INCLUDE_INTERPRETATION_IDS, + Arrays.asList(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS_ID.key(), + InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS_ID.key())); protected static Logger logger = LoggerFactory.getLogger(InterpretationManager.class); private UserManager userManager; private StudyManager studyManager; @@ -931,11 +929,51 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation // throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" // + " the Interpretation."); // } + if (clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + throw new CatalogException("Cannot update the Interpretation. Case status is " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } - if (interpretation.isLocked() || updateParams.getLocked() != null) { + InterpretationStudyConfiguration interpretationStudyConfiguration = study.getInternal().getConfiguration().getClinical() + .getInterpretation(); + // Get the interpretation status that are CLOSED and DONE + Set closedStatus = new HashSet<>(); + Set doneStatus = new HashSet<>(); + for (ClinicalStatusValue clinicalStatusValue : interpretationStudyConfiguration.getStatus()) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + closedStatus.add(clinicalStatusValue.getId()); + } else if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.DONE)) { + doneStatus.add(clinicalStatusValue.getId()); + } + } + + // If the current interpretation: + // - is locked + // - the user wants to update the locked status + // - the user wants to update the status to/from a done|closed status + boolean adminPermissionsChecked = false; + if (interpretation.isLocked() + || interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED + || interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.DONE + || updateParams.getLocked() != null + || (updateParams.getStatus() != null && (closedStatus.contains(updateParams.getStatus().getId()) + || doneStatus.contains(updateParams.getStatus().getId())))) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.ADMIN); - } else { + + // Current status is of type CLOSED + if (interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + // The only allowed action is to remove the CLOSED status + if (updateParams.getStatus() == null || closedStatus.contains(updateParams.getStatus().getId())) { + throw new CatalogException("Cannot update a Interpretation with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the Interpretation."); + } + } + + adminPermissionsChecked = true; + } + + if (!adminPermissionsChecked) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE); } @@ -1012,7 +1050,7 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation // Validate and get panels List panelIds = updateParams.getPanels().stream().map(PanelReferenceParam::getId).collect(Collectors.toList()); Query query = new Query(PanelDBAdaptor.QueryParams.ID.key(), panelIds); - OpenCGAResult panelResult = + OpenCGAResult panelResult = getPanelDBAdaptor(organizationId).get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); if (panelResult.getNumResults() < panelIds.size()) { throw new CatalogException("Some panels were not found or user doesn't have permissions to see them"); From be6da9c1381fa7288ae25975984b7eabdf400794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 22 Aug 2024 09:02:19 +0100 Subject: [PATCH 111/128] storage: Define start-stop row scan for deleting files from archive table. #TASK-6756 --- .../hadoop/utils/DeleteHBaseColumnDriver.java | 7 ++- .../variant/HadoopVariantStorageEngine.java | 6 +- .../archive/ArchiveDeleteHBaseColumnTask.java | 42 +++++++++++++ .../variant/archive/ArchiveRowKeyFactory.java | 7 +++ .../ArchiveDeleteHBaseColumnTaskTest.java | 59 +++++++++++++++++++ 5 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java create mode 100644 opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java index 52879f01eed..53473a977b2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java @@ -81,10 +81,15 @@ public void setupJob(Job job, String table) throws IOException { List scans; if (!regions.isEmpty()) { - scans = new ArrayList<>(regions.size() / 2); + LOGGER.info("Delete rows from {} table ranges (start - end)", regions.size()); + scans = new ArrayList<>(regions.size()); for (Pair region : regions) { Scan scan = new Scan(templateScan); scans.add(scan); + LOGGER.info(" - [ '" + + Bytes.toStringBinary(region.getFirst()) + "' , '" + + Bytes.toStringBinary(region.getSecond()) + + "' )"); if (region.getFirst() != null && region.getFirst().length != 0) { scan.setStartRow(region.getFirst()); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index e0cbbfb3963..aa183adb007 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -77,6 +77,7 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchemaManager; import org.opencb.opencga.storage.hadoop.variant.adaptors.sample.HBaseVariantSampleDataManager; import org.opencb.opencga.storage.hadoop.variant.annotation.HadoopDefaultVariantAnnotationManager; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDeleteHBaseColumnTask; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutorFactory; @@ -826,7 +827,10 @@ private void remove(String study, List files, List samples, URI archiveColumns.add(family + ':' + ArchiveTableHelper.getRefColumnName(fileId)); archiveColumns.add(family + ':' + ArchiveTableHelper.getNonRefColumnName(fileId)); } - String[] deleteFromArchiveArgs = DeleteHBaseColumnDriver.buildArgs(archiveTable, archiveColumns, options); + ObjectMap thisOptions = new ObjectMap(options); + ArchiveDeleteHBaseColumnTask.configureTask(thisOptions, fileIds); + + String[] deleteFromArchiveArgs = DeleteHBaseColumnDriver.buildArgs(archiveTable, archiveColumns, thisOptions); getMRExecutor().run(DeleteHBaseColumnDriver.class, deleteFromArchiveArgs, "Delete from archive table"); return stopWatch.now(TimeUnit.MILLISECONDS); }); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java new file mode 100644 index 00000000000..0fb9fb0adbf --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.storage.hadoop.variant.archive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.storage.hadoop.utils.DeleteHBaseColumnDriver; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ArchiveDeleteHBaseColumnTask extends DeleteHBaseColumnDriver.DeleteHBaseColumnTask { + + public static final String FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX = "fileBatchesWithFilesToDeleteFromArchiveIndex"; + + public static void configureTask(ObjectMap options, List fileIds) { + ArchiveRowKeyFactory keyFactory = new ArchiveRowKeyFactory(options); + Set fileBatchesSet = new HashSet<>(); + for (Integer fileId : fileIds) { + fileBatchesSet.add(keyFactory.getFileBatch(fileId)); + } + List fileBatches = new ArrayList<>(fileBatchesSet); + fileBatches.sort(Integer::compareTo); + options.put(DeleteHBaseColumnDriver.DELETE_HBASE_COLUMN_TASK_CLASS, ArchiveDeleteHBaseColumnTask.class.getName()); + options.put(ArchiveDeleteHBaseColumnTask.FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX, fileBatches); + } + + @Override + public List> getRegionsToDelete(Configuration configuration) { + int[] fileBatches = configuration.getInts(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX); + List> regions = new ArrayList<>(); + ArchiveRowKeyFactory archiveRowKeyFactory = new ArchiveRowKeyFactory(configuration); + for (int fileBatch : fileBatches) { + regions.add(new Pair<>( + Bytes.toBytes(archiveRowKeyFactory.generateBlockIdFromBatch(fileBatch)), + Bytes.toBytes(archiveRowKeyFactory.generateBlockIdFromBatch(fileBatch + 1)))); + } + return regions; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java index 43e9cf97cf7..0f660bf4032 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java @@ -147,6 +147,13 @@ public String generateBlockIdFromSlice(int fileId, String chrom, long slice) { return generateBlockIdFromSliceAndBatch(getFileBatch(fileId), chrom, slice); } + public String generateBlockIdFromBatch(int fileBatch) { + StringBuilder sb = new StringBuilder(FILE_BATCH_PAD + 1); + sb.append(StringUtils.leftPad(String.valueOf(fileBatch), FILE_BATCH_PAD, '0')); + sb.append(getSeparator()); + return sb.toString(); + } + public String generateBlockIdFromSliceAndBatch(int fileBatch, String chrom, long slice) { String chromosome = Region.normalizeChromosome(chrom); StringBuilder sb = new StringBuilder(FILE_BATCH_PAD + 1 + chromosome.length() + 1 + POSITION_PAD); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java new file mode 100644 index 00000000000..5513bcdcb7a --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java @@ -0,0 +1,59 @@ +package org.opencb.opencga.storage.hadoop.variant.archive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Assert; +import org.junit.Test; +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDeleteHBaseColumnTask.*; + +public class ArchiveDeleteHBaseColumnTaskTest { + + private ArchiveDeleteHBaseColumnTask task; + + @Test + public void testConfigureTask() { + ObjectMap options = new ObjectMap(); + task = new ArchiveDeleteHBaseColumnTask(); + configureTask(options, Arrays.asList(1, 2, 3)); + assertEquals(Arrays.asList(0), options.getAsIntegerList(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX)); + List> regionsToDelete = task.getRegionsToDelete(toConf(options)); + assertEquals(1, regionsToDelete.size()); + + assertArrayEquals(Bytes.toBytes("00000_"), regionsToDelete.get(0).getFirst()); + assertArrayEquals(Bytes.toBytes("00001_"), regionsToDelete.get(0).getSecond()); + } + + @Test + public void testConfigureTaskMultiRegions() { + ObjectMap options = new ObjectMap(); + task = new ArchiveDeleteHBaseColumnTask(); + configureTask(options, Arrays.asList(5300, 6053, 9032)); + assertEquals(Arrays.asList(5, 6, 9), options.getAsIntegerList(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX)); + List> regionsToDelete = task.getRegionsToDelete(toConf(options)); + assertEquals(3, regionsToDelete.size()); + + assertArrayEquals(Bytes.toBytes("00005_"), regionsToDelete.get(0).getFirst()); + assertArrayEquals(Bytes.toBytes("00006_"), regionsToDelete.get(0).getSecond()); + assertArrayEquals(Bytes.toBytes("00006_"), regionsToDelete.get(1).getFirst()); + assertArrayEquals(Bytes.toBytes("00007_"), regionsToDelete.get(1).getSecond()); + assertArrayEquals(Bytes.toBytes("00009_"), regionsToDelete.get(2).getFirst()); + assertArrayEquals(Bytes.toBytes("00010_"), regionsToDelete.get(2).getSecond()); + } + + private static Configuration toConf(ObjectMap options) { + Configuration conf = new Configuration(); + for (String key : options.keySet()) { + conf.set(key, options.getString(key)); + } + return conf; + } + +} \ No newline at end of file From 3d31b1b67fceffcea290d486d6e90d8fcc38a808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 22 Aug 2024 09:37:15 +0100 Subject: [PATCH 112/128] storage: Fix param `--force` for vairant-file-delete #TASK-6737 --- .../operations/VariantDeleteOperationManager.java | 9 +++++---- .../options/OperationsVariantStorageCommandOptions.java | 6 +++--- .../operations/variant/VariantFileDeleteParams.java | 6 ++++++ .../opencga/master/monitor/daemons/ExecutionDaemon.java | 1 - .../cli/client/options/StorageVariantCommandOptions.java | 5 ++++- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java index 882240d2865..bc3ba9cb2ab 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java @@ -63,7 +63,8 @@ public void removeFile(String study, List inputFiles, URI outdir, String String catalogIndexStatus = file.getInternal().getVariant().getIndex().getStatus().getId(); if (!catalogIndexStatus.equals(VariantIndexStatus.READY)) { // Might be partially loaded in VariantStorage. Check FileMetadata - FileMetadata fileMetadata = variantStorageEngine.getMetadataManager().getFileMetadata(studyMetadata.getId(), fileStr); + FileMetadata fileMetadata = variantStorageEngine.getMetadataManager() + .getFileMetadata(studyMetadata.getId(), file.getName()); boolean canBeRemoved; if (force) { // When forcing remove, just require the file to be registered in the storage @@ -81,9 +82,9 @@ public void removeFile(String study, List inputFiles, URI outdir, String // filePaths.add(file.getPath()); } - if (fileNames.isEmpty()) { - throw new CatalogException("Nothing to do!"); - } + } + if (fileNames.isEmpty()) { + throw new CatalogException("Nothing to do!"); } variantStorageEngine.removeFiles(study, fileNames, outdir); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index 4d398070124..ac7b0b968fe 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -380,13 +380,13 @@ public class DeleteVariantCommandOptions { @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--file"}, description = "List of file ids to delete. Use 'all' to remove the whole study", required = false, arity = 1) public String file; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume failed delete operation.", required = false, help = true, arity = 0) public boolean resume = false; - @Parameter(names = {"--force"}, description = "The body web service force parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--force"}, description = "Force delete operation. This would allow deleting partially loaded files.", required = false, help = true, arity = 0) public boolean force = false; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java index 85ba64ce78c..2234cb489e7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java @@ -16,6 +16,7 @@ package org.opencb.opencga.core.models.operations.variant; +import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.tools.ToolParams; import java.util.List; @@ -32,8 +33,13 @@ public VariantFileDeleteParams(List file, boolean resume) { this.resume = resume; } + @DataField(description = "List of file ids to delete. Use 'all' to remove the whole study", required = true) private List file; + + @DataField(description = "Resume failed delete operation.", defaultValue = "false") private boolean resume; + + @DataField(description = "Force delete operation. This would allow deleting partially loaded files.", defaultValue = "false") private boolean force; public List getFile() { diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index 5956eb68e65..01c02bde4ad 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -205,7 +205,6 @@ public class ExecutionDaemon extends MonitorParentDaemon implements Closeable { put(RvtestsWrapperAnalysis.ID, "variant " + RvtestsWrapperAnalysis.ID + "-run"); put(GatkWrapperAnalysis.ID, "variant " + GatkWrapperAnalysis.ID + "-run"); put(ExomiserWrapperAnalysis.ID, "variant " + ExomiserWrapperAnalysis.ID + "-run"); - put(VariantFileDeleteOperationTool.ID, "variant file-delete"); put(VariantSecondaryAnnotationIndexOperationTool.ID, "variant secondary-index"); put(VariantSecondaryIndexSamplesDeleteOperationTool.ID, "variant secondary-index-delete"); put(VariantScoreDeleteOperationTool.ID, "variant score-delete"); diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java index 398b7f775cd..acd37799e0e 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java @@ -218,7 +218,10 @@ public static class GenericVariantDeleteOptions { splitter = CommaParameterSplitter.class, required = true) public List file = null; - @Parameter(names = {"--resume"}, description = "Resume a previously failed indexation") + @Parameter(names = {"--force"}, description = "Force delete operation. This would allow deleting partially loaded files.") + public boolean force; + + @Parameter(names = {"--resume"}, description = "Resume failed delete operation.") public boolean resume; } From fdf290423f52554b94c0a98d78a17addef3635de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 22 Aug 2024 10:53:47 +0100 Subject: [PATCH 113/128] storage: Add tests for variant-file-delete `--force`. #TASK-6737 --- .../VariantDeleteOperationManager.java | 15 +++++- .../AbstractVariantOperationManagerTest.java | 6 ++- .../operations/RemoveVariantsTest.java | 46 ++++++++++++++++- .../core/cellbase/CellBaseValidator.java | 50 +++++++++++++++---- 4 files changed, 104 insertions(+), 13 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java index bc3ba9cb2ab..c0f57f0b04f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java @@ -27,6 +27,8 @@ import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.URI; import java.util.ArrayList; @@ -34,6 +36,8 @@ public class VariantDeleteOperationManager extends OperationManager { + private final Logger logger = LoggerFactory.getLogger(VariantDeleteOperationManager.class); + public VariantDeleteOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) { super(variantStorageManager, engine); } @@ -65,6 +69,12 @@ public void removeFile(String study, List inputFiles, URI outdir, String // Might be partially loaded in VariantStorage. Check FileMetadata FileMetadata fileMetadata = variantStorageEngine.getMetadataManager() .getFileMetadata(studyMetadata.getId(), file.getName()); + if (fileMetadata != null && !fileMetadata.getPath().equals(file.getUri().getPath())) { + // FileMetadata path does not match the catalog path. This file is not registered in the storage. + throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. " + + "File is not registered in the storage. " + + "Instead, found file with same name but different path '" + fileMetadata.getPath() + "'"); + } boolean canBeRemoved; if (force) { // When forcing remove, just require the file to be registered in the storage @@ -74,8 +84,9 @@ public void removeFile(String study, List inputFiles, URI outdir, String canBeRemoved = fileMetadata != null && fileMetadata.getIndexStatus() != TaskMetadata.Status.NONE; } if (!canBeRemoved) { - throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " - + "IndexStatus = " + catalogIndexStatus); + throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. " + + "IndexStatus = " + catalogIndexStatus + "." + + (fileMetadata == null ? " File not found in storage." : "")); } } fileNames.add(file.getName()); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java index 6063e8f1070..63b6ab090a3 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java @@ -197,7 +197,11 @@ protected File getSmallFile() throws IOException, CatalogException { } protected File create(String resourceName) throws IOException, CatalogException { - return create(studyId, getResourceUri(resourceName)); + return create(resourceName, "data/vcfs/"); + } + + protected File create(String resourceName, String path) throws IOException, CatalogException { + return create(studyId, getResourceUri(resourceName), path); } protected File create(String studyId, URI uri) throws IOException, CatalogException { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java index 74014489902..e5e190328b5 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java @@ -32,7 +32,9 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.testclassification.duration.MediumTests; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -44,6 +46,7 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; /** * Created on 10/07/17. @@ -83,6 +86,42 @@ public void testLoadAndRemoveOneWithOtherLoaded() throws Exception { testLoadAndRemoveOne(); } + @Test + public void testLoadAndRemoveForce() throws Exception { + File file77 = create("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); + File file78 = create("platinum/1K.end.platinum-genomes-vcf-NA12878_S1.genome.vcf.gz"); + File file79 = create("platinum/1K.end.platinum-genomes-vcf-NA12879_S1.genome.vcf.gz"); + indexFile(file77, new QueryOptions(), outputId); + indexFile(file78, new QueryOptions(), outputId); + + removeFile(file77, new QueryOptions()); + + try { + removeFile(file77, new QueryOptions()); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Unable to remove variants from file")); + } + removeFile(file77, new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + + try { + removeFile(file79, new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("File not found in storage.")); + } + Path file77Path = Paths.get(file77.getUri()); + Path otherDir = file77Path.getParent().resolve("other_dir"); + Files.createDirectory(otherDir); + Path otherFile = Files.copy(file77Path, otherDir.resolve(file77Path.getFileName())); + File file77_2 = create(studyFqn, otherFile.toUri(), "other_dir"); + + try { + removeFile(studyFqn, Collections.singletonList(file77_2.getPath()), new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Unable to remove variants from file")); + assertTrue(e.getMessage(), e.getMessage().contains("Instead, found file with same name but different path")); + } + } + @Test public void testLoadAndRemoveMany() throws Exception { @@ -127,8 +166,13 @@ private void removeFile(List files, QueryOptions options) throws Exception Study study = catalogManager.getFileManager().getStudy(ORGANIZATION, files.get(0), sessionId); String studyId = study.getFqn(); + removeFile(studyId, fileIds, options); + } + + private void removeFile(String studyId, List fileIds, QueryOptions options) throws Exception { + Path outdir = Paths.get(opencga.createTmpOutdir(studyId, "_REMOVE_", sessionId)); - variantManager.removeFile(studyId, fileIds, new QueryOptions(), outdir.toUri(), sessionId); + variantManager.removeFile(studyId, fileIds, new QueryOptions(options), outdir.toUri(), sessionId); // assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index 4dd5543e9b8..e6b7f7353c8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.Callable; import java.util.stream.Collectors; @@ -149,7 +150,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException String inputVersion = getVersion(); CellBaseDataResponse species; try { - species = cellBaseClient.getMetaClient().species(); + species = retryMetaSpecies(); } catch (RuntimeException e) { throw new IllegalArgumentException("Unable to access cellbase url '" + getURL() + "', version '" + inputVersion + "'", e); } @@ -158,7 +159,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException // Version might be missing the starting "v" cellBaseConfiguration.setVersion("v" + cellBaseConfiguration.getVersion()); cellBaseClient = newCellBaseClient(cellBaseConfiguration, getSpecies(), getAssembly()); - species = cellBaseClient.getMetaClient().species(); + species = retryMetaSpecies(); } } if (species == null || species.firstResult() == null) { @@ -308,7 +309,7 @@ private static String majorMinor(String version) { public String getVersionFromServer() throws IOException { if (serverVersion == null) { synchronized (this) { - ObjectMap result = retryMetaAbout(3); + ObjectMap result = retryMetaAbout(); if (result == null) { throw new IOException("Unable to get version from server for cellbase " + toString()); } @@ -322,12 +323,43 @@ public String getVersionFromServer() throws IOException { return serverVersion; } - private ObjectMap retryMetaAbout(int retries) throws IOException { - ObjectMap result = cellBaseClient.getMetaClient().about().firstResult(); - if (result == null && retries > 0) { - // Retry - logger.warn("Unable to get version from server for cellbase " + toString() + ". Retrying..."); - result = retryMetaAbout(retries - 1); + private ObjectMap retryMetaAbout() throws IOException { + return retry(3, () -> cellBaseClient.getMetaClient().about().firstResult()); + } + + private CellBaseDataResponse retryMetaSpecies() throws IOException { + return retry(3, () -> cellBaseClient.getMetaClient().species()); + } + + private T retry(int retries, Callable function) throws IOException { + if (retries <= 0) { + return null; + } + T result = null; + Exception e = null; + try { + result = function.call(); + } catch (Exception e1) { + e = e1; + } + if (result == null) { + try { + // Retry + logger.warn("Unable to get reach cellbase " + toString() + ". Retrying..."); + result = retry(retries - 1, function); + } catch (Exception e1) { + if (e == null) { + e = e1; + } else { + e.addSuppressed(e1); + } + if (e instanceof IOException) { + throw (IOException) e; + } else { + throw new IOException("Error reading from cellbase " + toString(), e); + } + } + } return result; } From 831c2cb9585cf13c5eceaff4c77762bf12824dbe Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 22 Aug 2024 12:57:34 +0200 Subject: [PATCH 114/128] catalog: allow updating status from type CLOSED, #TASK-5964 --- .../managers/ClinicalAnalysisManager.java | 14 ++++++- .../managers/InterpretationManager.java | 33 +++++++++++----- .../managers/ClinicalAnalysisManagerTest.java | 35 +++++++++++++++++ .../managers/InterpretationManagerTest.java | 38 +++++++++++++++++++ 4 files changed, 109 insertions(+), 11 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 7ab4c80bb32..20b228aeed8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -1432,10 +1432,22 @@ private OpenCGAResult update(String organizationId, Study stud // Current status is of type CLOSED if (clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { // The only allowed action is to remove the CLOSED status - if (updateParams.getStatus() == null || closedStatus.contains(updateParams.getStatus().getId())) { + if (updateParams.getStatus() == null || StringUtils.isEmpty(updateParams.getStatus().getId())) { throw new CatalogException("Cannot update a ClinicalAnalysis with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + "to perform further updates on the ClinicalAnalysis."); + } else if (closedStatus.contains(updateParams.getStatus().getId())) { + // Users should be able to change from one CLOSED status to a different one but we should still control that no further + // modifications are made + if (parameters.size() > 1) { + throw new CatalogException("Cannot update a ClinicalAnalysis with a " + + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status. You need to remove the " + + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able to perform further updates on " + + "the ClinicalAnalysis."); + } else if (clinicalAnalysis.getStatus().getId().equals(updateParams.getStatus().getId())) { + throw new CatalogException("ClinicalAnalysis already have the status '" + clinicalAnalysis.getStatus().getId() + + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 5ace84fca9b..417978cef13 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -920,6 +920,17 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation InterpretationStudyConfiguration interpretationConfiguration = study.getInternal().getConfiguration().getClinical().getInterpretation(); + ObjectMap parameters = new ObjectMap(); + if (updateParams != null) { + try { + parameters = updateParams.getUpdateMap(); + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); + } + } else { + throw new CatalogException("Missing interpretation update parameters"); + } + Map actionMap = options.getMap(Constants.ACTIONS); // Check if user has permissions to write clinical analysis @@ -963,10 +974,21 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation // Current status is of type CLOSED if (interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { // The only allowed action is to remove the CLOSED status - if (updateParams.getStatus() == null || closedStatus.contains(updateParams.getStatus().getId())) { + if (updateParams.getStatus() == null || StringUtils.isEmpty(updateParams.getStatus().getId())) { throw new CatalogException("Cannot update a Interpretation with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + "to perform further updates on the Interpretation."); + } else if (closedStatus.contains(updateParams.getStatus().getId())) { + // Users should be able to change from one CLOSED status to a different one but we should still control that no further + // modifications are made + if (parameters.size() > 1) { + throw new CatalogException("Cannot update a Interpretation with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the Interpretation."); + } else if (interpretation.getStatus().getId().equals(updateParams.getStatus().getId())) { + throw new CatalogException("Interpretation already have the status '" + interpretation.getStatus().getId() + + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } } } @@ -992,15 +1014,6 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation ParamUtils.checkDateFormat(updateParams.getModificationDate(), InterpretationDBAdaptor.QueryParams.MODIFICATION_DATE.key()); } - ObjectMap parameters = new ObjectMap(); - if (updateParams != null) { - try { - parameters = updateParams.getUpdateMap(); - } catch (JsonProcessingException e) { - throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); - } - } - // if (!parameters.isEmpty() && interpretation.isLocked() // && parameters.getBoolean(InterpretationDBAdaptor.QueryParams.LOCKED.key(), true)) { // throw new CatalogException("Could not update the Interpretation. Interpretation '" + interpretation.getId() diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index 3e11037847a..540602ede36 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -1395,6 +1395,41 @@ public void updateSecondaryFindings() throws CatalogException { } } + @Test + public void updateStatusTest() throws CatalogException { + ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); + + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + + // Create 2 allowed statuses of type CLOSED + ClinicalAnalysisStudyConfiguration studyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + List statusValueList = new ArrayList<>(); + for (ClinicalStatusValue status : studyConfiguration.getStatus()) { + if (!status.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + statusValueList.add(status); + } + } + // Add two statuses of type CLOSED + statusValueList.add(new ClinicalStatusValue("closed1", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + statusValueList.add(new ClinicalStatusValue("closed2", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + studyConfiguration.setStatus(statusValueList); + catalogManager.getClinicalAnalysisManager().configureStudy(studyFqn, studyConfiguration, studyAdminToken1); + + // Update status to one of the new statuses + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("closed1")), QueryOptions.empty(), studyAdminToken1); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), studyAdminToken1).first(); + assertEquals("closed1", ca.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, ca.getStatus().getType()); + + // Update status to the other new CLOSED status + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("closed2")), QueryOptions.empty(), studyAdminToken1); + assertEquals("closed1", ca.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, ca.getStatus().getType()); + } + @Test public void updateInterpretationComments() throws CatalogException { Individual individual = new Individual() diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java index 217aa3f4899..80b630d34ee 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java @@ -18,6 +18,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.StudyAclParams; import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.util.ArrayList; @@ -245,6 +246,43 @@ public void interpretationLockedTest() throws CatalogException { } } + @Test + public void interpretationStatusTest() throws CatalogException { + ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); + + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + + // Create 2 allowed statuses of type CLOSED + ClinicalAnalysisStudyConfiguration studyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + List statusValueList = new ArrayList<>(); + for (ClinicalStatusValue status : studyConfiguration.getInterpretation().getStatus()) { + if (!status.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + statusValueList.add(status); + } + } + // Add two statuses of type CLOSED + statusValueList.add(new ClinicalStatusValue("closed1", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + statusValueList.add(new ClinicalStatusValue("closed2", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + studyConfiguration.getInterpretation().setStatus(statusValueList); + catalogManager.getClinicalAnalysisManager().configureStudy(studyFqn, studyConfiguration, studyAdminToken1); + + // Update status to one of the new statuses + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), + new InterpretationUpdateParams().setStatus(new StatusParam("closed1")), null, QueryOptions.empty(), studyAdminToken1); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), studyAdminToken1).first(); + assertEquals("closed1", interpretation.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, interpretation.getStatus().getType()); + assertTrue(interpretation.isLocked()); + + // Update status to the other new CLOSED status + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), + new InterpretationUpdateParams().setStatus(new StatusParam("closed2")), null, QueryOptions.empty(), studyAdminToken1); + assertEquals("closed1", interpretation.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, interpretation.getStatus().getType()); + assertTrue(interpretation.isLocked()); + } + @Test public void createInterpretationWithSubsetOfPanels() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); From 0c6551baef2fc628368c4507b2b456f7addf03ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 30 Aug 2024 10:37:39 +0100 Subject: [PATCH 115/128] storage: Ensure SearchIndexVariantQueryExecutor is used with skip>500. #TASK-6789 --- .../SearchIndexVariantQueryExecutor.java | 4 +++ .../VariantQueryUsingSearchIndexTest.java | 36 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index bd6b8e6437f..bd7c79cb2e9 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -262,6 +262,10 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) { intersect = true; } else { if (options.getBoolean(QueryOptions.COUNT)) { + // The SearchIndex is better than anyone in terms of counting + intersect = true; + } else if (options.getInt(QueryOptions.SKIP, 0) > 500) { + // Large "skip" queries should use SearchIndex when possible intersect = true; } else { // TODO: Improve this heuristic diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java index 6194d25c6d8..cdc96e15377 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java @@ -2,16 +2,22 @@ import com.google.common.base.Throwables; import org.junit.ClassRule; +import org.junit.Test; import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager.SEARCH_ENGINE_ID; + /** * Created on 22/12/17. * @@ -87,4 +93,32 @@ public DataResult rank(int limit, Query query, String field, boolean asc) { throw Throwables.propagate(e); } } + + @Test + public void testQueryExecutor() { + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 0)).getSource(), + not(containsString(SEARCH_ENGINE_ID))); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, true) + .append(QueryOptions.SKIP, 0)).getSource(), + containsString(SEARCH_ENGINE_ID)); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 100)).getSource(), + not(containsString(SEARCH_ENGINE_ID))); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 1000)).getSource(), + containsString(SEARCH_ENGINE_ID)); + + } } From 11ecea37d3e98dcf9009b0e26de7d5cf5243f399 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 2 Sep 2024 16:29:46 +0200 Subject: [PATCH 116/128] Prepare release 2.12.7 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 14 +++++++------- 21 files changed, 27 insertions(+), 27 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 19e958d2f24..a0a4e7fea1a 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index 6e3fbc1de9b..6e89b341fb6 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 8bc8d823f06..5ff5cbffff9 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 38a2e260e5d..a0caf041474 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index b28dda9d198..dba5dca45ec 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index 264631c0715..b24b1818357 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index f7f42cdb79b..ee80b8747f4 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 14b4301df21..152d31ec4bb 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index c7bbc86990e..29b78cbb020 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index a64f7df1ae6..ea4348064fb 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index b2b6da5bea7..e0e4b0be942 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 00cef59a609..d1e1976bc10 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index d1701f8cc3f..76b997257ad 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index b78e0af52c8..d4a5684a493 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 5caa3dbe832..0555f9819e2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index de9e737a091..61c54ca5f13 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 53166582a8d..78d85dcaff6 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index b4a77d6c006..fe2ee012f54 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 0c79bf18478..885950e2ba1 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index c2d6741df0b..2a40c62db08 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 ../pom.xml diff --git a/pom.xml b/pom.xml index b5564242b70..26230a1d83a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7-SNAPSHOT + 2.12.7 pom OpenCGA @@ -43,12 +43,12 @@ - 2.12.7_dev - 2.12.7_dev - 5.8.5-SNAPSHOT - 2.12.3-SNAPSHOT - 4.12.1-SNAPSHOT - 2.12.7-SNAPSHOT + 2.12.7 + 2.12.7 + 5.8.4 + 2.12.3 + 4.12.0 + 2.12.7 0.2.0 2.11.4 From ce3d6c438fda06d686e1aa68f92c7c11786e4700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Mon, 2 Sep 2024 16:56:33 +0100 Subject: [PATCH 117/128] storage: Remove verbose logging. #TASK-5448 --- .../opencga/storage/hadoop/utils/HBaseManager.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java index 5152ea92c11..1f6cd77efd8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java @@ -420,7 +420,8 @@ public boolean splitAndMove(Admin admin, TableName tableName, byte[] expectedSpl regionInfo = getRegionInfo(admin, tableName, expectedSplit); getRegionInfoAttempts--; if (getRegionInfoAttempts == 0) { - throw new IOException("Split point not found after creation"); + throw new DoNotRetryRegionException( + "Split point " + Bytes.toStringBinary(expectedSplit) + " not found after creation"); } } } else if (count == 1) { @@ -434,8 +435,13 @@ public boolean splitAndMove(Admin admin, TableName tableName, byte[] expectedSpl return true; } catch (IOException | RuntimeException e) { if (ignoreExceptions) { - LOGGER.warn("Error splitting table {} at {}. Retry {}/{}", tableName, - Bytes.toStringBinary(expectedSplit), count, retries, e); + if (e instanceof DoNotRetryRegionException) { + LOGGER.warn("Error splitting table {} at {}. Retry {}/{} : {}", tableName, + Bytes.toStringBinary(expectedSplit), count, retries, e.getMessage()); + } else { + LOGGER.warn("Error splitting table {} at {}. Retry {}/{}", tableName, + Bytes.toStringBinary(expectedSplit), count, retries, e); + } } else { throw e; } From fc2e0d9e66dd2383c87db46da6bbf13456e6e62a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Mon, 2 Sep 2024 17:15:05 +0100 Subject: [PATCH 118/128] storage: Fix compilation issue. #TASK-5448 --- .../OperationsVariantStorageCommandExecutor.java | 16 ++++++++-------- .../variant/archive/ArchiveRowKeyFactory.java | 7 ------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 24b21b612bb..35f0c49c312 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -1174,14 +1174,14 @@ private RestResponse setupVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSetupParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "expectedSamples",commandOptions.expectedSamples, true); - putNestedIfNotNull(beanParams, "expectedFiles",commandOptions.expectedFiles, true); - putNestedIfNotNull(beanParams, "fileType",commandOptions.fileType, true); - putNestedIfNotEmpty(beanParams, "averageFileSize",commandOptions.averageFileSize, true); - putNestedIfNotNull(beanParams, "variantsPerSample",commandOptions.variantsPerSample, true); - putNestedIfNotNull(beanParams, "averageSamplesPerFile",commandOptions.averageSamplesPerFile, true); - putNestedIfNotNull(beanParams, "dataDistribution",commandOptions.dataDistribution, true); - putNestedIfNotNull(beanParams, "normalizeExtensions",commandOptions.normalizeExtensions, true); + putNestedIfNotNull(beanParams, "expectedSamples", commandOptions.expectedSamples, true); + putNestedIfNotNull(beanParams, "expectedFiles", commandOptions.expectedFiles, true); + putNestedIfNotNull(beanParams, "fileType", commandOptions.fileType, true); + putNestedIfNotEmpty(beanParams, "averageFileSize", commandOptions.averageFileSize, true); + putNestedIfNotNull(beanParams, "variantsPerSample", commandOptions.variantsPerSample, true); + putNestedIfNotNull(beanParams, "averageSamplesPerFile", commandOptions.averageSamplesPerFile, true); + putNestedIfNotNull(beanParams, "dataDistribution", commandOptions.dataDistribution, true); + putNestedIfNotNull(beanParams, "normalizeExtensions", commandOptions.normalizeExtensions, true); variantSetupParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java index ce40e405368..d20e5eee924 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java @@ -168,13 +168,6 @@ public String generateBlockIdFromSlice(int fileId, String chrom, long slice) { return generateBlockIdFromSliceAndBatch(getFileBatch(fileId), chrom, slice); } - public String generateBlockIdFromBatch(int fileBatch) { - StringBuilder sb = new StringBuilder(FILE_BATCH_PAD + 1); - sb.append(StringUtils.leftPad(String.valueOf(fileBatch), FILE_BATCH_PAD, '0')); - sb.append(getSeparator()); - return sb.toString(); - } - public String generateBlockIdFromSliceAndBatch(int fileBatch, String chrom, long slice) { String chromosome = Region.normalizeChromosome(chrom); StringBuilder sb = new StringBuilder(FILE_BATCH_PAD + 1 + chromosome.length() + 1 + POSITION_PAD); From 3412c6b87aac52987bad9a24c92f5d8edf489381 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 3 Sep 2024 16:22:23 +0200 Subject: [PATCH 119/128] preparing Port Patch 1.10.7 -> 2.2.1 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-deps-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp2.6/pom.xml | 2 +- .../opencga-storage-hadoop-deps-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index a0a4e7fea1a..7c32e987cdc 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index 6e89b341fb6..d0ab2466da5 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 5ff5cbffff9..4cbfa454138 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index a0caf041474..043c1c5788d 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index dba5dca45ec..02a59d91cf6 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index b24b1818357..c59da1f754d 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index ee80b8747f4..8c8ab880796 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 152d31ec4bb..2b5d462abff 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 29b78cbb020..2f760ec9164 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index ea4348064fb..7b26545a09d 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index e0e4b0be942..0db07bf99ed 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index d1e1976bc10..6d7a6649ad7 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index 76b997257ad..84c787463ec 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index d4a5684a493..f9c584917b1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 0555f9819e2..bc999c1656f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index 61c54ca5f13..b35c5350d64 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 78d85dcaff6..52de24ee0b9 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index fe2ee012f54..2a91d2753e4 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 885950e2ba1..4a47e44df35 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 2a40c62db08..2e7253d9889 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 26230a1d83a..884bb5c1d8a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.12.7 + 3.2.1-SNAPSHOT pom OpenCGA From 85cd1ee817f183a5ff42d7d5b9966b6f2ddef891 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 4 Sep 2024 18:33:22 +0200 Subject: [PATCH 120/128] cicd: Upload reference to develop branch in pull-request-approve to test-xetabase-workflow #TASK-6807 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 0ff62b00d48..4e0bb84734f 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -27,7 +27,7 @@ jobs: test: name: "Run all tests before merging" needs: calculate-xetabase-branch - uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop with: branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} task: ${{ github.event.pull_request.head.ref }} From b664fb1e73c5b2857ce180ac3a05e2d602015fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 6 Sep 2024 09:03:02 +0100 Subject: [PATCH 121/128] storage: Fix junit tests. #TASK-6708 --- .../variant/manager/operations/RemoveVariantsTest.java | 4 ++-- .../storage/core/variant/VariantStorageEngineBNDTest.java | 1 + .../storage/core/variant/VariantStorageEngineSVTest.java | 2 +- .../core/variant/adaptors/VariantDBAdaptorMultiFileTest.java | 2 ++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java index e5e190328b5..56e79581e2f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java @@ -32,6 +32,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.testclassification.duration.MediumTests; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import java.nio.file.Files; @@ -46,7 +47,6 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; -import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; /** * Created on 10/07/17. @@ -140,7 +140,7 @@ public void testLoadAndRemoveDifferentChromosomes() throws Exception { List files = new ArrayList<>(); files.add(create("1k.chr1.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz")); files.add(create("10k.chr22.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz")); - indexFiles(files, new QueryOptions(), outputId); + indexFiles(files, new QueryOptions(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME), outputId); removeFile(files.get(0), new QueryOptions()); } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index 685291398ef..19c23958e47 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -61,6 +61,7 @@ protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.NORMALIZATION_EXTENSIONS.key(), ParamConstants.NONE); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); studyMetadata = new StudyMetadata(1, "s1"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index af97483d71d..404411d09ce 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -62,6 +62,7 @@ public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.getOptions().put(VariantStorageOptions.NORMALIZATION_EXTENSIONS.key(), ParamConstants.NONE); variantStorageEngine.reloadCellbaseConfiguration(); if (!loaded) { @@ -74,7 +75,6 @@ protected void loadFiles() throws Exception { input1 = getResourceUri("variant-test-sv.vcf"); studyMetadata = new StudyMetadata(1, "s1"); - variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); pipelineResult1 = runDefaultETL(input1, variantStorageEngine, studyMetadata, new QueryOptions() .append(VariantStorageOptions.ANNOTATE.key(), true) .append(VariantStorageOptions.STATS_CALCULATE.key(), true) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java index 8071b86a14e..ee382b95557 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java @@ -10,6 +10,7 @@ import org.opencb.biodata.models.variant.avro.SampleEntry; import org.opencb.biodata.models.variant.stats.VariantStats; import org.opencb.commons.datastore.core.*; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; @@ -78,6 +79,7 @@ protected void load() throws Exception { VariantStorageEngine storageEngine = getVariantStorageEngine(); ObjectMap options = getOptions(); options.put(VariantStorageOptions.STATS_CALCULATE.key(), true); + options.put(VariantStorageOptions.NORMALIZATION_EXTENSIONS.key(), ParamConstants.NONE); int maxStudies = 2; int studyId = 1; From b19023673294a97928cd2f48d7e8f5781b7835a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 6 Sep 2024 10:00:49 +0100 Subject: [PATCH 122/128] app: Fix migration test. Missing projectMetadata #TASK-6708 --- .../{ => v3}/v3_2_0/VariantSetupMigration.java | 10 ++++++++-- .../opencb/opencga/app/migrations/MigrationsTest.java | 3 ++- .../opencga/catalog/migration/MigrationManager.java | 4 +++- 3 files changed, 13 insertions(+), 4 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{ => v3}/v3_2_0/VariantSetupMigration.java (87%) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java similarity index 87% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java index 9d02073d099..f7d6c03be9d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_2_0; +package org.opencb.opencga.app.migrations.v3.v3_2_0; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; @@ -9,6 +9,7 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import java.util.LinkedHashSet; +import java.util.List; @Migration(id = "variant_setup", description = "Add a dummy variant setup for studies with data", version = "3.2.0", domain = Migration.MigrationDomain.STORAGE, date = 20240516) @@ -17,7 +18,12 @@ public class VariantSetupMigration extends StorageMigrationTool { @Override protected void run() throws Exception { VariantStorageManager variantStorageManager = getVariantStorageManager(); - for (String study : getVariantStorageStudies()) { + List storageStudies = getVariantStorageStudies(); + if (storageStudies.isEmpty()) { + logger.info("No studies with variant storage found on organization '{}'", organizationId); + return; + } + for (String study : storageStudies) { logger.info("--- Checking study '{}'", study); if (variantStorageManager.hasVariantSetup(study, token)) { logger.info("Study '{}' already has a variant setup", study); diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java index 6d72b9149f5..57ee782a7cc 100644 --- a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java +++ b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java @@ -8,7 +8,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; -import org.opencb.opencga.app.migrations.v3_2_0.VariantSetupMigration; +import org.opencb.opencga.app.migrations.v3.v3_2_0.VariantSetupMigration; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; @@ -32,6 +32,7 @@ public void testVariantSetupMigration() throws Exception { String studyName = "test@1000G:phase1"; VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject("test@1000G").getMetadataManager(); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); StudyMetadata studyMetadata = metadataManager.createStudy(studyName); int fileId = metadataManager.registerFile(studyMetadata.getId(), "folder/file.vcf", Arrays.asList("s1", "s2")); metadataManager.addIndexedFiles(studyMetadata.getId(), Collections.singletonList(fileId)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java index fa8fd7b9ee0..aacf4949226 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java @@ -668,7 +668,8 @@ private MigrationRun run(String organizationId, Class r + "-" + RandomStringUtils.randomAlphanumeric(5); String logFile = startMigrationLogger(jobId, Paths.get(configuration.getJobDir()).resolve(path)); logger.info("------------------------------------------------------"); - logger.info("Executing migration '{}' for version '{}'", annotation.id(), annotation.version()); + logger.info("Executing migration '{}' for version '{}' in organization '{}'", annotation.id(), annotation.version(), + organizationId); logger.info(" {}", annotation.description()); logger.info("------------------------------------------------------"); @@ -707,6 +708,7 @@ private MigrationRun run(String organizationId, Class r logger.info("Migration '{}' finished with status {} : {}", annotation.id(), status, TimeUtils.durationToString(stopWatch)); } logger.info("------------------------------------------------------"); + logger.info(""); } catch (Exception e) { migrationRun.setStatus(MigrationRun.MigrationStatus.ERROR); String message; From f0b62afbcf209b057c00ae95d0daacb39ebb9773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 6 Sep 2024 10:01:23 +0100 Subject: [PATCH 123/128] anaysis: Fix OpenCGATestExternalResource creating analysis/exomiser. #TASK-6708 --- .../ExomiserInterpretationAnalysisTest.java | 3 + .../variant/OpenCGATestExternalResource.java | 19 +++-- .../exomiser/13.1/application.properties | 45 ---------- .../exomiser/13.1/exomiser-analysis.yml | 70 --------------- .../test/resources/exomiser/13.1/output.yml | 13 --- .../exomiser/14.0/application.properties | 47 ---------- .../exomiser/14.0/exomiser-analysis.yml | 85 ------------------- .../test/resources/exomiser/14.0/output.yml | 11 --- 8 files changed, 13 insertions(+), 280 deletions(-) delete mode 100644 opencga-analysis/src/test/resources/exomiser/13.1/application.properties delete mode 100644 opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml delete mode 100644 opencga-analysis/src/test/resources/exomiser/13.1/output.yml delete mode 100644 opencga-analysis/src/test/resources/exomiser/14.0/application.properties delete mode 100644 opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml delete mode 100644 opencga-analysis/src/test/resources/exomiser/14.0/output.yml diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index 46799bfab58..a6fd99aa3bc 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -81,6 +81,7 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE // String resourceVersion = "2109"; String exomiserVersion = "14.0"; String resourceVersion = "2402"; + assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); @@ -131,6 +132,7 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE // String resourceVersion = "2109"; String exomiserVersion = "14.0"; String resourceVersion = "2402"; + assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); @@ -180,6 +182,7 @@ public void trioSingleExomiserAnalysis() throws IOException, CatalogException, T // String resourceVersion = "2109"; String exomiserVersion = "14.0"; String resourceVersion = "2402"; + assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 6fca2e7c2aa..e3b69751771 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -22,7 +22,6 @@ import org.opencb.commons.datastore.mongodb.MongoDataStore; import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.analysis.StorageManager; -import org.opencb.opencga.analysis.clinical.exomiser.ExomiserInterpretationAnalysisTest; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; @@ -220,9 +219,9 @@ public Path isolateOpenCGA() throws IOException { catalogManagerExternalResource.getConfiguration().serialize( new FileOutputStream(conf.resolve("configuration.yml").toFile())); - InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); - - storageConfiguration = StorageConfiguration.load(inputStream, "yml"); + try (InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml")) { + storageConfiguration = StorageConfiguration.load(inputStream, "yml"); + } storageConfiguration.getVariant().setDefaultEngine(storageEngine); if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { @@ -259,20 +258,22 @@ public Path isolateOpenCGA() throws IOException { // Mutational signatue analysis Path analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/mutational-signature")).toAbsolutePath(); - inputStream = new FileInputStream("../opencga-app/app/analysis/mutational-signature/sv_clustering.R"); - Files.copy(inputStream, analysisPath.resolve("sv_clustering.R"), StandardCopyOption.REPLACE_EXISTING); + try (FileInputStream inputStream = new FileInputStream("../opencga-app/app/analysis/mutational-signature/sv_clustering.R")) { + Files.copy(inputStream, analysisPath.resolve("sv_clustering.R"), StandardCopyOption.REPLACE_EXISTING); + } // Pedigree graph analysis analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/pedigree-graph")).toAbsolutePath(); - inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R"); - Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); + try (FileInputStream inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R")) { + Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); + } // Exomiser analysis files List exomiserVersions = Arrays.asList("13.1", "14.0"); List exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml"); for (String exomiserVersion : exomiserVersions) { analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).toAbsolutePath()); - Path exomiserPath = Paths.get(ExomiserInterpretationAnalysisTest.class.getClassLoader().getResource("exomiser").getPath()); + Path exomiserPath = Paths.get("../opencga-app/app/analysis/exomiser"); for (String exomiserFile : exomiserFiles) { String resource = exomiserVersion + "/" + exomiserFile; Files.copy(exomiserPath.resolve(resource).toAbsolutePath(), analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); diff --git a/opencga-analysis/src/test/resources/exomiser/13.1/application.properties b/opencga-analysis/src/test/resources/exomiser/13.1/application.properties deleted file mode 100644 index 1775e32d14c..00000000000 --- a/opencga-analysis/src/test/resources/exomiser/13.1/application.properties +++ /dev/null @@ -1,45 +0,0 @@ -# -# The Exomiser - A tool to annotate and prioritize genomic variants -# -# Copyright (c) 2016-2021 Queen Mary University of London. -# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# - -## exomiser root data directory ## -# root path where data is to be downloaded and worked on it is assumed that all the files required by exomiser listed -# in this properties file will be found in the data directory, unless specifically overridden here. -#exomiser.data-directory=/exomiser-data -exomiser.data-directory=/data - -#remm.version=0.3.1.post1 -#cadd.version=1.4 -#exomiser.hg38.data-version=1811 -#exomiser.hg38.data-version=2109 -#exomiser.hg38.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg38.tsv.gz -#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz -#exomiser.phenotype.data-version=2109 -#logging.file.name=/exomiser-data/logs/exomiser.log - - -#remm.version=0.3.1.post1 -#cadd.version=1.4 -#exomiser.hg19.data-version=1811 -exomiser.hg38.data-version=2109 -#exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz -#exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=2109_hg38_clinvar_whitelist.tsv.gz -exomiser.phenotype.data-version=2109 -logging.file.name=/jobdir/exomiser.log diff --git a/opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml b/opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml deleted file mode 100644 index 5ec7d93d508..00000000000 --- a/opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml +++ /dev/null @@ -1,70 +0,0 @@ ---- -analysisMode: PASS_ONLY -inheritanceModes: { - AUTOSOMAL_DOMINANT: 0.1, - AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, - AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, - X_DOMINANT: 0.1, - X_RECESSIVE_HOM_ALT: 0.1, - X_RECESSIVE_COMP_HET: 2.0, - MITOCHONDRIAL: 0.2 -} -frequencySources: [ - THOUSAND_GENOMES, - TOPMED, - UK10K, - - ESP_AFRICAN_AMERICAN, ESP_EUROPEAN_AMERICAN, ESP_ALL, - - EXAC_AFRICAN_INC_AFRICAN_AMERICAN, EXAC_AMERICAN, - EXAC_SOUTH_ASIAN, EXAC_EAST_ASIAN, - EXAC_FINNISH, EXAC_NON_FINNISH_EUROPEAN, - EXAC_OTHER, - - GNOMAD_E_AFR, - GNOMAD_E_AMR, - # GNOMAD_E_ASJ, - GNOMAD_E_EAS, - GNOMAD_E_FIN, - GNOMAD_E_NFE, - GNOMAD_E_OTH, - GNOMAD_E_SAS, - - GNOMAD_G_AFR, - GNOMAD_G_AMR, - # GNOMAD_G_ASJ, - GNOMAD_G_EAS, - GNOMAD_G_FIN, - GNOMAD_G_NFE, - GNOMAD_G_OTH, - GNOMAD_G_SAS -] -# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM -# REMM is trained on non-coding regulatory regions -# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files -# and updated their location in the application.properties. Exomiser will not run without this. -pathogenicitySources: [ REVEL, MVP ] -#this is the standard exomiser order. -steps: [ - failedVariantFilter: { }, - variantEffectFilter: { - remove: [ - FIVE_PRIME_UTR_EXON_VARIANT, - FIVE_PRIME_UTR_INTRON_VARIANT, - THREE_PRIME_UTR_EXON_VARIANT, - THREE_PRIME_UTR_INTRON_VARIANT, - NON_CODING_TRANSCRIPT_EXON_VARIANT, - NON_CODING_TRANSCRIPT_INTRON_VARIANT, - CODING_TRANSCRIPT_INTRON_VARIANT, - UPSTREAM_GENE_VARIANT, - DOWNSTREAM_GENE_VARIANT, - INTERGENIC_VARIANT, - REGULATORY_REGION_VARIANT - ] - }, - frequencyFilter: { maxFrequency: 2.0 }, - pathogenicityFilter: { keepNonPathogenic: true }, - inheritanceFilter: { }, - omimPrioritiser: { }, - hiPhivePrioritiser: { } -] diff --git a/opencga-analysis/src/test/resources/exomiser/13.1/output.yml b/opencga-analysis/src/test/resources/exomiser/13.1/output.yml deleted file mode 100644 index 3203c545356..00000000000 --- a/opencga-analysis/src/test/resources/exomiser/13.1/output.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -outputContributingVariantsOnly: true -#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results -numGenes: 20 -minExomiserGeneScore: 0.0 -#outputPrefix options: specify the path/filename without an extension and this will be added -# according to the outputFormats option. If unspecified this will default to the following: -# {exomiserDir}/results/input-vcf-name-exomiser-results.html -# alternatively, specify a fully qualifed path only. e.g. /users/jules/exomes/analysis -outputPrefix: /jobdir/exomiser_output -#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) -#outputFormats: [HTML, JSON, TSV_GENE, TSV_VARIANT, VCF] -outputFormats: [TSV_VARIANT, JSON, HTML] diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/application.properties b/opencga-analysis/src/test/resources/exomiser/14.0/application.properties deleted file mode 100644 index d1658370370..00000000000 --- a/opencga-analysis/src/test/resources/exomiser/14.0/application.properties +++ /dev/null @@ -1,47 +0,0 @@ -# -# The Exomiser - A tool to annotate and prioritize genomic variants -# -# Copyright (c) 2016-2021 Queen Mary University of London. -# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# - -## exomiser root data directory ## -# root path where data is to be downloaded and worked on it is assumed that all the files required by exomiser listed -# in this properties file will be found in the data directory, unless specifically overridden here. -#exomiser.data-directory=/exomiser-data -exomiser.data-directory=/data - -#remm.version=0.3.1.post1 -#cadd.version=1.4 -#exomiser.hg38.data-version=1811 -#exomiser.hg38.data-version=2109 -#exomiser.hg38.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg38.tsv.gz -#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz -#exomiser.phenotype.data-version=2109 -#logging.file.name=/exomiser-data/logs/exomiser.log - - -#remm.version=0.3.1.post1 -#cadd.version=1.4 -#exomiser.hg19.data-version=1811 -exomiser.hg38.data-version=2402 -#exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz -#exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz -exomiser.phenotype.data-version=2402 -exomiser.hg38.clin-var-data-version=2402 -exomiser.hg38.use-clinvar-white-list=true -logging.file.name=/jobdir/exomiser.log diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml b/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml deleted file mode 100644 index e3d064ba07b..00000000000 --- a/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- -analysisMode: PASS_ONLY -inheritanceModes: { - AUTOSOMAL_DOMINANT: 0.1, - AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, - AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, - X_DOMINANT: 0.1, - X_RECESSIVE_COMP_HET: 2.0, - X_RECESSIVE_HOM_ALT: 0.1, - MITOCHONDRIAL: 0.2 -} -frequencySources: [ - UK10K, - - GNOMAD_E_AFR, - GNOMAD_E_AMR, - # GNOMAD_E_ASJ, - GNOMAD_E_EAS, - # GNOMAD_E_FIN, - GNOMAD_E_NFE, - # GNOMAD_E_OTH, - GNOMAD_E_SAS, - - GNOMAD_G_AFR, - GNOMAD_G_AMR, - # GNOMAD_G_ASJ, - GNOMAD_G_EAS, - # GNOMAD_G_FIN, - GNOMAD_G_NFE, - # GNOMAD_G_OTH, - GNOMAD_G_SAS -] -# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM -# REMM is trained on non-coding regulatory regions -# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files -# and updated their location in the application.properties. Exomiser will not run without this. -pathogenicitySources: [ REVEL, MVP ] -#this is the standard exomiser order. -#all steps are optional -steps: [ - #hiPhivePrioritiser: {}, - #priorityScoreFilter: {priorityType: HIPHIVE_PRIORITY, minPriorityScore: 0.500}, - #intervalFilter: {interval: 'chr10:123256200-123256300'}, - # or for multiple intervals: - #intervalFilter: {intervals: ['chr10:123256200-123256300', 'chr10:123256290-123256350']}, - # or using a BED file - NOTE this should be 0-based, Exomiser otherwise uses 1-based coordinates in line with VCF - #intervalFilter: {bed: /full/path/to/bed_file.bed}, - #genePanelFilter: {geneSymbols: ['FGFR1','FGFR2']}, - #geneBlacklistFilter: { }, - failedVariantFilter: { }, - #qualityFilter: {minQuality: 50.0}, - variantEffectFilter: { - remove: [ - FIVE_PRIME_UTR_EXON_VARIANT, - FIVE_PRIME_UTR_INTRON_VARIANT, - THREE_PRIME_UTR_EXON_VARIANT, - THREE_PRIME_UTR_INTRON_VARIANT, - NON_CODING_TRANSCRIPT_EXON_VARIANT, - UPSTREAM_GENE_VARIANT, - INTERGENIC_VARIANT, - REGULATORY_REGION_VARIANT, - CODING_TRANSCRIPT_INTRON_VARIANT, - NON_CODING_TRANSCRIPT_INTRON_VARIANT, - DOWNSTREAM_GENE_VARIANT - ] - }, - # removes variants represented in the database - #knownVariantFilter: {}, - frequencyFilter: {maxFrequency: 2.0}, - pathogenicityFilter: {keepNonPathogenic: true}, - # inheritanceFilter and omimPrioritiser should always run AFTER all other filters have completed - # they will analyse genes according to the specified modeOfInheritance above- UNDEFINED will not be analysed. - inheritanceFilter: {}, - # omimPrioritiser isn't mandatory. - omimPrioritiser: {}, - #priorityScoreFilter: {minPriorityScore: 0.4}, - # Other prioritisers: Only combine omimPrioritiser with one of these. - # Don't include any if you only want to filter the variants. - hiPhivePrioritiser: {}, - # or run hiPhive in benchmarking mode: - #hiPhivePrioritiser: {runParams: 'mouse'}, - #phivePrioritiser: {} - #phenixPrioritiser: {} - #exomeWalkerPrioritiser: {seedGeneIds: [11111, 22222, 33333]} -] diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/output.yml b/opencga-analysis/src/test/resources/exomiser/14.0/output.yml deleted file mode 100644 index 695255a573b..00000000000 --- a/opencga-analysis/src/test/resources/exomiser/14.0/output.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -outputContributingVariantsOnly: true -#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results -numGenes: 20 -minExomiserGeneScore: 0.0 -# Path to the desired output directory. Will default to the 'results' subdirectory of the exomiser install directory -outputDirectory: /jobdir/ -# Filename for the output files. Will default to {input-vcf-filename}-exomiser -outputFileName: exomiser_output -#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) -outputFormats: [TSV_VARIANT, JSON, HTML] From 117ca3805af96c8fbb74c3bf1b5ccf845cb26481 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 9 Sep 2024 16:39:24 +0200 Subject: [PATCH 124/128] cicd: Fix xetabase branch calculation #TASK-6807 --- .github/workflows/pull-request-approved.yml | 7 ++++++- .github/workflows/scripts/get-xetabase-branch.sh | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 4e0bb84734f..db5c1aaa5a4 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -20,9 +20,14 @@ jobs: name: "Get current branch for Xetabase from target branch" run: | chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" + echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" + echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT + env: + ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} test: name: "Run all tests before merging" diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index e971f990c2e..a1eb7e529c6 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -7,9 +7,9 @@ get_xetabase_branch() { # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then - if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then - echo "$GIT_BRANCH"; - exit 0; + if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then + echo $input_branch; + return 0; fi fi From e85b34bbf559427f7e8ac86abdc4b8f97e25ee0d Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 10 Sep 2024 09:11:05 +0200 Subject: [PATCH 125/128] cicd: Fix xetabase branch calculation #TASK-6807 --- .github/workflows/pull-request-approved.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index db5c1aaa5a4..12be5a929aa 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -16,6 +16,8 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: '10' + ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found" + persist-credentials: false - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | From 31448258b6b55f5f9766f6cc0cde3a06c963affe Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 10 Sep 2024 09:26:46 +0200 Subject: [PATCH 126/128] cicd: Fix xetabase branch calculation #TASK-6807 --- .github/workflows/pull-request-approved.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 12be5a929aa..a39ff011460 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -24,7 +24,6 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" - echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From d6a88581a915f080f24410b7753f15de25e46449 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 13 Sep 2024 14:06:28 +0200 Subject: [PATCH 127/128] Prepare release 3.2.1 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- opencga-storage/opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-compat-api/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.0/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.2/pom.xml | 2 +- .../opencga-storage-hadoop-compat-hbase2.4/pom.xml | 2 +- .../opencga-storage-hadoop-compat/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.13/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdi5.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 14 +++++++------- 27 files changed, 33 insertions(+), 33 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 1d127746157..44dfcada226 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index b738ad32c0e..7c86232e020 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index c55f55fadd5..32968f648dc 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 043c1c5788d..7af1fa42fb8 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 02a59d91cf6..3aad7a17509 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index c59da1f754d..93b53c60ad9 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 73192c29e58..dfd75099282 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index eabe8a65d7c..9a48c3cf8e0 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index df69dcc79dd..ff1b502532f 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index 12400612512..d2ef8e6ef94 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 0a392fe9020..16196aae558 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml index ca935dae3fa..bc904deccc8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml index 241ecd7a87b..bf109e1371e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml index 3fdd2a30431..f6cb60ee7af 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml index 112516d4f72..ebc420d03a1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml index 1b1393dfceb..2a4381a8c66 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 876fb2c1054..91cc355092a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml index 44bbbe4acbf..abd28d36f1a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml index 674db7f319c..d5e8de35cc4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml index 312d6ce845d..616d8c0d0da 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml index 3529773711c..beac4cdb3ce 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml index 9f791862b43..477bc94cc09 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index e72f24d30d9..69c94b8b2fd 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 2a91d2753e4..f69ed9ee435 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 83f9de6ce82..5e77d32e404 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 88dbdeb7783..dccd304226c 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 95f54ac4a10..cba85a598b8 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1-SNAPSHOT + 3.2.1 pom OpenCGA @@ -43,12 +43,12 @@ - 3.2.1_dev - 3.2.1_dev - 6.2.1-SNAPSHOT - 3.2.1-SNAPSHOT - 5.2.1-SNAPSHOT - 3.2.1-SNAPSHOT + 3.2.1 + 3.2.1 + 6.2.1 + 3.2.1 + 5.2.1 + 3.2.1 0.2.0 From 6d1979797752d4063c5b6f5cbdc08ebb468e3ff5 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 3 Oct 2024 13:52:37 +0200 Subject: [PATCH 128/128] Port Patch 3.2.1 -> 4.0.0 XB 2.2.1 -> 3.0.0 #TASK-6780 --- opencga-analysis/pom.xml | 2 +- opencga-app/pom.xml | 2 +- opencga-catalog/pom.xml | 2 +- opencga-client/pom.xml | 2 +- opencga-clinical/pom.xml | 2 +- opencga-core/pom.xml | 2 +- opencga-master/pom.xml | 2 +- opencga-server/pom.xml | 2 +- opencga-storage/opencga-storage-app/pom.xml | 2 +- .../opencga-storage-benchmark/pom.xml | 2 +- opencga-storage/opencga-storage-core/pom.xml | 2 +- .../opencga-storage-hadoop-compat-api/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../opencga-storage-hadoop-compat/pom.xml | 2 +- .../opencga-storage-hadoop-core/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-emr6.13/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdi5.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib-hdp3.1/pom.xml | 2 +- .../opencga-storage-hadoop-lib/pom.xml | 2 +- opencga-storage/opencga-storage-hadoop/pom.xml | 2 +- opencga-storage/opencga-storage-server/pom.xml | 2 +- opencga-storage/pom.xml | 2 +- opencga-test/pom.xml | 2 +- pom.xml | 17 ++++++++--------- 27 files changed, 34 insertions(+), 35 deletions(-) diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 44dfcada226..ad1a6f765e8 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index 7c86232e020..76ea292d610 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 32968f648dc..df6a3d06d78 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 7af1fa42fb8..ee6e9dbf305 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 3aad7a17509..226fd9bac1d 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index 93b53c60ad9..c1fb9ac5068 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index dfd75099282..ca1c6c3c219 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 9a48c3cf8e0..0a226d06dfb 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index ff1b502532f..b4771cd977b 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index d2ef8e6ef94..c6d485f3091 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 16196aae558..0142ea1ddfa 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml index bc904deccc8..3b408a1127e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml index bf109e1371e..ce30f130cf2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml index f6cb60ee7af..145210cc55b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml index ebc420d03a1..e78bfbe0823 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml index 2a4381a8c66..05bdba53885 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 91cc355092a..59a99ca7132 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml index abd28d36f1a..71c3d744640 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml index d5e8de35cc4..1752fa6a3d3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml index 616d8c0d0da..6556d12cbc1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml index beac4cdb3ce..2d8e814b74d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml index 477bc94cc09..4753a66d853 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 69c94b8b2fd..acb4cd4b097 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index f69ed9ee435..8d40be5a821 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 5e77d32e404..7e3cf23f5aa 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index dccd304226c..0dcd3f4db49 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index cba85a598b8..f82fccd957a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.1 + 4.0.0-SNAPSHOT pom OpenCGA @@ -43,14 +43,13 @@ - 3.2.1 - 3.2.1 - 6.2.1 - 3.2.1 - 5.2.1 - 3.2.1 - - 0.2.0 + 4.0.0_dev + 4.0.0_dev + 7.0.0-SNAPSHOT + 4.0.0-SNAPSHOT + 6.0.0-SNAPSHOT + 4.0.0-SNAPSHOT + 0.2.0 2.14.3 2.30.1