diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java index bc6e1dc674b..c494f8db7ca 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java @@ -16,9 +16,13 @@ package org.opencb.opencga.analysis; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.commons.utils.FileUtils; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +32,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; public class ConfigurationUtils { private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class); + private ConfigurationUtils() { + throw new IllegalStateException("Utility class"); + } /** * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml. * @@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome) .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml")); } } + + public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException { + List tools = new ArrayList<>(); + for (AnalysisTool tool : configuration.getAnalysis().getTools()) { + if (tool.getId().equals(toolId)) { + tools.add(tool); + } + } + if (CollectionUtils.isEmpty(tools)) { + throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file"); + } + if (tools.size() == 1) { + return tools.get(0).getVersion(); + } + String defaultVersion = null; + for (AnalysisTool tool : tools) { + if (tool.isDefaultVersion()) { + if (!StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("More than one default version found for tool ID '" + toolId + "'"); + } else { + defaultVersion = tool.getVersion(); + } + } + } + if (StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true"); + } + return defaultVersion; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 06e1c7f4838..bbf685d1a03 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -30,8 +30,10 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.clinical.InterpretationAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -44,10 +46,10 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import java.io.BufferedReader; import java.io.File; @@ -69,6 +71,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis { private String clinicalAnalysisId; private String sampleId; private ClinicalAnalysis.Type clinicalAnalysisType; + private String exomiserVersion; private ClinicalAnalysis clinicalAnalysis; @@ -97,8 +100,7 @@ protected void check() throws Exception { try { clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(), token); - } catch ( - CatalogException e) { + } catch (CatalogException e) { throw new ToolException(e); } if (clinicalAnalysisQueryResult.getNumResults() != 1) { @@ -117,6 +119,7 @@ protected void check() throws Exception { } sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId(); + // Check clinical analysis type if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) { clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY; } else { @@ -125,6 +128,13 @@ protected void check() throws Exception { logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(), clinicalAnalysisType); + // Check exomiser version + if (StringUtils.isEmpty(exomiserVersion)) { + // Missing exomiser version use the default one + exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); + } + // Update executor params with OpenCGA home and session ID setUpStorageEngineExecutor(studyId); } @@ -134,25 +144,28 @@ protected void run() throws ToolException { step(() -> { executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID); - getToolExecutor(ExomiserWrapperAnalysisExecutor.class) + ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(studyId) .setSampleId(sampleId) .setClinicalAnalysisType(clinicalAnalysisType) - .execute(); + .setExomiserVersion(exomiserVersion); + + exomiserExecutor.execute(); - saveInterpretation(studyId, clinicalAnalysis); + saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion()); }); } - protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException, + protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion) + throws ToolException, StorageEngineException, CatalogException, IOException { // Interpretation method InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(), GitRepositoryState.getInstance().getCommitId(), Collections.singletonList( new Software() .setName("Exomiser") - .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME) - .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION))); + .setRepository("Docker: " + dockerImage) + .setVersion(dockerImageVersion))); // Analyst ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token); @@ -274,8 +287,17 @@ private List getPrimaryFindings() throws IOException, StorageEn // Convert variants to clinical variants for (Variant variant : variantResults.getResults()) { ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant); - List exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv - .get(variant.toStringSimple()))); + List exomiserTranscripts = new ArrayList<>(); + if (normalizedToTsv.containsKey(variant.toStringSimple())) { + if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) { + exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple()))); + } else { + logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(), + normalizedToTsv.get(variant.toStringSimple())); + } + } else { + logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple()); + } for (String[] fields : variantTsvMap.get(variant.toStringSimple())) { ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]); Map attributes = getAttributesFromTsv(fields); @@ -463,4 +485,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly this.clinicalAnalysisId = clinicalAnalysisId; return this; } + + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java index 5a88635e45c..6024761d596 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java @@ -552,7 +552,7 @@ protected final T getToolExecutor(Class clazz toolExecutor.getSource(), toolExecutor.getFramework())); - toolExecutor.setUp(erm, executorParams, outDir); + toolExecutor.setUp(erm, executorParams, outDir, configuration); return toolExecutor; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java index a5d2484b2d7..270e9e0b213 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java @@ -295,7 +295,7 @@ protected void run() throws ToolException { step("gwas", () -> { GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class); - gwasExecutor.setConfiguration(gwasConfiguration) + gwasExecutor.setGwasConfiguration(gwasConfiguration) .setStudy(study) .setSampleList1(caseCohortSamples) .setSampleList2(controlCohortSamples) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 6fb44855fb7..bd71414a4e8 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -71,8 +71,9 @@ import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -88,6 +89,7 @@ import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat; import org.opencb.opencga.storage.core.variant.query.ParsedQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor; @@ -490,6 +492,18 @@ public void aggregate(String studyStr, VariantAggregateParams params, String tok }); } + public VariantSetupResult variantSetup(String studyStr, VariantSetupParams params, String token) + throws CatalogException, StorageEngineException { + return secureOperation(VariantSetupOperationManager.ID, studyStr, params.toObjectMap(), token, + engine -> new VariantSetupOperationManager(this, engine).setup(getStudyFqn(studyStr, token), params, token)); + } + + public boolean hasVariantSetup(String studyStr, String token) throws CatalogException { + Study study = catalogManager.getStudyManager().get(studyStr, + new QueryOptions(INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token).first(); + return VariantSetupOperationManager.hasVariantSetup(study); + } + public ObjectMap configureProject(String projectStr, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperationByProject("configure", projectStr, params, token, engine -> { DataStore dataStore = getDataStoreByProjectId(projectStr, token); @@ -1181,7 +1195,7 @@ private interface VariantOperationFunction { private R secureOperationByProject(String operationName, String project, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineByProject(project, params, token)) { - return secureTool(operationName, true, params, token, variantStorageEngine, operation); + return secureTool(operationName, true, null, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1190,7 +1204,7 @@ private R secureOperationByProject(String operationName, String project, Obj private R secureOperation(String operationName, String study, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) { - return secureTool(operationName, true, params, token, variantStorageEngine, operation); + return secureTool(operationName, true, study, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1199,7 +1213,7 @@ private R secureOperation(String operationName, String study, ObjectMap para private R secureAnalysis(String operationName, String study, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) { - return secureTool(operationName, false, params, token, variantStorageEngine, operation); + return secureTool(operationName, false, study, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1221,7 +1235,7 @@ private R secureOperationByProject(String operationName, String projectStr, return secureOperationByProject(operationName, projectStr, params, token, operation); } - private R secureTool(String toolId, boolean isOperation, ObjectMap params, String token, + private R secureTool(String toolId, boolean isOperation, String study, ObjectMap params, String token, VariantStorageEngine variantStorageEngine, VariantOperationFunction operation) throws CatalogException, StorageEngineException { @@ -1241,6 +1255,15 @@ private R secureTool(String toolId, boolean isOperation, ObjectMap params, S throw new StorageEngineException("Unable to execute operation '" + toolId + "'. " + "The storage engine is in mode=" + storageConfiguration.getMode()); } + if (isOperation && study != null && !VariantSetupOperationManager.ID.equals(toolId)) { + // Ensure that the variant setup has been executed + // do not check for the setup operation itself + // Project level operations can not be checked for setup. + if (!hasVariantSetup(study, token)) { + throw new StorageEngineException("Unable to execute operation '" + toolId + "'. " + + "The variant storage has not been setup for study '" + study + "'"); + } + } result = operation.apply(variantStorageEngine); return result; } catch (CatalogException | StorageEngineException e) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java index 882240d2865..c0f57f0b04f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java @@ -27,6 +27,8 @@ import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.URI; import java.util.ArrayList; @@ -34,6 +36,8 @@ public class VariantDeleteOperationManager extends OperationManager { + private final Logger logger = LoggerFactory.getLogger(VariantDeleteOperationManager.class); + public VariantDeleteOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) { super(variantStorageManager, engine); } @@ -63,7 +67,14 @@ public void removeFile(String study, List inputFiles, URI outdir, String String catalogIndexStatus = file.getInternal().getVariant().getIndex().getStatus().getId(); if (!catalogIndexStatus.equals(VariantIndexStatus.READY)) { // Might be partially loaded in VariantStorage. Check FileMetadata - FileMetadata fileMetadata = variantStorageEngine.getMetadataManager().getFileMetadata(studyMetadata.getId(), fileStr); + FileMetadata fileMetadata = variantStorageEngine.getMetadataManager() + .getFileMetadata(studyMetadata.getId(), file.getName()); + if (fileMetadata != null && !fileMetadata.getPath().equals(file.getUri().getPath())) { + // FileMetadata path does not match the catalog path. This file is not registered in the storage. + throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. " + + "File is not registered in the storage. " + + "Instead, found file with same name but different path '" + fileMetadata.getPath() + "'"); + } boolean canBeRemoved; if (force) { // When forcing remove, just require the file to be registered in the storage @@ -73,17 +84,18 @@ public void removeFile(String study, List inputFiles, URI outdir, String canBeRemoved = fileMetadata != null && fileMetadata.getIndexStatus() != TaskMetadata.Status.NONE; } if (!canBeRemoved) { - throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " - + "IndexStatus = " + catalogIndexStatus); + throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. " + + "IndexStatus = " + catalogIndexStatus + "." + + (fileMetadata == null ? " File not found in storage." : "")); } } fileNames.add(file.getName()); // filePaths.add(file.getPath()); } - if (fileNames.isEmpty()) { - throw new CatalogException("Nothing to do!"); - } + } + if (fileNames.isEmpty()) { + throw new CatalogException("Nothing to do!"); } variantStorageEngine.removeFiles(study, fileNames, outdir); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java new file mode 100644 index 00000000000..0003ca105bf --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -0,0 +1,166 @@ +package org.opencb.opencga.analysis.variant.manager.operations; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VariantSetupOperationManager extends OperationManager { + + + public static final String ID = "variant-setup"; + private static Logger logger = LoggerFactory.getLogger(VariantSetupOperationManager.class); + + public VariantSetupOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine variantStorageEngine) { + super(variantStorageManager, variantStorageEngine); + } + + public VariantSetupResult setup(String studyFqn, VariantSetupParams params, String token) + throws CatalogException, StorageEngineException { + // Copy params to avoid modifying input object + params = new VariantSetupParams(params); + check(studyFqn, params, token); + + VariantSetupResult result = new VariantSetupResult(); + result.setDate(TimeUtils.getTime()); + result.setUserId(catalogManager.getUserManager().getUserIdContextStudy(studyFqn, token)); + result.setParams(params.toObjectMap()); + result.setStatus(VariantSetupResult.Status.READY); + + inferParams(params); + + ObjectMap options = variantStorageEngine.inferConfigurationParams(params); + result.setOptions(options); + + catalogManager.getStudyManager().setVariantEngineSetupOptions(studyFqn, result, token); + + return result; + } + + /** + * Infer some parameters from others. + * - averageFileSize inferred from fileType + * - samplesPerFile inferred from dataDistribution or expectedSamplesNumber and expectedFilesNumber + * - numberOfVariantsPerSample inferred from fileType + * @param params params to infer + */ + private void inferParams(VariantSetupParams params) { + if (params.getFileType() != null) { + switch (params.getFileType()) { + case GENOME_gVCF: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize("1GiB"); + } + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); + } + break; + case GENOME_VCF: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize("500MiB"); + } + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); + } + break; + case EXOME: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize("100MiB"); + } + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(100000); + } + break; + default: + throw new IllegalArgumentException("Unknown fileType " + params.getFileType()); + } + } + // Unable to tell. Use a default value for numberOfVariantsPerSample + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); + } + + if (params.getAverageSamplesPerFile() == null) { + if (params.getDataDistribution() == null) { + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); + } else { + switch (params.getDataDistribution()) { + case SINGLE_SAMPLE_PER_FILE: + params.setAverageSamplesPerFile(1f); + break; + case MULTIPLE_SAMPLES_PER_FILE: + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); + break; + case MULTIPLE_FILES_PER_SAMPLE: + // Hard to tell. Let's assume 2 samples per file + params.setAverageSamplesPerFile(2f); + break; + case FILES_SPLIT_BY_CHROMOSOME: + case FILES_SPLIT_BY_REGION: + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue()); + break; + default: + throw new IllegalArgumentException("Unknown dataDistribution " + params.getDataDistribution()); + } + } + } + } + + private void check(String studyStr, VariantSetupParams params, String token) throws CatalogException, StorageEngineException { + Study study = catalogManager.getStudyManager().get(studyStr, + new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token) + .first(); + + VariantStorageMetadataManager metadataManager = variantStorageEngine.getMetadataManager(); + if (metadataManager.studyExists(studyStr)) { + int studyId = metadataManager.getStudyId(studyStr); + if (!metadataManager.getIndexedFiles(studyId).isEmpty()) { + throw new IllegalArgumentException("Unable to execute variant-setup on study '" + studyStr + "'. " + + "It already has indexed files."); + } + } + if (hasVariantSetup(study)) { + logger.info("Study {} was already setup. Re executing variant-setup", studyStr); + } + + if (params.getExpectedFiles() == null || params.getExpectedFiles() <= 0) { + throw new IllegalArgumentException("Missing expectedFiles"); + } + if (params.getExpectedSamples() == null || params.getExpectedSamples() <= 0) { + throw new IllegalArgumentException("Missing expectedSamples"); + } + + if (params.getAverageFileSize() == null && params.getFileType() == null) { + throw new IllegalArgumentException("Missing averageFileSize or fileType"); + } + } + + public static boolean hasVariantSetup(Study study) { + boolean hasSetup = false; + VariantSetupResult setup = getVariantSetupResult(study); + if (setup != null && setup.getStatus() == VariantSetupResult.Status.READY) { + hasSetup = true; + } + return hasSetup; + } + + private static VariantSetupResult getVariantSetupResult(Study study) { + if (study.getInternal() != null + && study.getInternal().getConfiguration() != null + && study.getInternal().getConfiguration().getVariantEngine() != null) { + return study.getInternal().getConfiguration().getVariantEngine().getSetup(); + } + return null; + } + +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java index d4e7270d2e6..766962db671 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java @@ -77,7 +77,9 @@ protected void check() throws Exception { params.putIfNotEmpty(VariantStorageOptions.INCLUDE_GENOTYPE.key(), indexParams.getIncludeGenotypes()); params.put(VariantStorageOptions.STATS_AGGREGATION.key(), indexParams.getAggregated()); params.putIfNotEmpty(VariantStorageOptions.STATS_AGGREGATION_MAPPING_FILE.key(), indexParams.getAggregationMappingFile()); - params.put(VariantStorageOptions.GVCF.key(), indexParams.isGvcf()); + if (indexParams.isGvcf()) { + params.put(VariantStorageOptions.GVCF.key(), indexParams.isGvcf()); + } // queryOptions.putIfNotNull(VariantFileIndexerStorageOperation.TRANSFORMED_FILES, indexParams.transformedPaths); @@ -92,7 +94,9 @@ protected void check() throws Exception { params.put(VariantStorageOptions.FAMILY.key(), indexParams.isFamily()); params.put(VariantStorageOptions.SOMATIC.key(), indexParams.isSomatic()); params.putIfNotEmpty(VariantStorageOptions.LOAD_SPLIT_DATA.key(), indexParams.getLoadSplitData()); - params.put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), indexParams.isLoadMultiFileData()); + if (indexParams.isLoadMultiFileData()) { + params.put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), indexParams.isLoadMultiFileData()); + } params.putIfNotEmpty(VariantStorageOptions.LOAD_SAMPLE_INDEX.key(), indexParams.getLoadSampleIndex()); params.putIfNotEmpty(VariantStorageOptions.LOAD_ARCHIVE.key(), indexParams.getLoadArchive()); params.putIfNotEmpty(VariantStorageOptions.LOAD_HOM_REF.key(), indexParams.getLoadHomRef()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java index 04606492bff..6121cf1e247 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java @@ -134,7 +134,6 @@ private void rebuildSampleFileIds(VariantStorageMetadataManager metadataManager, for (Map.Entry> entry : batch.entrySet()) { Integer sampleId = entry.getKey(); List fileIds = entry.getValue(); - List actualFiles = metadataManager.getSampleMetadata(studyId, sampleId).getFiles(); if (actualFiles.size() != fileIds.size() || !actualFiles.containsAll(fileIds)) { fixedSamples++; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index d68f363b36d..0f1fc2bec67 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -7,8 +7,8 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; -import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.slf4j.Logger; @@ -18,12 +18,9 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; -public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { +public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { public final static String DOCKER_INPUT_PATH = "/data/input"; public final static String DOCKER_OUTPUT_PATH = "/data/output"; @@ -31,14 +28,51 @@ public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor public static final String STDOUT_FILENAME = "stdout.txt"; public static final String STDERR_FILENAME = "stderr.txt"; + public String getDockerImageName() throws ToolException { + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[0]; + } + public static final String DOCKER_CLI_MSG = "Docker CLI: "; - public String getDockerImageName() { - return "opencb/opencga-ext-tools"; + public String getDockerImageVersion() throws ToolException { + if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) { + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1]; + } else { + return GitRepositoryState.getInstance().getBuildVersion(); + } + } + + protected AnalysisTool getAnalysisTool(String toolId, String version) throws ToolException { + for (AnalysisTool tool : getConfiguration().getAnalysis().getTools()) { + if (toolId.equals(tool.getId()) && version.equals(tool.getVersion())) { + return tool; + } + } + throw new ToolException("Missing analyis tool (ID = " + toolId + ", version = " + version + ") in configuration file"); + } + + public String getDockerImageName(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + return tool.getDockerId().split(":")[0]; + } + + public String getDockerImageVersion(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + if (tool.getDockerId().contains(":")) { + return tool.getDockerId().split(":")[1]; + } else { + return null; + } } - public String getDockerImageVersion() { - return GitRepositoryState.getInstance().getBuildVersion(); + protected String getToolResource(String toolId, String version, String resourceKey) throws ToolException { + // Get resources from the configuration file + AnalysisTool tool = getAnalysisTool(toolId, version); + if (!tool.getResources().containsKey(resourceKey)) { + throw new ToolException("Error getting resource " + resourceKey + " of analysis tool (ID = " + toolId + ", version = " + + version + "): it does not exist in the configuration file"); + } + return tool.getResources().get(resourceKey); } private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class); @@ -59,6 +93,14 @@ protected StringBuilder initCommandLine() { return new StringBuilder("docker run --log-driver=none -a stdin -a stdout -a stderr "); } + protected StringBuilder initCommandLine(String user) { + StringBuilder sb = initCommandLine(); + if (StringUtils.isNotEmpty(user)) { + sb.append("--user ").append(user); + } + return sb; + } + protected Map appendMounts(List> inputFilenames, StringBuilder sb) { Map mountMap = new HashMap<>(); @@ -84,7 +126,7 @@ protected Map appendMounts(List> inputFilen return mountMap; } - protected void appendCommand(String command, StringBuilder sb) { + protected void appendCommand(String command, StringBuilder sb) throws ToolException { // Docker image and version sb.append(getDockerImageName()); if (StringUtils.isNotEmpty(getDockerImageVersion())) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index c4a9d87d4f7..a6d34af3f37 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; @@ -34,6 +35,14 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy { public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants" + " from whole-exome or whole-genome sequencing data."; + // It must match the tool prefix in the tool keys for exomiser in the configuration file + public final static String EXOMISER_PREFIX = "exomiser-"; + + // It must match the resources key in the exomiser/tool section in the configuration file + public final static String HG19_RESOURCE_KEY = "HG19"; + public final static String HG38_RESOURCE_KEY = "HG38"; + public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE"; + @ToolParams protected final ExomiserWrapperParams analysisParams = new ExomiserWrapperParams(); @@ -43,6 +52,14 @@ protected void check() throws Exception { if (StringUtils.isEmpty(getStudy())) { throw new ToolException("Missing study"); } + + // Check exomiser version + if (StringUtils.isEmpty(analysisParams.getExomiserVersion())) { + // Missing exomiser version use the default one + String exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); + analysisParams.setExomiserVersion(exomiserVersion); + } } @Override @@ -53,6 +70,7 @@ protected void run() throws Exception { getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(study) .setSampleId(analysisParams.getSample()) + .setExomiserVersion(analysisParams.getExomiserVersion()) .setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType())) .execute(); }); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index a6325932fb9..2d0d0dfb0ab 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; @@ -10,6 +9,7 @@ import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.exec.Command; +import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; @@ -30,10 +30,14 @@ import java.io.*; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import static org.opencb.commons.utils.FileUtils.copyFile; +import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.*; + @ToolExecutor(id = ExomiserWrapperAnalysisExecutor.ID, tool = ExomiserWrapperAnalysis.ID, source = ToolExecutor.Source.STORAGE, @@ -46,24 +50,32 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private final static String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; private static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; - public final static String DOCKER_IMAGE_NAME = "exomiser/exomiser-cli"; - public final static String DOCKER_IMAGE_VERSION = "13.1.0"; - private String studyId; private String sampleId; + private String exomiserVersion; private ClinicalAnalysis.Type clinicalAnalysisType; private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void run() throws ToolException { + public void run() throws ToolException, IOException, CatalogException { // Check HPOs, it will use a set to avoid duplicate HPOs, // and it will check both phenotypes and disorders - logger.info("{}: Checking individual for sample {} in study {}", ID, sampleId, studyId); + logger.info("Checking individual for sample {} in study {}", sampleId, studyId); Set hpos = new HashSet<>(); Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, getVariantStorageManager().getCatalogManager(), getToken()); + // Check assembly + String assembly = IndividualQcUtils.getAssembly(studyId, getVariantStorageManager().getCatalogManager(), getToken()); + if (assembly.equalsIgnoreCase("GRCh38")) { + assembly = "hg38"; +// } else if (assembly.equalsIgnoreCase("GRCh37")) { +// assembly = "hg19"; + } else { + throw new ToolException("Invalid assembly '" + assembly + "'. Supported assemblies are: GRCh38"); + } + // Set father and mother if necessary (family ?) if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { individual.setFather(IndividualQcUtils.getIndividualById(studyId, individual.getFather().getId(), @@ -74,7 +86,7 @@ public void run() throws ToolException { getVariantStorageManager().getCatalogManager(), getToken())); } - logger.info("{}: Individual found: {}", ID, individual.getId()); + logger.info("Individual found: {}", individual.getId()); if (CollectionUtils.isNotEmpty(individual.getPhenotypes())) { for (Phenotype phenotype : individual.getPhenotypes()) { if (phenotype.getId().startsWith("HP:")) { @@ -94,7 +106,7 @@ public void run() throws ToolException { throw new ToolException("Missing phenotypes, i.e. HPO terms, for individual/sample (" + individual.getId() + "/" + sampleId + ")"); } - logger.info("{}: Getting HPO for individual {}: {}", ID, individual.getId(), StringUtils.join(hpos, ",")); + logger.info("Getting HPO for individual {}: {}", individual.getId(), StringUtils.join(hpos, ",")); List samples = new ArrayList<>(); samples.add(sampleId); @@ -137,8 +149,8 @@ public void run() throws ToolException { QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id,studies.samples"); - logger.info("{}: Exomiser exports variants using the query: {}", ID, query.toJson()); - logger.info("{}: Exomiser exports variants using the query options: {}", ID, queryOptions.toJson()); + logger.info("Exomiser exports variants using the query: {}", query.toJson()); + logger.info("Exomiser exports variants using the query options: {}", queryOptions.toJson()); try { getVariantStorageManager().exportData(vcfPath.toString(), VariantWriterFactory.VariantOutputFormat.VCF_GZ, null, query, @@ -153,30 +165,34 @@ public void run() throws ToolException { // Copy the analysis try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME) + .toFile(), getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser analysis file", e); } - // Copy the application.properties + // Copy the application.properties and update data according to Exomiser version try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile()); + Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME) + .toFile(), target.toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser properties file", e); } // Copy the output options try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser output options file", e); } // Build the docker command line to run Exomiser - StringBuilder sb = initCommandLine(); + String[] userAndGroup = FileUtils.getUserAndGroup(getOutDir(), true); + String dockerUser = userAndGroup[0] + ":" + userAndGroup[1]; + logger.info("Docker user: {}", dockerUser); + StringBuilder sb = initCommandLine(dockerUser); // Append mounts sb.append(" --mount type=bind,source=" + exomiserDataPath + ",target=/data") @@ -192,7 +208,8 @@ public void run() throws ToolException { sb.append(" --ped /jobdir/").append(pedigreeFile.getName()); } sb.append(" --vcf /jobdir/" + vcfPath.getFileName()) - .append(" --assembly hg38 --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) + .append(" --assembly ").append(assembly) + .append(" --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); // Execute command and redirect stdout and stderr to the files @@ -365,7 +382,7 @@ private Path getAnalysisDataPath(String analysisId) throws ToolException { } private Path getExomiserDataPath(Path openCgaHome) throws ToolException { - Path exomiserDataPath = openCgaHome.resolve("analysis/resources/exomiser"); + Path exomiserDataPath = openCgaHome.resolve("analysis/resources/" + ExomiserWrapperAnalysis.ID); if (!exomiserDataPath.toFile().exists()) { if (!exomiserDataPath.toFile().mkdirs()) { throw new ToolException("Error creating the Exomiser data directory"); @@ -374,33 +391,37 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { // Mutex management to avoid multiple downloadings at the same time // the first to come, download data, others have to wait for - File readyFile = exomiserDataPath.resolve("READY").toFile(); - File preparingFile = exomiserDataPath.resolve("PREPARING").toFile(); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); + String resourceVersion = Paths.get(resource).getFileName().toString().split("[_]")[0]; + File readyFile = exomiserDataPath.resolve("READY-" + resourceVersion).toFile(); + File preparingFile = exomiserDataPath.resolve("PREPARING-" + resourceVersion).toFile(); // If all is ready, then return if (readyFile.exists()) { - logger.info("{}: Exomiser data is already downloaded, so Exomiser analysis is ready to be executed.", ID); + logger.info("Exomiser {} data {} is already downloaded, so Exomiser analysis is ready to be executed.", exomiserVersion, + resourceVersion); return exomiserDataPath; } // If it is preparing, then wait for ready and then return if (preparingFile.exists()) { long startTime = System.currentTimeMillis(); - logger.info("{}: Exomiser data is downloading, waiting for it...", ID); + logger.info("Exomiser {} data {} is downloading, waiting for it...", exomiserVersion, resourceVersion); while (!readyFile.exists()) { try { Thread.sleep(10000); } catch (InterruptedException e) { // Nothing to do here + preparingFile.delete(); throw new ToolException(e); } long elapsedTime = System.currentTimeMillis() - startTime; if (elapsedTime > 18000000) { - throw new ToolException("Unable to run the Exomiser analysis because of Exomiser data is not ready yet: maximum" - + " waiting time exceeded"); + throw new ToolException("Unable to run the Exomiser analysis because of Exomiser " + exomiserVersion + " data " + + resourceVersion + " is not ready yet: maximum waiting time exceeded"); } } - logger.info("{}: Exomiser data is now downloaded: Exomiser analysis is ready to be executed", ID); + logger.info("Exomiser {} data is now downloaded: Exomiser analysis is ready to be executed", exomiserVersion); return exomiserDataPath; } @@ -408,38 +429,49 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { try { preparingFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error creating the Exomiser data directory"); + preparingFile.delete(); + throw new ToolException("Error creating the Exomiser " + exomiserVersion + " data " + resourceVersion + " directory"); } - // Download and unzip files + // Download resources and unzip files try { - downloadAndUnzip(exomiserDataPath, "2109_hg38.zip"); - downloadAndUnzip(exomiserDataPath, "2109_phenotype.zip"); + downloadAndUnzip(exomiserDataPath, HG38_RESOURCE_KEY); + downloadAndUnzip(exomiserDataPath, PHENOTYPE_RESOURCE_KEY); } catch (ToolException e) { // If something wrong happened, the preparing file has to be deleted preparingFile.delete(); - throw new ToolException("Something wrong happened when preparing Exomiser data", e); + throw new ToolException("Something wrong happened when preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } // Mutex management, signal exomiser data is ready try { readyFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error preparing Exomiser data", e); + throw new ToolException("Error preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } preparingFile.delete(); return exomiserDataPath; } + private String getHg38DataVersion() throws ToolException { + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, HG38_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + + private String getPhenotypeDataVersion() throws ToolException { + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + @Override - public String getDockerImageName() { - return DOCKER_IMAGE_NAME; + public String getDockerImageName() throws ToolException { + return getDockerImageName(ExomiserWrapperAnalysis.ID, exomiserVersion); } @Override - public String getDockerImageVersion() { - return DOCKER_IMAGE_VERSION; + public String getDockerImageVersion() throws ToolException { + return getDockerImageVersion(ExomiserWrapperAnalysis.ID, exomiserVersion); } public String getStudyId() { @@ -451,21 +483,38 @@ public ExomiserWrapperAnalysisExecutor setStudyId(String studyId) { return this; } - private void downloadAndUnzip(Path exomiserDataPath, String filename) throws ToolException { - URL url = null; - - // Download data - try { - url = new URL("http://resources.opencb.org/opencb/opencga/analysis/exomiser/" + filename); - logger.info("{}: Downloading Exomiser data: {} in {}", ID, url, exomiserDataPath); - ResourceUtils.downloadThirdParty(url, exomiserDataPath); - } catch (IOException e) { - throw new ToolException("Error downloading Exomiser data from " + url, e); + private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws ToolException { + String filename; + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, resourceKey); + if (resource.startsWith("file://")) { + // Copy resouce + try { + Path sourcePath = Paths.get(resource); + filename = sourcePath.getFileName().toString(); + Files.copy(sourcePath, exomiserDataPath.resolve(filename)); + } catch (IOException e) { + throw new ToolException("Error copying Exomiser data from " + resource, e); + } + } else { + // Download resource + String url; + if (resource.startsWith("http://") || resource.startsWith("https://") || resource.startsWith("ftp://")) { + url = resource; + } else { + url = getConfiguration().getAnalysis().getResourceUrl() + resource; + } + logger.info("Downloading Exomiser data: {} in {}", url, exomiserDataPath); + try { + ResourceUtils.downloadThirdParty(new URL(url), exomiserDataPath); + filename = Paths.get(url).getFileName().toString(); + } catch (IOException e) { + throw new ToolException("Error downloading Exomiser data from " + url, e); + } } // Unzip try { - logger.info("{}: Decompressing Exomiser data: {}", ID, filename); + logger.info("Decompressing Exomiser {} data: {}", exomiserDataPath, filename); new Command("unzip -o -d " + exomiserDataPath + " " + exomiserDataPath + "/" + filename) .setOutputOutputStream(new DataOutputStream(new FileOutputStream(getOutDir().resolve("stdout_unzip_" + filename + ".txt").toFile()))) @@ -473,11 +522,11 @@ private void downloadAndUnzip(Path exomiserDataPath, String filename) throws Too + filename + ".txt").toFile()))) .run(); } catch (FileNotFoundException e) { - throw new ToolException("Error unzipping Exomiser data: " + filename, e); + throw new ToolException("Error unzipping Exomiser " + exomiserVersion + " data: " + filename, e); } // Free disk space - logger.info("{}: Deleting Exomiser data: {}", ID, filename); + logger.info("Deleting Exomiser data: {}", filename); exomiserDataPath.resolve(filename).toFile().delete(); } @@ -490,6 +539,15 @@ public ExomiserWrapperAnalysisExecutor setSampleId(String sampleId) { return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperAnalysisExecutor setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } + public ClinicalAnalysis.Type getClinicalAnalysisType() { return clinicalAnalysisType; } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java index f77a1025377..87a8d1d2a26 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java @@ -24,6 +24,7 @@ import org.opencb.biodata.models.variant.avro.SequenceOntologyTerm; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractClinicalManagerTest; @@ -51,25 +52,17 @@ public static AbstractClinicalManagerTest getClinicalTest(OpenCGATestExternalRes clinicalTest.catalogManagerResource = opencga.getCatalogManagerExternalResource(); clinicalTest.setUp(); - // Exomiser analysis - Path exomiserDataPath = opencga.getOpencgaHome().resolve("analysis/exomiser"); - Files.createDirectories(exomiserDataPath); - Path parent = Paths.get(ClinicalAnalysisUtilsTest.class.getClassLoader().getResource("pheno").getPath()).getParent(); - Files.copy(parent.resolve("exomiser/application.properties"), exomiserDataPath.resolve("application.properties"), - StandardCopyOption.REPLACE_EXISTING); - Files.copy(parent.resolve("exomiser/output.yml"), exomiserDataPath.resolve("output.yml"), - StandardCopyOption.REPLACE_EXISTING); - // Storage ObjectMap storageOptions = new ObjectMap() .append(VariantStorageOptions.ANNOTATE.key(), true) .append(VariantStorageOptions.STATS_CALCULATE.key(), false); - VariantStorageManager variantStorageManager = new VariantStorageManager(opencga.getCatalogManager(), opencga.getStorageEngineFactory()); + VariantStorageManager variantStorageManager = opencga.getVariantStorageManager(); Path outDir = Paths.get("target/test-data").resolve("junit_clinical_analysis_" + RandomStringUtils.randomAlphabetic(10)); Files.createDirectories(outDir); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, clinicalTest.studyFqn, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "family.vcf", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "exomiser.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "HG004.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index 441f03d06a3..a6fd99aa3bc 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -1,12 +1,15 @@ package org.opencb.opencga.analysis.clinical.exomiser; import org.apache.commons.lang3.StringUtils; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.tools.variant.VariantNormalizer; -import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; @@ -34,6 +37,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeThat; +import static org.junit.Assume.assumeTrue; @Category(MediumTests.class) public class ExomiserInterpretationAnalysisTest { @@ -72,8 +76,13 @@ public void testNormalization() throws NonStandardCompliantSampleField { } @Test - public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_single")); @@ -83,11 +92,14 @@ public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ClinicalAnalysis clinicalAnalysis = caResult.getResults().get(0); assertEquals(0, clinicalAnalysis.getSecondaryInterpretations().size()); - ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); + System.out.println("opencga.getOpencgaHome() = " + opencga.getOpencgaHome().toAbsolutePath()); + System.out.println("outDir = " + outDir); + ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID2); + .setClinicalAnalysisId(clinicalTest.CA_ID2) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); @@ -115,8 +127,13 @@ public void singleSingleExomiserAnalysis() throws IOException, CatalogException, } @Test - public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_family")); @@ -129,7 +146,8 @@ public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, T exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID3); + .setClinicalAnalysisId(clinicalTest.CA_ID3) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); @@ -160,7 +178,12 @@ public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, T @Test public void trioSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assertTrue(opencga.getOpencgaHome().resolve("analysis").resolve("exomiser").resolve(exomiserVersion).toFile().exists()); + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_single")); @@ -173,7 +196,8 @@ public void trioSingleExomiserAnalysis() throws IOException, CatalogException, T exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID4); + .setClinicalAnalysisId(clinicalTest.CA_ID4) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 5c5956d7cf7..e3b69751771 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -24,12 +24,14 @@ import org.opencb.opencga.analysis.StorageManager; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; @@ -39,20 +41,20 @@ import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; +import org.reflections.Reflections; +import org.reflections.scanners.ResourcesScanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.net.URI; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Created on 26/08/15 @@ -72,7 +74,8 @@ public class OpenCGATestExternalResource extends ExternalResource { private ToolRunner toolRunner; - public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource = new HadoopVariantStorageTest.HadoopExternalResource(); + public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource + = new HadoopVariantStorageTest.HadoopExternalResource(); public OpenCGATestExternalResource() { this(false); @@ -216,9 +219,9 @@ public Path isolateOpenCGA() throws IOException { catalogManagerExternalResource.getConfiguration().serialize( new FileOutputStream(conf.resolve("configuration.yml").toFile())); - InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); - - storageConfiguration = StorageConfiguration.load(inputStream, "yml"); + try (InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml")) { + storageConfiguration = StorageConfiguration.load(inputStream, "yml"); + } storageConfiguration.getVariant().setDefaultEngine(storageEngine); if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { @@ -234,6 +237,9 @@ public Path isolateOpenCGA() throws IOException { StorageEngineFactory.configure(storageConfiguration); storageEngineFactory = StorageEngineFactory.get(storageConfiguration); + if (storageEngine.equals(DummyVariantStorageEngine.STORAGE_ENGINE_ID)) { + DummyVariantStorageEngine.configure(getStorageEngineFactory(), true); + } // inputStream = StorageEngine.class.getClassLoader().getResourceAsStream("client-configuration-test.yml"); // Files.copy(inputStream, conf.resolve("client-configuration.yml"), StandardCopyOption.REPLACE_EXISTING); @@ -252,20 +258,26 @@ public Path isolateOpenCGA() throws IOException { // Mutational signatue analysis Path analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/mutational-signature")).toAbsolutePath(); - inputStream = new FileInputStream("../opencga-app/app/analysis/mutational-signature/sv_clustering.R"); - Files.copy(inputStream, analysisPath.resolve("sv_clustering.R"), StandardCopyOption.REPLACE_EXISTING); + try (FileInputStream inputStream = new FileInputStream("../opencga-app/app/analysis/mutational-signature/sv_clustering.R")) { + Files.copy(inputStream, analysisPath.resolve("sv_clustering.R"), StandardCopyOption.REPLACE_EXISTING); + } // Pedigree graph analysis analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/pedigree-graph")).toAbsolutePath(); - inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R"); - Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); + try (FileInputStream inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R")) { + Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); + } // Exomiser analysis files - analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser")).toAbsolutePath(); + List exomiserVersions = Arrays.asList("13.1", "14.0"); List exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml"); - for (String exomiserFile : exomiserFiles) { - inputStream = new FileInputStream("../opencga-app/app/analysis/exomiser/" + exomiserFile); - Files.copy(inputStream, analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + for (String exomiserVersion : exomiserVersions) { + analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).toAbsolutePath()); + Path exomiserPath = Paths.get("../opencga-app/app/analysis/exomiser"); + for (String exomiserFile : exomiserFiles) { + String resource = exomiserVersion + "/" + exomiserFile; + Files.copy(exomiserPath.resolve(resource).toAbsolutePath(), analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + } } return opencgaHome; @@ -357,6 +369,34 @@ public String createTmpOutdir(String suffix) throws IOException { // return getCatalogManager().getJobManager().createJobOutDir(studyId, "I_tmp_" + date + sufix, sessionId).toString(); } + public void restore(URL resource) throws Exception { + if (resource.getProtocol().equals("jar")) { + Reflections reflections = new Reflections(resource.getPath().replace('/','.'), new ResourcesScanner()); + Set resources = reflections.getResources(x -> true); + for (String file : resources) { + catalogManagerExternalResource.getResourceUri(file.replace('.', '/')); + } + MongoBackupUtils.restore(getCatalogManager(), opencgaHome, opencgaHome + .resolve("resources") + .resolve(resource.getPath()) + .resolve("mongodb")); + } else { + MongoBackupUtils.restore(getCatalogManager(), opencgaHome, Paths.get(resource.toURI()).resolve("mongodb")); + } + catalogManagerExternalResource.resetCatalogManager(); + } + + public final VariantStorageEngine getVariantStorageEngineByProject(String projectFqn) throws Exception { + DataStore dataStore = getVariantStorageManager().getDataStoreByProjectId(projectFqn, getAdminToken()); + VariantStorageEngine variantStorageEngine = storageEngineFactory + .getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); + if (dataStore.getOptions() != null) { + variantStorageEngine.getOptions().putAll(dataStore.getOptions()); + } + return variantStorageEngine; + } + + // private class StorageLocalExecutorManager extends LocalExecutorManager { // // public StorageLocalExecutorManager(String sessionId) { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index c91909d3abb..f9e9392be80 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -44,6 +44,7 @@ import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool; @@ -179,6 +180,9 @@ public void setUp() throws Throwable { setUpCatalogManager(); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, STUDY, token); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, CANCER_STUDY, token); + file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token); variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index 8ad7f3c3479..ef6b79bd153 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.variant.manager; import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.*; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -32,6 +33,7 @@ import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; import org.opencb.opencga.analysis.variant.operations.*; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; @@ -53,10 +55,14 @@ import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -67,6 +73,7 @@ import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.slf4j.LoggerFactory; import java.nio.file.Path; import java.nio.file.Paths; @@ -85,8 +92,9 @@ public class VariantOperationsTest { public static final String USER = "user"; public static final String PASSWORD = TestParamConstants.PASSWORD; public static final String PROJECT = "project"; + public static final String PROJECT_FQN = ORGANIZATION + '@' + PROJECT; public static final String STUDY = "study"; - public static final String STUDY_FQN = ORGANIZATION + '@' + PROJECT + ':' + STUDY; + public static final String STUDY_FQN = PROJECT_FQN + ':' + STUDY; public static final String PHENOTYPE_NAME = "myPhenotype"; public static final Phenotype PHENOTYPE = new Phenotype(PHENOTYPE_NAME, PHENOTYPE_NAME, "mySource") .setStatus(Phenotype.Status.OBSERVED); @@ -156,8 +164,8 @@ public void tearDown() { if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) engine.getDBAdaptor()), Paths.get(opencga.createTmpOutdir("_hbase_print_variants_AFTER")).toUri()); } - } catch (Exception ignore) { - ignore.printStackTrace(); + } catch (Exception e) { + LoggerFactory.getLogger(getClass()).error("Ignoring exception printing variants", e); } hadoopExternalResource.after(); @@ -224,6 +232,8 @@ private void loadDataset() throws Throwable { solrExternalResource.configure(variantStorageManager.getVariantStorageEngineForStudyOperation(STUDY, new ObjectMap(), token)); } + dummyVariantSetup(variantStorageManager, STUDY, token); + file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token); // variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token); toolRunner.execute(VariantIndexOperationTool.class, STUDY, @@ -285,6 +295,15 @@ private void loadDataset() throws Throwable { } } + public static void dummyVariantSetup(VariantStorageManager variantStorageManager, String study, String token) + throws CatalogException, StorageEngineException { + variantStorageManager.variantSetup(study, new VariantSetupParams() + .setAverageFileSize("100B") + .setExpectedFiles(5) + .setExpectedSamples(5) + .setVariantsPerSample(1000), token); + } + public void setUpCatalogManager() throws Exception { catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), opencga.getAdminToken()); @@ -309,6 +328,84 @@ public void setUpCatalogManager() throws Exception { } + @Test + public void testSetup() throws Exception { + String study2 = "study2"; + String study2fqn = catalogManager.getStudyManager() + .create(PROJECT, study2, null, "Phase 1", "Done", null, null, null, null, null, token) + .first().getFqn(); + File file = opencga.createFile(study2, "variant-test-file.vcf.gz", token); + + try { + toolRunner.execute(VariantIndexOperationTool.class, study2, + new VariantIndexParams() + .setFile(file.getId()) + .setAnnotate(false) + .setLoadHomRef(YesNoAuto.YES.name()), + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); + fail("Should have thrown an exception"); + } catch (ToolException e) { + MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); + } + + try { + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_SAMPLES_PER_FILE) + .setExpectedFiles(20) + .setExpectedSamples(100) + .setNormalizeExtensions(Arrays.asList("VS", "SV")); + variantStorageManager.variantSetup(study2, setupParams, token); + fail("should have failed"); + } catch (Exception e) { + System.err.println(e.getMessage()); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Unsupported normalize extensions")); + } + + try { + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_SAMPLES_PER_FILE) + .setExpectedSamples(100) + .setNormalizeExtensions(Arrays.asList("VS", "SV")); + variantStorageManager.variantSetup(study2, setupParams, token); + fail("should have failed"); + } catch (Exception e) { + System.err.println(e.getMessage()); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Missing expectedFiles")); + } + + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_FILES_PER_SAMPLE) + .setExpectedFiles(20) + .setAverageSamplesPerFile(2.5f) + .setExpectedSamples(10) + .setNormalizeExtensions(Arrays.asList("SV", "VAF")); + VariantSetupResult result = variantStorageManager.variantSetup(study2, setupParams, token); + assertEquals(VariantSetupResult.Status.READY, result.getStatus()); + + toolRunner.execute(VariantIndexOperationTool.class, study2, + new VariantIndexParams() + .setFile(file.getId()) + .setLoadHomRef(YesNoAuto.YES.name()), + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); + + VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject(PROJECT_FQN).getMetadataManager(); + int studyId = metadataManager.getStudyId(study2fqn); + int sampleId = metadataManager.getSampleId(studyId, "NA19600"); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); + assertEquals(VariantStorageEngine.SplitData.MULTI, sampleMetadata.getSplitData()); + + try { + variantStorageManager.variantSetup(STUDY, setupParams, token); + fail("Should fail"); + } catch (Exception e) { + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Unable to execute variant-setup on study")); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("It already has indexed files.")); + } + } + @Test public void testVariantFileReload() throws Exception { try { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java index 0a5ac63293d..4aeedde871f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java @@ -17,12 +17,15 @@ package org.opencb.opencga.analysis.variant.manager; import org.apache.commons.lang3.RandomStringUtils; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.manager.operations.AbstractVariantOperationManagerTest; +import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.models.file.File; @@ -64,6 +67,18 @@ public void testConfigure() throws CatalogException, StorageEngineException { variantManager.configureStudy(studyFqn, expectedStudyConfiguration1, sessionId); variantManager.configureStudy(studyId2, expectedStudyConfiguration2, sessionId); + + try { + Study study = catalogManager.getStudyManager().create(projectId, "s_no_setup", "s_no_setup", "s_no_setup", + "Study 1", null, null, null, Collections.singletonMap(VariantStatsAnalysis.STATS_AGGREGATION_CATALOG, getAggregation()), null, sessionId) + .first(); + // Variant setup mandatory for configuring study + variantManager.configureStudy(study.getFqn(), expectedStudyConfiguration1, sessionId); + fail("Expect exception. Study not setup"); + } catch (Exception e) { + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); + } + ObjectMap configuration = variantManager.getDataStoreByProjectId(projectId, sessionId).getOptions(); assertEquals(expectedConfiguration, configuration); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java index 6063e8f1070..73838816fb3 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java @@ -27,6 +27,7 @@ import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.analysis.variant.operations.OperationTool; import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; @@ -169,6 +170,10 @@ public final void setUpAbstract() throws Exception { true, null, QueryOptions.empty(), sessionId).first().getId(); files = Arrays.asList(new File[5]); + + + VariantOperationsTest.dummyVariantSetup(variantManager, studyFqn, sessionId); + VariantOperationsTest.dummyVariantSetup(variantManager, studyId2, sessionId); } @After @@ -197,7 +202,11 @@ protected File getSmallFile() throws IOException, CatalogException { } protected File create(String resourceName) throws IOException, CatalogException { - return create(studyId, getResourceUri(resourceName)); + return create(resourceName, "data/vcfs/"); + } + + protected File create(String resourceName, String path) throws IOException, CatalogException { + return create(studyId, getResourceUri(resourceName), path); } protected File create(String studyId, URI uri) throws IOException, CatalogException { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java index 74014489902..56e79581e2f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java @@ -32,7 +32,10 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.testclassification.duration.MediumTests; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -83,6 +86,42 @@ public void testLoadAndRemoveOneWithOtherLoaded() throws Exception { testLoadAndRemoveOne(); } + @Test + public void testLoadAndRemoveForce() throws Exception { + File file77 = create("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); + File file78 = create("platinum/1K.end.platinum-genomes-vcf-NA12878_S1.genome.vcf.gz"); + File file79 = create("platinum/1K.end.platinum-genomes-vcf-NA12879_S1.genome.vcf.gz"); + indexFile(file77, new QueryOptions(), outputId); + indexFile(file78, new QueryOptions(), outputId); + + removeFile(file77, new QueryOptions()); + + try { + removeFile(file77, new QueryOptions()); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Unable to remove variants from file")); + } + removeFile(file77, new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + + try { + removeFile(file79, new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("File not found in storage.")); + } + Path file77Path = Paths.get(file77.getUri()); + Path otherDir = file77Path.getParent().resolve("other_dir"); + Files.createDirectory(otherDir); + Path otherFile = Files.copy(file77Path, otherDir.resolve(file77Path.getFileName())); + File file77_2 = create(studyFqn, otherFile.toUri(), "other_dir"); + + try { + removeFile(studyFqn, Collections.singletonList(file77_2.getPath()), new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Unable to remove variants from file")); + assertTrue(e.getMessage(), e.getMessage().contains("Instead, found file with same name but different path")); + } + } + @Test public void testLoadAndRemoveMany() throws Exception { @@ -101,7 +140,7 @@ public void testLoadAndRemoveDifferentChromosomes() throws Exception { List files = new ArrayList<>(); files.add(create("1k.chr1.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz")); files.add(create("10k.chr22.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz")); - indexFiles(files, new QueryOptions(), outputId); + indexFiles(files, new QueryOptions(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME), outputId); removeFile(files.get(0), new QueryOptions()); } @@ -127,8 +166,13 @@ private void removeFile(List files, QueryOptions options) throws Exception Study study = catalogManager.getFileManager().getStudy(ORGANIZATION, files.get(0), sessionId); String studyId = study.getFqn(); + removeFile(studyId, fileIds, options); + } + + private void removeFile(String studyId, List fileIds, QueryOptions options) throws Exception { + Path outdir = Paths.get(opencga.createTmpOutdir(studyId, "_REMOVE_", sessionId)); - variantManager.removeFile(studyId, fileIds, new QueryOptions(), outdir.toUri(), sessionId); + variantManager.removeFile(studyId, fileIds, new QueryOptions(options), outdir.toUri(), sessionId); // assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), diff --git a/opencga-analysis/src/test/resources/exomiser/output.yml b/opencga-analysis/src/test/resources/exomiser/output.yml deleted file mode 100644 index 6a70c778641..00000000000 --- a/opencga-analysis/src/test/resources/exomiser/output.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -#outputContributingVariantsOnly: true -#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results -#numGenes: 20 -#minExomiserGeneScore: 0.7 -#outputPrefix options: specify the path/filename without an extension and this will be added -# according to the outputFormats option. If unspecified this will default to the following: -# {exomiserDir}/results/input-vcf-name-exomiser-results.html -# alternatively, specify a fully qualifed path only. e.g. /users/jules/exomes/analysis -outputPrefix: /jobdir/exomiser_output -#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) -#outputFormats: [HTML, JSON, TSV_GENE, TSV_VARIANT, VCF] -outputFormats: [TSV_VARIANT, JSON, HTML] diff --git a/opencga-analysis/src/test/resources/exomiser/application.properties b/opencga-app/app/analysis/exomiser/13.1/application.properties similarity index 95% rename from opencga-analysis/src/test/resources/exomiser/application.properties rename to opencga-app/app/analysis/exomiser/13.1/application.properties index e722c2fd89b..1775e32d14c 100644 --- a/opencga-analysis/src/test/resources/exomiser/application.properties +++ b/opencga-app/app/analysis/exomiser/13.1/application.properties @@ -40,6 +40,6 @@ exomiser.data-directory=/data exomiser.hg38.data-version=2109 #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.hg38.variant-white-list-path=2109_hg38_clinvar_whitelist.tsv.gz exomiser.phenotype.data-version=2109 logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/exomiser-analysis.yml rename to opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml diff --git a/opencga-app/app/analysis/exomiser/output.yml b/opencga-app/app/analysis/exomiser/13.1/output.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/output.yml rename to opencga-app/app/analysis/exomiser/13.1/output.yml diff --git a/opencga-app/app/analysis/exomiser/application.properties b/opencga-app/app/analysis/exomiser/14.0/application.properties similarity index 88% rename from opencga-app/app/analysis/exomiser/application.properties rename to opencga-app/app/analysis/exomiser/14.0/application.properties index e722c2fd89b..d1658370370 100644 --- a/opencga-app/app/analysis/exomiser/application.properties +++ b/opencga-app/app/analysis/exomiser/14.0/application.properties @@ -37,9 +37,11 @@ exomiser.data-directory=/data #remm.version=0.3.1.post1 #cadd.version=1.4 #exomiser.hg19.data-version=1811 -exomiser.hg38.data-version=2109 +exomiser.hg38.data-version=2402 #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz -exomiser.phenotype.data-version=2109 +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.phenotype.data-version=2402 +exomiser.hg38.clin-var-data-version=2402 +exomiser.hg38.use-clinvar-white-list=true logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml new file mode 100644 index 00000000000..e3d064ba07b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml @@ -0,0 +1,85 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + X_DOMINANT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + X_RECESSIVE_HOM_ALT: 0.1, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + UK10K, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + # GNOMAD_E_FIN, + GNOMAD_E_NFE, + # GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + # GNOMAD_G_FIN, + GNOMAD_G_NFE, + # GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +#all steps are optional +steps: [ + #hiPhivePrioritiser: {}, + #priorityScoreFilter: {priorityType: HIPHIVE_PRIORITY, minPriorityScore: 0.500}, + #intervalFilter: {interval: 'chr10:123256200-123256300'}, + # or for multiple intervals: + #intervalFilter: {intervals: ['chr10:123256200-123256300', 'chr10:123256290-123256350']}, + # or using a BED file - NOTE this should be 0-based, Exomiser otherwise uses 1-based coordinates in line with VCF + #intervalFilter: {bed: /full/path/to/bed_file.bed}, + #genePanelFilter: {geneSymbols: ['FGFR1','FGFR2']}, + #geneBlacklistFilter: { }, + failedVariantFilter: { }, + #qualityFilter: {minQuality: 50.0}, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + UPSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + DOWNSTREAM_GENE_VARIANT + ] + }, + # removes variants represented in the database + #knownVariantFilter: {}, + frequencyFilter: {maxFrequency: 2.0}, + pathogenicityFilter: {keepNonPathogenic: true}, + # inheritanceFilter and omimPrioritiser should always run AFTER all other filters have completed + # they will analyse genes according to the specified modeOfInheritance above- UNDEFINED will not be analysed. + inheritanceFilter: {}, + # omimPrioritiser isn't mandatory. + omimPrioritiser: {}, + #priorityScoreFilter: {minPriorityScore: 0.4}, + # Other prioritisers: Only combine omimPrioritiser with one of these. + # Don't include any if you only want to filter the variants. + hiPhivePrioritiser: {}, + # or run hiPhive in benchmarking mode: + #hiPhivePrioritiser: {runParams: 'mouse'}, + #phivePrioritiser: {} + #phenixPrioritiser: {} + #exomeWalkerPrioritiser: {seedGeneIds: [11111, 22222, 33333]} +] diff --git a/opencga-app/app/analysis/exomiser/14.0/output.yml b/opencga-app/app/analysis/exomiser/14.0/output.yml new file mode 100644 index 00000000000..4fc4a4faa6b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/output.yml @@ -0,0 +1,11 @@ +--- +outputContributingVariantsOnly: true +#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results +numGenes: 20 +minExomiserGeneScore: 0.7 +# Path to the desired output directory. Will default to the 'results' subdirectory of the exomiser install directory +outputDirectory: /jobdir/ +# Filename for the output files. Will default to {input-vcf-filename}-exomiser +outputFileName: exomiser_output +#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) +outputFormats: [TSV_VARIANT, JSON, HTML] diff --git a/opencga-app/app/misc/clients/javascript_client_generator.py b/opencga-app/app/misc/clients/javascript_client_generator.py index 2ac2fb88667..e471d09ac48 100644 --- a/opencga-app/app/misc/clients/javascript_client_generator.py +++ b/opencga-app/app/misc/clients/javascript_client_generator.py @@ -116,7 +116,7 @@ def get_method_definition(self, category, endpoint): f'"{self.get_endpoint_subcategory()}"' if self.subcategory else "null", self.get_endpoint_id2() if self.get_endpoint_id2() else "null", f'"{self.get_endpoint_action()}"' if self.get_endpoint_action() else "null", - "data" if self.has_body() else False, + "data" if self.has_body() else ("null" if self.get_endpoint_method(endpoint).lower() == "post" else False), query_string_params ] if s) return (f' {self.get_method_doc(endpoint)}' diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index af4cf1bec4a..76ea292d610 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -239,12 +239,43 @@ jline-terminal-jna runtime + + org.opencb.opencga opencga-storage-core test-jar test + + org.opencb.opencga + opencga-storage-hadoop-core + test + + + org.opencb.opencga + opencga-storage-hadoop-core + test + test-jar + + + org.mockito + mockito-core + test + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + test + test-jar + + + org.yaml + snakeyaml + + + @@ -726,10 +757,15 @@ + Ensure that only one opencga-hadoop-shaded*.jar and opencga-storage-hadoop-compat*.jar libs are included + + + + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java index ac18f769392..4800ec88c19 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java @@ -317,7 +317,8 @@ private void exomiserInterpretation() throws Exception { ExomiserInterpretationAnalysis exomiserInterpretationAnalysis = new ExomiserInterpretationAnalysis(); exomiserInterpretationAnalysis.setUp(opencgaHome.toString(), new ObjectMap(), outDir, token); exomiserInterpretationAnalysis.setStudyId(cliOptions.study) - .setClinicalAnalysisId(cliOptions.clinicalAnalysis); + .setClinicalAnalysisId(cliOptions.clinicalAnalysis) + .setExomiserVersion(cliOptions.exomiserVersion); // exomiserInterpretationAnalysis.setPrimary(cliOptions.primary); exomiserInterpretationAnalysis.start(); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index 11d18532218..7d20deccb09 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -62,7 +62,6 @@ import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.exceptions.AnalysisExecutionException; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.operations.variant.*; @@ -1021,6 +1020,7 @@ private void exomiser() throws Exception { ObjectMap params = new ExomiserWrapperParams( cliOptions.sample, cliOptions.clinicalAnalysisType, + cliOptions.exomiserVersion, cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java index 853d280736d..c2bac1cd1aa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java @@ -23,6 +23,8 @@ import static org.opencb.opencga.analysis.clinical.InterpretationAnalysis.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.*; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @Parameters(commandNames = {"clinical"}, commandDescription = "Clinical analysis commands") @@ -335,9 +337,13 @@ public class ExomiserInterpretationCommandOptions extends GeneralCliOptions.Stud @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = "Clinical analysis", required = true, arity = 1) + @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, + required = true, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion; + @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", arity = 1) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index aa25fe630e1..46ef28a52f6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -70,6 +70,8 @@ import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSampleQueryCommandOptions.SAMPLE_QUERY_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexCommandOptions.SECONDARY_INDEX_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexDeleteCommandOptions.SECONDARY_INDEX_DELETE_COMMAND; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_SAMPLE_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.core.api.ParamConstants.*; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND_DESCRIPTION; @@ -1842,12 +1844,15 @@ public class ExomiserAnalysisCommandOptions { @Parameter(names = {"--study"}, description = "Study where all the samples belong to.") public String study; - @Parameter(names = {"--sample"}, description = FieldConstants.SAMPLE_ID_DESCRIPTION, required = true) + @Parameter(names = {"--sample"}, description = EXOMISER_SAMPLE_DESCRIPTION, required = true) public String sample; @Parameter(names = {"--clinical-analysis-type"}, description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION) public String clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE.name(); + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion; + @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index 4d1ed716a48..fa5b3482284 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -104,7 +104,7 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List operationsList = asList( "cellbase-configure","variant-aggregate","variant-annotation-delete","variant-annotation-index","variant-annotation-save","variant-configure","variant-delete","variant-family-aggregate","variant-family-index","variant-index","variant-index-launcher","variant-julie-run","variant-metadata-repair","variant-metadata-synchronize","variant-prune","variant-sample-delete","variant-sample-index","variant-sample-index-configure","variant-score-delete","variant-score-index","variant-secondary-annotation-index","variant-secondary-sample-index","configure-variant-secondary-sample-index","variant-secondary-index","variant-secondary-index-delete","variant-stats-delete","variant-stats-index","variant-study-delete") + private List operationsList = asList( "cellbase-configure","variant-aggregate","variant-annotation-delete","variant-annotation-index","variant-annotation-save","variant-configure","variant-delete","variant-family-aggregate","variant-family-index","variant-index","variant-index-launcher","variant-julie-run","variant-metadata-repair","variant-metadata-synchronize","variant-prune","variant-sample-delete","variant-sample-index","variant-sample-index-configure","variant-score-delete","variant-score-index","variant-secondary-annotation-index","variant-secondary-sample-index","configure-variant-secondary-sample-index","variant-secondary-index","variant-secondary-index-delete","variant-setup","variant-stats-delete","variant-stats-index","variant-study-delete") .stream() .map(Candidate::new) .collect(toList()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index d99dbba3488..cfd4cee9919 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -364,6 +364,7 @@ public OpencgaCliOptionsParser() { operationsVariantStorageSubCommands.addCommand("configure-variant-secondary-sample-index", operationsVariantStorageCommandOptions.configureVariantSecondarySampleIndexCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-secondary-index", operationsVariantStorageCommandOptions.secondaryIndexVariantCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-secondary-index-delete", operationsVariantStorageCommandOptions.deleteVariantSecondaryIndexCommandOptions); + operationsVariantStorageSubCommands.addCommand("variant-setup", operationsVariantStorageCommandOptions.setupVariantCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-stats-delete", operationsVariantStorageCommandOptions.deleteVariantStatsCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-stats-index", operationsVariantStorageCommandOptions.indexVariantStatsCommandOptions); operationsVariantStorageSubCommands.addCommand("variant-study-delete", operationsVariantStorageCommandOptions.deleteVariantStudyCommandOptions); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java index e78430b3f96..99135ede6b0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java @@ -130,9 +130,9 @@ private RestResponse installCatalog() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InstallationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "secretKey",commandOptions.secretKey, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "secretKey", commandOptions.secretKey, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); installationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -161,7 +161,7 @@ private RestResponse jwtCatalog() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JWTParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "secretKey",commandOptions.secretKey, true); + putNestedIfNotEmpty(beanParams, "secretKey", commandOptions.secretKey, true); jWTParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -186,11 +186,11 @@ private RestResponse createUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "organization", commandOptions.organization, true); userCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -219,11 +219,11 @@ private RestResponse importUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserImportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "authenticationOriginId",commandOptions.authenticationOriginId, true); - putNestedIfNotNull(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "resourceType",commandOptions.resourceType, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotEmpty(beanParams, "studyGroup",commandOptions.studyGroup, true); + putNestedIfNotEmpty(beanParams, "authenticationOriginId", commandOptions.authenticationOriginId, true); + putNestedIfNotNull(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "resourceType", commandOptions.resourceType, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotEmpty(beanParams, "studyGroup", commandOptions.studyGroup, true); userImportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -287,12 +287,12 @@ private RestResponse syncUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupSyncParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "authenticationOriginId",commandOptions.authenticationOriginId, true); - putNestedIfNotEmpty(beanParams, "from",commandOptions.from, true); - putNestedIfNotEmpty(beanParams, "to",commandOptions.to, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotNull(beanParams, "syncAll",commandOptions.syncAll, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotEmpty(beanParams, "authenticationOriginId", commandOptions.authenticationOriginId, true); + putNestedIfNotEmpty(beanParams, "from", commandOptions.from, true); + putNestedIfNotEmpty(beanParams, "to", commandOptions.to, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotNull(beanParams, "syncAll", commandOptions.syncAll, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); groupSyncParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -322,8 +322,8 @@ private RestResponse usersUpdateGroups() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserUpdateGroup.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "studyIds",commandOptions.studyIds, true); - putNestedIfNotNull(beanParams, "groupIds",commandOptions.groupIds, true); + putNestedIfNotNull(beanParams, "studyIds", commandOptions.studyIds, true); + putNestedIfNotNull(beanParams, "groupIds", commandOptions.groupIds, true); userUpdateGroup = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index ddc77f0009b..566389f36b2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -140,12 +140,12 @@ private RestResponse runBwa() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), BwaWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "fastaFile",commandOptions.fastaFile, true); - putNestedIfNotEmpty(beanParams, "fastq1File",commandOptions.fastq1File, true); - putNestedIfNotEmpty(beanParams, "fastq2File",commandOptions.fastq2File, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "bwaParams",commandOptions.bwaParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "fastaFile", commandOptions.fastaFile, true); + putNestedIfNotEmpty(beanParams, "fastq1File", commandOptions.fastq1File, true); + putNestedIfNotEmpty(beanParams, "fastq2File", commandOptions.fastq2File, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "bwaParams", commandOptions.bwaParams, true); bwaWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -184,9 +184,9 @@ private RestResponse runCoverageIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CoverageIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFileId",commandOptions.bamFileId, true); - putNestedIfNotEmpty(beanParams, "baiFileId",commandOptions.baiFileId, true); - putNestedIfNotNull(beanParams, "windowSize",commandOptions.windowSize, true); + putNestedIfNotEmpty(beanParams, "bamFileId", commandOptions.bamFileId, true); + putNestedIfNotEmpty(beanParams, "baiFileId", commandOptions.baiFileId, true); + putNestedIfNotNull(beanParams, "windowSize", commandOptions.windowSize, true); coverageIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -225,9 +225,9 @@ private RestResponse coverageQcGeneCoverageStatsRun() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentGeneCoverageStatsParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFile",commandOptions.bamFile, true); - putNestedIfNotNull(beanParams, "genes",commandOptions.genes, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "bamFile", commandOptions.bamFile, true); + putNestedIfNotNull(beanParams, "genes", commandOptions.genes, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); alignmentGeneCoverageStatsParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -323,9 +323,9 @@ private RestResponse runDeeptools() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), DeeptoolsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "deeptoolsParams",commandOptions.deeptoolsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "deeptoolsParams", commandOptions.deeptoolsParams, true); deeptoolsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -364,9 +364,9 @@ private RestResponse runFastqc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FastqcWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "inputFile",commandOptions.inputFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "fastqcParams",commandOptions.fastqcParams, true); + putNestedIfNotEmpty(beanParams, "inputFile", commandOptions.inputFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "fastqcParams", commandOptions.fastqcParams, true); fastqcWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -405,8 +405,8 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "fileId",commandOptions.fileId, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "fileId", commandOptions.fileId, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); alignmentIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -445,9 +445,9 @@ private RestResponse runPicard() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PicardWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "picardParams",commandOptions.picardParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "picardParams", commandOptions.picardParams, true); picardWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -486,10 +486,10 @@ private RestResponse runQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentQcParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFile",commandOptions.bamFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "bamFile", commandOptions.bamFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); alignmentQcParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -560,10 +560,10 @@ private RestResponse runSamtools() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SamtoolsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "inputFile",commandOptions.inputFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "samtoolsParams",commandOptions.samtoolsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "inputFile", commandOptions.inputFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "samtoolsParams", commandOptions.samtoolsParams, true); samtoolsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index 84eeffe1be6..07dbf9230f9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -238,8 +238,8 @@ private RestResponse updateAcl() throws Exception .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); clinicalAnalysisAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -273,7 +273,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -305,7 +305,7 @@ private RestResponse updateClinicalConfiguration() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisStudyConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "interpretation.defaultFilter",commandOptions.interpretationDefaultFilter, true); + putNestedMapIfNotEmpty(beanParams, "interpretation.defaultFilter", commandOptions.interpretationDefaultFilter, true); clinicalAnalysisStudyConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -341,47 +341,48 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "disorder.id",commandOptions.disorderId, true); - putNestedIfNotEmpty(beanParams, "proband.id",commandOptions.probandId, true); - putNestedIfNotEmpty(beanParams, "family.id",commandOptions.familyId, true); - putNestedIfNotNull(beanParams, "panelLock",commandOptions.panelLock, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "report.title",commandOptions.reportTitle, true); - putNestedIfNotEmpty(beanParams, "report.overview",commandOptions.reportOverview, true); - putNestedIfNotEmpty(beanParams, "report.logo",commandOptions.reportLogo, true); - putNestedIfNotEmpty(beanParams, "report.signedBy",commandOptions.reportSignedBy, true); - putNestedIfNotEmpty(beanParams, "report.signature",commandOptions.reportSignature, true); - putNestedIfNotEmpty(beanParams, "report.date",commandOptions.reportDate, true); - putNestedIfNotEmpty(beanParams, "request.id",commandOptions.requestId, true); - putNestedIfNotEmpty(beanParams, "request.justification",commandOptions.requestJustification, true); - putNestedIfNotEmpty(beanParams, "request.date",commandOptions.requestDate, true); - putNestedIfNotNull(beanParams, "request.attributes",commandOptions.requestAttributes, true); - putNestedIfNotEmpty(beanParams, "responsible.id",commandOptions.responsibleId, true); - putNestedIfNotEmpty(beanParams, "responsible.name",commandOptions.responsibleName, true); - putNestedIfNotEmpty(beanParams, "responsible.email",commandOptions.responsibleEmail, true); - putNestedIfNotEmpty(beanParams, "responsible.organization",commandOptions.responsibleOrganization, true); - putNestedIfNotEmpty(beanParams, "responsible.department",commandOptions.responsibleDepartment, true); - putNestedIfNotEmpty(beanParams, "responsible.address",commandOptions.responsibleAddress, true); - putNestedIfNotEmpty(beanParams, "responsible.city",commandOptions.responsibleCity, true); - putNestedIfNotEmpty(beanParams, "responsible.postcode",commandOptions.responsiblePostcode, true); - putNestedIfNotEmpty(beanParams, "interpretation.description",commandOptions.interpretationDescription, true); - putNestedIfNotEmpty(beanParams, "interpretation.clinicalAnalysisId",commandOptions.interpretationClinicalAnalysisId, true); - putNestedIfNotEmpty(beanParams, "interpretation.creationDate",commandOptions.interpretationCreationDate, true); - putNestedIfNotEmpty(beanParams, "interpretation.modificationDate",commandOptions.interpretationModificationDate, true); - putNestedIfNotNull(beanParams, "interpretation.locked",commandOptions.interpretationLocked, true); - putNestedIfNotNull(beanParams, "interpretation.attributes",commandOptions.interpretationAttributes, true); - putNestedIfNotNull(beanParams, "qualityControl.summary",commandOptions.qualityControlSummary, true); - putNestedIfNotNull(beanParams, "qualityControl.comments",commandOptions.qualityControlComments, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "dueDate",commandOptions.dueDate, true); - putNestedIfNotEmpty(beanParams, "priority.id",commandOptions.priorityId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "disorder.id", commandOptions.disorderId, true); + putNestedIfNotEmpty(beanParams, "proband.id", commandOptions.probandId, true); + putNestedIfNotEmpty(beanParams, "family.id", commandOptions.familyId, true); + putNestedIfNotNull(beanParams, "panelLocked", commandOptions.panelLocked, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "report.title", commandOptions.reportTitle, true); + putNestedIfNotEmpty(beanParams, "report.overview", commandOptions.reportOverview, true); + putNestedIfNotEmpty(beanParams, "report.logo", commandOptions.reportLogo, true); + putNestedIfNotEmpty(beanParams, "report.signedBy", commandOptions.reportSignedBy, true); + putNestedIfNotEmpty(beanParams, "report.signature", commandOptions.reportSignature, true); + putNestedIfNotEmpty(beanParams, "report.date", commandOptions.reportDate, true); + putNestedIfNotEmpty(beanParams, "request.id", commandOptions.requestId, true); + putNestedIfNotEmpty(beanParams, "request.justification", commandOptions.requestJustification, true); + putNestedIfNotEmpty(beanParams, "request.date", commandOptions.requestDate, true); + putNestedMapIfNotEmpty(beanParams, "request.attributes", commandOptions.requestAttributes, true); + putNestedIfNotEmpty(beanParams, "responsible.id", commandOptions.responsibleId, true); + putNestedIfNotEmpty(beanParams, "responsible.name", commandOptions.responsibleName, true); + putNestedIfNotEmpty(beanParams, "responsible.email", commandOptions.responsibleEmail, true); + putNestedIfNotEmpty(beanParams, "responsible.organization", commandOptions.responsibleOrganization, true); + putNestedIfNotEmpty(beanParams, "responsible.department", commandOptions.responsibleDepartment, true); + putNestedIfNotEmpty(beanParams, "responsible.address", commandOptions.responsibleAddress, true); + putNestedIfNotEmpty(beanParams, "responsible.city", commandOptions.responsibleCity, true); + putNestedIfNotEmpty(beanParams, "responsible.postcode", commandOptions.responsiblePostcode, true); + putNestedIfNotEmpty(beanParams, "interpretation.name", commandOptions.interpretationName, true); + putNestedIfNotEmpty(beanParams, "interpretation.description", commandOptions.interpretationDescription, true); + putNestedIfNotEmpty(beanParams, "interpretation.clinicalAnalysisId", commandOptions.interpretationClinicalAnalysisId, true); + putNestedIfNotEmpty(beanParams, "interpretation.creationDate", commandOptions.interpretationCreationDate, true); + putNestedIfNotEmpty(beanParams, "interpretation.modificationDate", commandOptions.interpretationModificationDate, true); + putNestedIfNotNull(beanParams, "interpretation.locked", commandOptions.interpretationLocked, true); + putNestedMapIfNotEmpty(beanParams, "interpretation.attributes", commandOptions.interpretationAttributes, true); + putNestedIfNotNull(beanParams, "qualityControl.summary", commandOptions.qualityControlSummary, true); + putNestedIfNotNull(beanParams, "qualityControl.comments", commandOptions.qualityControlComments, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "dueDate", commandOptions.dueDate, true); + putNestedIfNotEmpty(beanParams, "priority.id", commandOptions.priorityId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); clinicalAnalysisCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -419,6 +420,7 @@ private RestResponse distinct() throws Exception { queryParams.putIfNotEmpty("dueDate", commandOptions.dueDate); queryParams.putIfNotEmpty("qualityControlSummary", commandOptions.qualityControlSummary); queryParams.putIfNotEmpty("release", commandOptions.release); + queryParams.putIfNotNull("snapshot", commandOptions.snapshot); queryParams.putIfNotEmpty("status", commandOptions.status); queryParams.putIfNotEmpty("internalStatus", commandOptions.internalStatus); queryParams.putIfNotEmpty("annotation", commandOptions.annotation); @@ -439,6 +441,7 @@ private RestResponse distinctInterpretation() throws Exception { queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotEmpty("clinicalAnalysisId", commandOptions.clinicalAnalysisId); queryParams.putIfNotEmpty("analystId", commandOptions.analystId); queryParams.putIfNotEmpty("methodName", commandOptions.methodName); @@ -471,6 +474,7 @@ private RestResponse searchInterpretation() throws Exception { queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotEmpty("clinicalAnalysisId", commandOptions.clinicalAnalysisId); queryParams.putIfNotEmpty("analystId", commandOptions.analystId); queryParams.putIfNotEmpty("methodName", commandOptions.methodName); @@ -537,9 +541,9 @@ private RestResponse runInterpreterCancerTiering() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CancerTieringInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "discardedVariants",commandOptions.discardedVariants, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "discardedVariants", commandOptions.discardedVariants, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); cancerTieringInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -578,7 +582,8 @@ private RestResponse runInterpreterExomiser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ExomiserInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion", commandOptions.exomiserVersion, true); exomiserInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -617,10 +622,10 @@ private RestResponse runInterpreterTeam() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TeamInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "panels",commandOptions.panels, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "panels", commandOptions.panels, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); teamInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -659,10 +664,10 @@ private RestResponse runInterpreterTiering() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TieringInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "panels",commandOptions.panels, true); - putNestedIfNotNull(beanParams, "penetrance",commandOptions.penetrance, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "panels", commandOptions.panels, true); + putNestedIfNotNull(beanParams, "penetrance", commandOptions.penetrance, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); tieringInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -701,57 +706,57 @@ private RestResponse runInterpreterZetta() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ZettaInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotEmpty(beanParams, "fileData",commandOptions.fileData, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "sampleData",commandOptions.sampleData, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "sampleMetadata",commandOptions.sampleMetadata, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMgf",commandOptions.cohortStatsMgf, true); - putNestedIfNotEmpty(beanParams, "cohortStatsPass",commandOptions.cohortStatsPass, true); - putNestedIfNotEmpty(beanParams, "score",commandOptions.score, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "familyDisorder",commandOptions.familyDisorder, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotEmpty(beanParams, "familyMembers",commandOptions.familyMembers, true); - putNestedIfNotEmpty(beanParams, "familyProband",commandOptions.familyProband, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "geneTraitId",commandOptions.geneTraitId, true); - putNestedIfNotEmpty(beanParams, "go",commandOptions.go, true); - putNestedIfNotEmpty(beanParams, "expression",commandOptions.expression, true); - putNestedIfNotEmpty(beanParams, "proteinKeyword",commandOptions.proteinKeyword, true); - putNestedIfNotEmpty(beanParams, "drug",commandOptions.drug, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "customAnnotation",commandOptions.customAnnotation, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotEmpty(beanParams, "trait",commandOptions.trait, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotEmpty(beanParams, "fileData", commandOptions.fileData, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "sampleData", commandOptions.sampleData, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "sampleMetadata", commandOptions.sampleMetadata, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMgf", commandOptions.cohortStatsMgf, true); + putNestedIfNotEmpty(beanParams, "cohortStatsPass", commandOptions.cohortStatsPass, true); + putNestedIfNotEmpty(beanParams, "score", commandOptions.score, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "familyDisorder", commandOptions.familyDisorder, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotEmpty(beanParams, "familyMembers", commandOptions.familyMembers, true); + putNestedIfNotEmpty(beanParams, "familyProband", commandOptions.familyProband, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "geneTraitId", commandOptions.geneTraitId, true); + putNestedIfNotEmpty(beanParams, "go", commandOptions.go, true); + putNestedIfNotEmpty(beanParams, "expression", commandOptions.expression, true); + putNestedIfNotEmpty(beanParams, "proteinKeyword", commandOptions.proteinKeyword, true); + putNestedIfNotEmpty(beanParams, "drug", commandOptions.drug, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "customAnnotation", commandOptions.customAnnotation, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotEmpty(beanParams, "trait", commandOptions.trait, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); zettaInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -790,7 +795,7 @@ private RestResponse load() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisLoadParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); clinicalAnalysisLoadParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -945,7 +950,7 @@ private RestResponse runRgaIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RgaAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); rgaAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1144,6 +1149,7 @@ private RestResponse search() throws Exception { queryParams.putIfNotEmpty("dueDate", commandOptions.dueDate); queryParams.putIfNotEmpty("qualityControlSummary", commandOptions.qualityControlSummary); queryParams.putIfNotEmpty("release", commandOptions.release); + queryParams.putIfNotNull("snapshot", commandOptions.snapshot); queryParams.putIfNotEmpty("status", commandOptions.status); queryParams.putIfNotEmpty("internalStatus", commandOptions.internalStatus); queryParams.putIfNotEmpty("annotation", commandOptions.annotation); @@ -1288,42 +1294,43 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "disorder.id",commandOptions.disorderId, true); - putNestedIfNotNull(beanParams, "panelLock",commandOptions.panelLock, true); - putNestedIfNotEmpty(beanParams, "proband.id",commandOptions.probandId, true); - putNestedIfNotEmpty(beanParams, "family.id",commandOptions.familyId, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "report.title",commandOptions.reportTitle, true); - putNestedIfNotEmpty(beanParams, "report.overview",commandOptions.reportOverview, true); - putNestedIfNotEmpty(beanParams, "report.logo",commandOptions.reportLogo, true); - putNestedIfNotEmpty(beanParams, "report.signedBy",commandOptions.reportSignedBy, true); - putNestedIfNotEmpty(beanParams, "report.signature",commandOptions.reportSignature, true); - putNestedIfNotEmpty(beanParams, "report.date",commandOptions.reportDate, true); - putNestedIfNotEmpty(beanParams, "request.id",commandOptions.requestId, true); - putNestedIfNotEmpty(beanParams, "request.justification",commandOptions.requestJustification, true); - putNestedIfNotEmpty(beanParams, "request.date",commandOptions.requestDate, true); - putNestedIfNotNull(beanParams, "request.attributes",commandOptions.requestAttributes, true); - putNestedIfNotEmpty(beanParams, "responsible.id",commandOptions.responsibleId, true); - putNestedIfNotEmpty(beanParams, "responsible.name",commandOptions.responsibleName, true); - putNestedIfNotEmpty(beanParams, "responsible.email",commandOptions.responsibleEmail, true); - putNestedIfNotEmpty(beanParams, "responsible.organization",commandOptions.responsibleOrganization, true); - putNestedIfNotEmpty(beanParams, "responsible.department",commandOptions.responsibleDepartment, true); - putNestedIfNotEmpty(beanParams, "responsible.address",commandOptions.responsibleAddress, true); - putNestedIfNotEmpty(beanParams, "responsible.city",commandOptions.responsibleCity, true); - putNestedIfNotEmpty(beanParams, "responsible.postcode",commandOptions.responsiblePostcode, true); - putNestedIfNotNull(beanParams, "qualityControl.summary",commandOptions.qualityControlSummary, true); - putNestedIfNotNull(beanParams, "qualityControl.comments",commandOptions.qualityControlComments, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "dueDate",commandOptions.dueDate, true); - putNestedIfNotEmpty(beanParams, "priority.id",commandOptions.priorityId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "disorder.id", commandOptions.disorderId, true); + putNestedIfNotNull(beanParams, "panelLocked", commandOptions.panelLocked, true); + putNestedIfNotEmpty(beanParams, "proband.id", commandOptions.probandId, true); + putNestedIfNotEmpty(beanParams, "family.id", commandOptions.familyId, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "report.title", commandOptions.reportTitle, true); + putNestedIfNotEmpty(beanParams, "report.overview", commandOptions.reportOverview, true); + putNestedIfNotEmpty(beanParams, "report.logo", commandOptions.reportLogo, true); + putNestedIfNotEmpty(beanParams, "report.signedBy", commandOptions.reportSignedBy, true); + putNestedIfNotEmpty(beanParams, "report.signature", commandOptions.reportSignature, true); + putNestedIfNotEmpty(beanParams, "report.date", commandOptions.reportDate, true); + putNestedIfNotEmpty(beanParams, "request.id", commandOptions.requestId, true); + putNestedIfNotEmpty(beanParams, "request.justification", commandOptions.requestJustification, true); + putNestedIfNotEmpty(beanParams, "request.date", commandOptions.requestDate, true); + putNestedMapIfNotEmpty(beanParams, "request.attributes", commandOptions.requestAttributes, true); + putNestedIfNotEmpty(beanParams, "responsible.id", commandOptions.responsibleId, true); + putNestedIfNotEmpty(beanParams, "responsible.name", commandOptions.responsibleName, true); + putNestedIfNotEmpty(beanParams, "responsible.email", commandOptions.responsibleEmail, true); + putNestedIfNotEmpty(beanParams, "responsible.organization", commandOptions.responsibleOrganization, true); + putNestedIfNotEmpty(beanParams, "responsible.department", commandOptions.responsibleDepartment, true); + putNestedIfNotEmpty(beanParams, "responsible.address", commandOptions.responsibleAddress, true); + putNestedIfNotEmpty(beanParams, "responsible.city", commandOptions.responsibleCity, true); + putNestedIfNotEmpty(beanParams, "responsible.postcode", commandOptions.responsiblePostcode, true); + putNestedIfNotNull(beanParams, "qualityControl.summary", commandOptions.qualityControlSummary, true); + putNestedIfNotNull(beanParams, "qualityControl.comments", commandOptions.qualityControlComments, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "dueDate", commandOptions.dueDate, true); + putNestedIfNotEmpty(beanParams, "priority.id", commandOptions.priorityId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotNull(beanParams, "panelLock", commandOptions.panelLock, true); clinicalAnalysisUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1368,6 +1375,7 @@ private RestResponse info() throws Exception { queryParams.putIfNotEmpty("exclude", commandOptions.exclude); queryParams.putIfNotNull("flattenAnnotations", commandOptions.flattenAnnotations); queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("version", commandOptions.version); queryParams.putIfNotNull("deleted", commandOptions.deleted); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -1403,17 +1411,18 @@ private RestResponse createInterpretation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InterpretationCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysisId",commandOptions.clinicalAnalysisId, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "method.name",commandOptions.methodName, true); - putNestedIfNotEmpty(beanParams, "method.version",commandOptions.methodVersion, true); - putNestedIfNotEmpty(beanParams, "method.commit",commandOptions.methodCommit, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysisId", commandOptions.clinicalAnalysisId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "method.name", commandOptions.methodName, true); + putNestedIfNotEmpty(beanParams, "method.version", commandOptions.methodVersion, true); + putNestedIfNotEmpty(beanParams, "method.commit", commandOptions.methodCommit, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); interpretationCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1492,16 +1501,17 @@ private RestResponse updateInterpretation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InterpretationUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "method.name",commandOptions.methodName, true); - putNestedIfNotEmpty(beanParams, "method.version",commandOptions.methodVersion, true); - putNestedIfNotEmpty(beanParams, "method.commit",commandOptions.methodCommit, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "method.name", commandOptions.methodName, true); + putNestedIfNotEmpty(beanParams, "method.version", commandOptions.methodVersion, true); + putNestedIfNotEmpty(beanParams, "method.commit", commandOptions.methodCommit, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); interpretationUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1537,15 +1547,15 @@ private RestResponse updateReport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalReport.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "title",commandOptions.title, true); - putNestedIfNotEmpty(beanParams, "overview",commandOptions.overview, true); - putNestedIfNotEmpty(beanParams, "discussion.author",commandOptions.discussionAuthor, true); - putNestedIfNotEmpty(beanParams, "discussion.date",commandOptions.discussionDate, true); - putNestedIfNotEmpty(beanParams, "discussion.text",commandOptions.discussionText, true); - putNestedIfNotEmpty(beanParams, "logo",commandOptions.logo, true); - putNestedIfNotEmpty(beanParams, "signedBy",commandOptions.signedBy, true); - putNestedIfNotEmpty(beanParams, "signature",commandOptions.signature, true); - putNestedIfNotEmpty(beanParams, "date",commandOptions.date, true); + putNestedIfNotEmpty(beanParams, "title", commandOptions.title, true); + putNestedIfNotEmpty(beanParams, "overview", commandOptions.overview, true); + putNestedIfNotEmpty(beanParams, "discussion.author", commandOptions.discussionAuthor, true); + putNestedIfNotEmpty(beanParams, "discussion.date", commandOptions.discussionDate, true); + putNestedIfNotEmpty(beanParams, "discussion.text", commandOptions.discussionText, true); + putNestedIfNotEmpty(beanParams, "logo", commandOptions.logo, true); + putNestedIfNotEmpty(beanParams, "signedBy", commandOptions.signedBy, true); + putNestedIfNotEmpty(beanParams, "signature", commandOptions.signature, true); + putNestedIfNotEmpty(beanParams, "date", commandOptions.date, true); clinicalReport = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 3427a69163c..57edea737fa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -318,10 +318,10 @@ private RestResponse runCircos() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CircosAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "title",commandOptions.title, true); - putNestedIfNotEmpty(beanParams, "density",commandOptions.density, true); - putNestedIfNotNull(beanParams, "query",commandOptions.query, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "title", commandOptions.title, true); + putNestedIfNotEmpty(beanParams, "density", commandOptions.density, true); + putNestedMapIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); circosAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -389,11 +389,11 @@ private RestResponse runCohortStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortVariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); cohortVariantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -432,9 +432,10 @@ private RestResponse runExomiser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ExomiserWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysisType",commandOptions.clinicalAnalysisType, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion", commandOptions.exomiserVersion, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysisType", commandOptions.clinicalAnalysisType, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); exomiserWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -476,93 +477,93 @@ private RestResponse runExport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantExportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotNull(beanParams, "panelIntersection",commandOptions.panelIntersection, true); - putNestedIfNotEmpty(beanParams, "panelFeatureType",commandOptions.panelFeatureType, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "project",commandOptions.bodyProject, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotEmpty(beanParams, "savedFilter",commandOptions.savedFilter, true); - putNestedIfNotEmpty(beanParams, "chromosome",commandOptions.chromosome, true); - putNestedIfNotEmpty(beanParams, "reference",commandOptions.reference, true); - putNestedIfNotEmpty(beanParams, "alternate",commandOptions.alternate, true); - putNestedIfNotEmpty(beanParams, "release",commandOptions.release, true); - putNestedIfNotEmpty(beanParams, "includeStudy",commandOptions.includeStudy, true); - putNestedIfNotEmpty(beanParams, "includeSample",commandOptions.includeSample, true); - putNestedIfNotEmpty(beanParams, "includeFile",commandOptions.includeFile, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotNull(beanParams, "includeSampleId",commandOptions.includeSampleId, true); - putNestedIfNotNull(beanParams, "includeGenotype",commandOptions.includeGenotype, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "fileData",commandOptions.fileData, true); - putNestedIfNotEmpty(beanParams, "genotype",commandOptions.genotype, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "sampleLimit",commandOptions.sampleLimit, true); - putNestedIfNotNull(beanParams, "sampleSkip",commandOptions.sampleSkip, true); - putNestedIfNotEmpty(beanParams, "sampleData",commandOptions.sampleData, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "familyMembers",commandOptions.familyMembers, true); - putNestedIfNotEmpty(beanParams, "familyDisorder",commandOptions.familyDisorder, true); - putNestedIfNotEmpty(beanParams, "familyProband",commandOptions.familyProband, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "cohortStatsPass",commandOptions.cohortStatsPass, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMgf",commandOptions.cohortStatsMgf, true); - putNestedIfNotEmpty(beanParams, "missingAlleles",commandOptions.missingAlleles, true); - putNestedIfNotEmpty(beanParams, "missingGenotypes",commandOptions.missingGenotypes, true); - putNestedIfNotNull(beanParams, "annotationExists",commandOptions.annotationExists, true); - putNestedIfNotEmpty(beanParams, "score",commandOptions.score, true); - putNestedIfNotEmpty(beanParams, "polyphen",commandOptions.polyphen, true); - putNestedIfNotEmpty(beanParams, "sift",commandOptions.sift, true); - putNestedIfNotEmpty(beanParams, "geneRoleInCancer",commandOptions.geneRoleInCancer, true); - putNestedIfNotEmpty(beanParams, "geneTraitId",commandOptions.geneTraitId, true); - putNestedIfNotEmpty(beanParams, "geneTraitName",commandOptions.geneTraitName, true); - putNestedIfNotEmpty(beanParams, "trait",commandOptions.trait, true); - putNestedIfNotEmpty(beanParams, "cosmic",commandOptions.cosmic, true); - putNestedIfNotEmpty(beanParams, "clinvar",commandOptions.clinvar, true); - putNestedIfNotEmpty(beanParams, "hpo",commandOptions.hpo, true); - putNestedIfNotEmpty(beanParams, "go",commandOptions.go, true); - putNestedIfNotEmpty(beanParams, "expression",commandOptions.expression, true); - putNestedIfNotEmpty(beanParams, "proteinKeyword",commandOptions.proteinKeyword, true); - putNestedIfNotEmpty(beanParams, "drug",commandOptions.drug, true); - putNestedIfNotEmpty(beanParams, "customAnnotation",commandOptions.customAnnotation, true); - putNestedIfNotEmpty(beanParams, "unknownGenotype",commandOptions.unknownGenotype, true); - putNestedIfNotNull(beanParams, "sampleMetadata",commandOptions.sampleMetadata, true); - putNestedIfNotNull(beanParams, "sort",commandOptions.sort, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotEmpty(beanParams, "outputFileFormat",commandOptions.outputFileFormat, true); - putNestedIfNotEmpty(beanParams, "variantsFile",commandOptions.variantsFile, true); - putNestedIfNotEmpty(beanParams, "include",commandOptions.bodyInclude, true); - putNestedIfNotEmpty(beanParams, "exclude",commandOptions.bodyExclude, true); - putNestedIfNotNull(beanParams, "limit",commandOptions.limit, true); - putNestedIfNotNull(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotNull(beanParams, "summary",commandOptions.summary, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotNull(beanParams, "panelIntersection", commandOptions.panelIntersection, true); + putNestedIfNotEmpty(beanParams, "panelFeatureType", commandOptions.panelFeatureType, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "project", commandOptions.bodyProject, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotEmpty(beanParams, "savedFilter", commandOptions.savedFilter, true); + putNestedIfNotEmpty(beanParams, "chromosome", commandOptions.chromosome, true); + putNestedIfNotEmpty(beanParams, "reference", commandOptions.reference, true); + putNestedIfNotEmpty(beanParams, "alternate", commandOptions.alternate, true); + putNestedIfNotEmpty(beanParams, "release", commandOptions.release, true); + putNestedIfNotEmpty(beanParams, "includeStudy", commandOptions.includeStudy, true); + putNestedIfNotEmpty(beanParams, "includeSample", commandOptions.includeSample, true); + putNestedIfNotEmpty(beanParams, "includeFile", commandOptions.includeFile, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotNull(beanParams, "includeSampleId", commandOptions.includeSampleId, true); + putNestedIfNotNull(beanParams, "includeGenotype", commandOptions.includeGenotype, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "fileData", commandOptions.fileData, true); + putNestedIfNotEmpty(beanParams, "genotype", commandOptions.genotype, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "sampleLimit", commandOptions.sampleLimit, true); + putNestedIfNotNull(beanParams, "sampleSkip", commandOptions.sampleSkip, true); + putNestedIfNotEmpty(beanParams, "sampleData", commandOptions.sampleData, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "familyMembers", commandOptions.familyMembers, true); + putNestedIfNotEmpty(beanParams, "familyDisorder", commandOptions.familyDisorder, true); + putNestedIfNotEmpty(beanParams, "familyProband", commandOptions.familyProband, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "cohortStatsPass", commandOptions.cohortStatsPass, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMgf", commandOptions.cohortStatsMgf, true); + putNestedIfNotEmpty(beanParams, "missingAlleles", commandOptions.missingAlleles, true); + putNestedIfNotEmpty(beanParams, "missingGenotypes", commandOptions.missingGenotypes, true); + putNestedIfNotNull(beanParams, "annotationExists", commandOptions.annotationExists, true); + putNestedIfNotEmpty(beanParams, "score", commandOptions.score, true); + putNestedIfNotEmpty(beanParams, "polyphen", commandOptions.polyphen, true); + putNestedIfNotEmpty(beanParams, "sift", commandOptions.sift, true); + putNestedIfNotEmpty(beanParams, "geneRoleInCancer", commandOptions.geneRoleInCancer, true); + putNestedIfNotEmpty(beanParams, "geneTraitId", commandOptions.geneTraitId, true); + putNestedIfNotEmpty(beanParams, "geneTraitName", commandOptions.geneTraitName, true); + putNestedIfNotEmpty(beanParams, "trait", commandOptions.trait, true); + putNestedIfNotEmpty(beanParams, "cosmic", commandOptions.cosmic, true); + putNestedIfNotEmpty(beanParams, "clinvar", commandOptions.clinvar, true); + putNestedIfNotEmpty(beanParams, "hpo", commandOptions.hpo, true); + putNestedIfNotEmpty(beanParams, "go", commandOptions.go, true); + putNestedIfNotEmpty(beanParams, "expression", commandOptions.expression, true); + putNestedIfNotEmpty(beanParams, "proteinKeyword", commandOptions.proteinKeyword, true); + putNestedIfNotEmpty(beanParams, "drug", commandOptions.drug, true); + putNestedIfNotEmpty(beanParams, "customAnnotation", commandOptions.customAnnotation, true); + putNestedIfNotEmpty(beanParams, "unknownGenotype", commandOptions.unknownGenotype, true); + putNestedIfNotNull(beanParams, "sampleMetadata", commandOptions.sampleMetadata, true); + putNestedIfNotNull(beanParams, "sort", commandOptions.sort, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotEmpty(beanParams, "outputFileFormat", commandOptions.outputFileFormat, true); + putNestedIfNotEmpty(beanParams, "variantsFile", commandOptions.variantsFile, true); + putNestedIfNotEmpty(beanParams, "include", commandOptions.bodyInclude, true); + putNestedIfNotEmpty(beanParams, "exclude", commandOptions.bodyExclude, true); + putNestedIfNotNull(beanParams, "limit", commandOptions.limit, true); + putNestedIfNotNull(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotNull(beanParams, "summary", commandOptions.summary, true); variantExportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -619,10 +620,10 @@ private RestResponse runFamilyQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "relatednessMethod",commandOptions.relatednessMethod, true); - putNestedIfNotEmpty(beanParams, "relatednessMaf",commandOptions.relatednessMaf, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "relatednessMethod", commandOptions.relatednessMethod, true); + putNestedIfNotEmpty(beanParams, "relatednessMaf", commandOptions.relatednessMaf, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); familyQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -684,9 +685,9 @@ private RestResponse runGatk() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GatkWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "gatkParams",commandOptions.gatkParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "gatkParams", commandOptions.gatkParams, true); gatkWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -725,11 +726,11 @@ private RestResponse runGenomePlot() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GenomePlotAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "configFile",commandOptions.configFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "configFile", commandOptions.configFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); genomePlotAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -768,18 +769,18 @@ private RestResponse runGwas() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GwasAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "phenotype",commandOptions.phenotype, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "indexScoreId",commandOptions.indexScoreId, true); - putNestedIfNotNull(beanParams, "method",commandOptions.method, true); - putNestedIfNotNull(beanParams, "fisherMode",commandOptions.fisherMode, true); - putNestedIfNotEmpty(beanParams, "caseCohort",commandOptions.caseCohort, true); - putNestedIfNotEmpty(beanParams, "caseCohortSamplesAnnotation",commandOptions.caseCohortSamplesAnnotation, true); - putNestedIfNotNull(beanParams, "caseCohortSamples",commandOptions.caseCohortSamples, true); - putNestedIfNotEmpty(beanParams, "controlCohort",commandOptions.controlCohort, true); - putNestedIfNotEmpty(beanParams, "controlCohortSamplesAnnotation",commandOptions.controlCohortSamplesAnnotation, true); - putNestedIfNotNull(beanParams, "controlCohortSamples",commandOptions.controlCohortSamples, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "phenotype", commandOptions.phenotype, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "indexScoreId", commandOptions.indexScoreId, true); + putNestedIfNotNull(beanParams, "method", commandOptions.method, true); + putNestedIfNotNull(beanParams, "fisherMode", commandOptions.fisherMode, true); + putNestedIfNotEmpty(beanParams, "caseCohort", commandOptions.caseCohort, true); + putNestedIfNotEmpty(beanParams, "caseCohortSamplesAnnotation", commandOptions.caseCohortSamplesAnnotation, true); + putNestedIfNotNull(beanParams, "caseCohortSamples", commandOptions.caseCohortSamples, true); + putNestedIfNotEmpty(beanParams, "controlCohort", commandOptions.controlCohort, true); + putNestedIfNotEmpty(beanParams, "controlCohortSamplesAnnotation", commandOptions.controlCohortSamplesAnnotation, true); + putNestedIfNotNull(beanParams, "controlCohortSamples", commandOptions.controlCohortSamples, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); gwasAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -818,19 +819,19 @@ private RestResponse runHrDetect() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), HRDetectAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "sampleId",commandOptions.sampleId, true); - putNestedIfNotEmpty(beanParams, "snvFittingId",commandOptions.snvFittingId, true); - putNestedIfNotEmpty(beanParams, "svFittingId",commandOptions.svFittingId, true); - putNestedIfNotEmpty(beanParams, "cnvQuery",commandOptions.cnvQuery, true); - putNestedIfNotEmpty(beanParams, "indelQuery",commandOptions.indelQuery, true); - putNestedIfNotEmpty(beanParams, "snv3CustomName",commandOptions.snv3CustomName, true); - putNestedIfNotEmpty(beanParams, "snv8CustomName",commandOptions.snv8CustomName, true); - putNestedIfNotEmpty(beanParams, "sv3CustomName",commandOptions.sv3CustomName, true); - putNestedIfNotEmpty(beanParams, "sv8CustomName",commandOptions.sv8CustomName, true); - putNestedIfNotNull(beanParams, "bootstrap",commandOptions.bootstrap, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sampleId", commandOptions.sampleId, true); + putNestedIfNotEmpty(beanParams, "snvFittingId", commandOptions.snvFittingId, true); + putNestedIfNotEmpty(beanParams, "svFittingId", commandOptions.svFittingId, true); + putNestedIfNotEmpty(beanParams, "cnvQuery", commandOptions.cnvQuery, true); + putNestedIfNotEmpty(beanParams, "indelQuery", commandOptions.indelQuery, true); + putNestedIfNotEmpty(beanParams, "snv3CustomName", commandOptions.snv3CustomName, true); + putNestedIfNotEmpty(beanParams, "snv8CustomName", commandOptions.snv8CustomName, true); + putNestedIfNotEmpty(beanParams, "sv3CustomName", commandOptions.sv3CustomName, true); + putNestedIfNotEmpty(beanParams, "sv8CustomName", commandOptions.sv8CustomName, true); + putNestedIfNotNull(beanParams, "bootstrap", commandOptions.bootstrap, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); hRDetectAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -869,36 +870,36 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "transform",commandOptions.transform, true); - putNestedIfNotNull(beanParams, "gvcf",commandOptions.gvcf, true); - putNestedIfNotNull(beanParams, "normalizationSkip",commandOptions.normalizationSkip, true); - putNestedIfNotEmpty(beanParams, "referenceGenome",commandOptions.referenceGenome, true); - putNestedIfNotEmpty(beanParams, "failOnMalformedLines",commandOptions.failOnMalformedLines, true); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "load",commandOptions.load, true); - putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); - putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); - putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "loadArchive",commandOptions.loadArchive, true); - putNestedIfNotEmpty(beanParams, "loadHomRef",commandOptions.loadHomRef, true); - putNestedIfNotEmpty(beanParams, "postLoadCheck",commandOptions.postLoadCheck, true); - putNestedIfNotEmpty(beanParams, "includeGenotypes",commandOptions.includeGenotypes, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotEmpty(beanParams, "merge",commandOptions.merge, true); - putNestedIfNotEmpty(beanParams, "deduplicationPolicy",commandOptions.deduplicationPolicy, true); - putNestedIfNotNull(beanParams, "calculateStats",commandOptions.calculateStats, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexSearch",commandOptions.indexSearch, true); - putNestedIfNotNull(beanParams, "skipIndexedFiles",commandOptions.skipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "transform", commandOptions.transform, true); + putNestedIfNotNull(beanParams, "gvcf", commandOptions.gvcf, true); + putNestedIfNotNull(beanParams, "normalizationSkip", commandOptions.normalizationSkip, true); + putNestedIfNotEmpty(beanParams, "referenceGenome", commandOptions.referenceGenome, true); + putNestedIfNotEmpty(beanParams, "failOnMalformedLines", commandOptions.failOnMalformedLines, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotNull(beanParams, "load", commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload", commandOptions.forceReload, true); + putNestedIfNotEmpty(beanParams, "loadSplitData", commandOptions.loadSplitData, true); + putNestedIfNotNull(beanParams, "loadMultiFileData", commandOptions.loadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "loadSampleIndex", commandOptions.loadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "loadArchive", commandOptions.loadArchive, true); + putNestedIfNotEmpty(beanParams, "loadHomRef", commandOptions.loadHomRef, true); + putNestedIfNotEmpty(beanParams, "postLoadCheck", commandOptions.postLoadCheck, true); + putNestedIfNotEmpty(beanParams, "includeGenotypes", commandOptions.includeGenotypes, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotEmpty(beanParams, "merge", commandOptions.merge, true); + putNestedIfNotEmpty(beanParams, "deduplicationPolicy", commandOptions.deduplicationPolicy, true); + putNestedIfNotNull(beanParams, "calculateStats", commandOptions.calculateStats, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexSearch", commandOptions.indexSearch, true); + putNestedIfNotNull(beanParams, "skipIndexedFiles", commandOptions.skipIndexedFiles, true); variantIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -937,10 +938,10 @@ private RestResponse runIndividualQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "inferredSexMethod",commandOptions.inferredSexMethod, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "inferredSexMethod", commandOptions.inferredSexMethod, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); individualQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -979,9 +980,9 @@ private RestResponse runInferredSex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InferredSexAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); inferredSexAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1054,16 +1055,16 @@ private RestResponse runKnockout() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), KnockoutAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotNull(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "consequenceType",commandOptions.consequenceType, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotNull(beanParams, "skipGenesFile",commandOptions.skipGenesFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotNull(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "consequenceType", commandOptions.consequenceType, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotNull(beanParams, "skipGenesFile", commandOptions.skipGenesFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); knockoutAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1102,10 +1103,10 @@ private RestResponse runMendelianError() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), MendelianErrorAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); mendelianErrorAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1197,22 +1198,22 @@ private RestResponse runMutationalSignature() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), MutationalSignatureAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "query",commandOptions.query, true); - putNestedIfNotEmpty(beanParams, "fitId",commandOptions.fitId, true); - putNestedIfNotEmpty(beanParams, "fitMethod",commandOptions.fitMethod, true); - putNestedIfNotNull(beanParams, "fitNBoot",commandOptions.fitNBoot, true); - putNestedIfNotEmpty(beanParams, "fitSigVersion",commandOptions.fitSigVersion, true); - putNestedIfNotEmpty(beanParams, "fitOrgan",commandOptions.fitOrgan, true); - putNestedIfNotNull(beanParams, "fitThresholdPerc",commandOptions.fitThresholdPerc, true); - putNestedIfNotNull(beanParams, "fitThresholdPval",commandOptions.fitThresholdPval, true); - putNestedIfNotNull(beanParams, "fitMaxRareSigs",commandOptions.fitMaxRareSigs, true); - putNestedIfNotEmpty(beanParams, "fitSignaturesFile",commandOptions.fitSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "fitRareSignaturesFile",commandOptions.fitRareSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotEmpty(beanParams, "fitId", commandOptions.fitId, true); + putNestedIfNotEmpty(beanParams, "fitMethod", commandOptions.fitMethod, true); + putNestedIfNotNull(beanParams, "fitNBoot", commandOptions.fitNBoot, true); + putNestedIfNotEmpty(beanParams, "fitSigVersion", commandOptions.fitSigVersion, true); + putNestedIfNotEmpty(beanParams, "fitOrgan", commandOptions.fitOrgan, true); + putNestedIfNotNull(beanParams, "fitThresholdPerc", commandOptions.fitThresholdPerc, true); + putNestedIfNotNull(beanParams, "fitThresholdPval", commandOptions.fitThresholdPval, true); + putNestedIfNotNull(beanParams, "fitMaxRareSigs", commandOptions.fitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "fitSignaturesFile", commandOptions.fitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "fitRareSignaturesFile", commandOptions.fitRareSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); mutationalSignatureAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1251,8 +1252,8 @@ private RestResponse runPlink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PlinkWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "plinkParams",commandOptions.plinkParams, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "plinkParams", commandOptions.plinkParams, true); plinkWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1381,11 +1382,11 @@ private RestResponse runRelatedness() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RelatednessAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "individuals",commandOptions.individuals, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "minorAlleleFreq",commandOptions.minorAlleleFreq, true); - putNestedIfNotEmpty(beanParams, "method",commandOptions.method, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "individuals", commandOptions.individuals, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "minorAlleleFreq", commandOptions.minorAlleleFreq, true); + putNestedIfNotEmpty(beanParams, "method", commandOptions.method, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); relatednessAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1424,9 +1425,9 @@ private RestResponse runRvtests() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RvtestsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "rvtestsParams",commandOptions.rvtestsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "rvtestsParams", commandOptions.rvtestsParams, true); rvtestsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1500,9 +1501,9 @@ private RestResponse runSampleEligibility() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleEligibilityAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "query",commandOptions.query, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "cohortId",commandOptions.cohortId, true); + putNestedIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "cohortId", commandOptions.cohortId, true); sampleEligibilityAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1541,53 +1542,53 @@ private RestResponse runSampleQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "vsId",commandOptions.vsId, true); - putNestedIfNotEmpty(beanParams, "vsDescription",commandOptions.vsDescription, true); - putNestedIfNotEmpty(beanParams, "vsQuery.id",commandOptions.vsQueryId, true); - putNestedIfNotEmpty(beanParams, "vsQuery.region",commandOptions.vsQueryRegion, true); - putNestedIfNotEmpty(beanParams, "vsQuery.gene",commandOptions.vsQueryGene, true); - putNestedIfNotEmpty(beanParams, "vsQuery.type",commandOptions.vsQueryType, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panel",commandOptions.vsQueryPanel, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelModeOfInheritance",commandOptions.vsQueryPanelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelConfidence",commandOptions.vsQueryPanelConfidence, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelRoleInCancer",commandOptions.vsQueryPanelRoleInCancer, true); - putNestedIfNotNull(beanParams, "vsQuery.panelIntersection",commandOptions.vsQueryPanelIntersection, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelFeatureType",commandOptions.vsQueryPanelFeatureType, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsRef",commandOptions.vsQueryCohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsAlt",commandOptions.vsQueryCohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsMaf",commandOptions.vsQueryCohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "vsQuery.ct",commandOptions.vsQueryCt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.xref",commandOptions.vsQueryXref, true); - putNestedIfNotEmpty(beanParams, "vsQuery.biotype",commandOptions.vsQueryBiotype, true); - putNestedIfNotEmpty(beanParams, "vsQuery.proteinSubstitution",commandOptions.vsQueryProteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "vsQuery.conservation",commandOptions.vsQueryConservation, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyMaf",commandOptions.vsQueryPopulationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyAlt",commandOptions.vsQueryPopulationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyRef",commandOptions.vsQueryPopulationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "vsQuery.transcriptFlag",commandOptions.vsQueryTranscriptFlag, true); - putNestedIfNotEmpty(beanParams, "vsQuery.functionalScore",commandOptions.vsQueryFunctionalScore, true); - putNestedIfNotEmpty(beanParams, "vsQuery.clinical",commandOptions.vsQueryClinical, true); - putNestedIfNotEmpty(beanParams, "vsQuery.clinicalSignificance",commandOptions.vsQueryClinicalSignificance, true); - putNestedIfNotNull(beanParams, "vsQuery.clinicalConfirmedStatus",commandOptions.vsQueryClinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "msId",commandOptions.msId, true); - putNestedIfNotEmpty(beanParams, "msDescription",commandOptions.msDescription, true); - putNestedIfNotEmpty(beanParams, "msQuery",commandOptions.msQuery, true); - putNestedIfNotEmpty(beanParams, "msFitId",commandOptions.msFitId, true); - putNestedIfNotEmpty(beanParams, "msFitMethod",commandOptions.msFitMethod, true); - putNestedIfNotNull(beanParams, "msFitNBoot",commandOptions.msFitNBoot, true); - putNestedIfNotEmpty(beanParams, "msFitSigVersion",commandOptions.msFitSigVersion, true); - putNestedIfNotEmpty(beanParams, "msFitOrgan",commandOptions.msFitOrgan, true); - putNestedIfNotNull(beanParams, "msFitThresholdPerc",commandOptions.msFitThresholdPerc, true); - putNestedIfNotNull(beanParams, "msFitThresholdPval",commandOptions.msFitThresholdPval, true); - putNestedIfNotNull(beanParams, "msFitMaxRareSigs",commandOptions.msFitMaxRareSigs, true); - putNestedIfNotEmpty(beanParams, "msFitSignaturesFile",commandOptions.msFitSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "msFitRareSignaturesFile",commandOptions.msFitRareSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "gpId",commandOptions.gpId, true); - putNestedIfNotEmpty(beanParams, "gpDescription",commandOptions.gpDescription, true); - putNestedIfNotEmpty(beanParams, "gpConfigFile",commandOptions.gpConfigFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "vsId", commandOptions.vsId, true); + putNestedIfNotEmpty(beanParams, "vsDescription", commandOptions.vsDescription, true); + putNestedIfNotEmpty(beanParams, "vsQuery.id", commandOptions.vsQueryId, true); + putNestedIfNotEmpty(beanParams, "vsQuery.region", commandOptions.vsQueryRegion, true); + putNestedIfNotEmpty(beanParams, "vsQuery.gene", commandOptions.vsQueryGene, true); + putNestedIfNotEmpty(beanParams, "vsQuery.type", commandOptions.vsQueryType, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panel", commandOptions.vsQueryPanel, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelModeOfInheritance", commandOptions.vsQueryPanelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelConfidence", commandOptions.vsQueryPanelConfidence, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelRoleInCancer", commandOptions.vsQueryPanelRoleInCancer, true); + putNestedIfNotNull(beanParams, "vsQuery.panelIntersection", commandOptions.vsQueryPanelIntersection, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelFeatureType", commandOptions.vsQueryPanelFeatureType, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsRef", commandOptions.vsQueryCohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsAlt", commandOptions.vsQueryCohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsMaf", commandOptions.vsQueryCohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "vsQuery.ct", commandOptions.vsQueryCt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.xref", commandOptions.vsQueryXref, true); + putNestedIfNotEmpty(beanParams, "vsQuery.biotype", commandOptions.vsQueryBiotype, true); + putNestedIfNotEmpty(beanParams, "vsQuery.proteinSubstitution", commandOptions.vsQueryProteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "vsQuery.conservation", commandOptions.vsQueryConservation, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyMaf", commandOptions.vsQueryPopulationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyAlt", commandOptions.vsQueryPopulationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyRef", commandOptions.vsQueryPopulationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "vsQuery.transcriptFlag", commandOptions.vsQueryTranscriptFlag, true); + putNestedIfNotEmpty(beanParams, "vsQuery.functionalScore", commandOptions.vsQueryFunctionalScore, true); + putNestedIfNotEmpty(beanParams, "vsQuery.clinical", commandOptions.vsQueryClinical, true); + putNestedIfNotEmpty(beanParams, "vsQuery.clinicalSignificance", commandOptions.vsQueryClinicalSignificance, true); + putNestedIfNotNull(beanParams, "vsQuery.clinicalConfirmedStatus", commandOptions.vsQueryClinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "msId", commandOptions.msId, true); + putNestedIfNotEmpty(beanParams, "msDescription", commandOptions.msDescription, true); + putNestedIfNotEmpty(beanParams, "msQuery", commandOptions.msQuery, true); + putNestedIfNotEmpty(beanParams, "msFitId", commandOptions.msFitId, true); + putNestedIfNotEmpty(beanParams, "msFitMethod", commandOptions.msFitMethod, true); + putNestedIfNotNull(beanParams, "msFitNBoot", commandOptions.msFitNBoot, true); + putNestedIfNotEmpty(beanParams, "msFitSigVersion", commandOptions.msFitSigVersion, true); + putNestedIfNotEmpty(beanParams, "msFitOrgan", commandOptions.msFitOrgan, true); + putNestedIfNotNull(beanParams, "msFitThresholdPerc", commandOptions.msFitThresholdPerc, true); + putNestedIfNotNull(beanParams, "msFitThresholdPval", commandOptions.msFitThresholdPval, true); + putNestedIfNotNull(beanParams, "msFitMaxRareSigs", commandOptions.msFitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "msFitSignaturesFile", commandOptions.msFitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "msFitRareSignaturesFile", commandOptions.msFitRareSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "gpId", commandOptions.gpId, true); + putNestedIfNotEmpty(beanParams, "gpDescription", commandOptions.gpDescription, true); + putNestedIfNotEmpty(beanParams, "gpConfigFile", commandOptions.gpConfigFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); sampleQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1644,36 +1645,36 @@ private RestResponse runSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleVariantFilterParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotNull(beanParams, "panelIntersection",commandOptions.panelIntersection, true); - putNestedIfNotEmpty(beanParams, "panelFeatureType",commandOptions.panelFeatureType, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotNull(beanParams, "genotypes",commandOptions.genotypes, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "samplesInAllVariants",commandOptions.samplesInAllVariants, true); - putNestedIfNotNull(beanParams, "maxVariants",commandOptions.maxVariants, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotNull(beanParams, "panelIntersection", commandOptions.panelIntersection, true); + putNestedIfNotEmpty(beanParams, "panelFeatureType", commandOptions.panelFeatureType, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotNull(beanParams, "genotypes", commandOptions.genotypes, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "samplesInAllVariants", commandOptions.samplesInAllVariants, true); + putNestedIfNotNull(beanParams, "maxVariants", commandOptions.maxVariants, true); sampleVariantFilterParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1740,42 +1741,42 @@ private RestResponse runSampleStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleVariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "variantQuery.id",commandOptions.variantQueryId, true); - putNestedIfNotEmpty(beanParams, "variantQuery.region",commandOptions.variantQueryRegion, true); - putNestedIfNotEmpty(beanParams, "variantQuery.gene",commandOptions.variantQueryGene, true); - putNestedIfNotEmpty(beanParams, "variantQuery.type",commandOptions.variantQueryType, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panel",commandOptions.variantQueryPanel, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelModeOfInheritance",commandOptions.variantQueryPanelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelConfidence",commandOptions.variantQueryPanelConfidence, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelRoleInCancer",commandOptions.variantQueryPanelRoleInCancer, true); - putNestedIfNotNull(beanParams, "variantQuery.panelIntersection",commandOptions.variantQueryPanelIntersection, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelFeatureType",commandOptions.variantQueryPanelFeatureType, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsRef",commandOptions.variantQueryCohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsAlt",commandOptions.variantQueryCohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsMaf",commandOptions.variantQueryCohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "variantQuery.ct",commandOptions.variantQueryCt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.xref",commandOptions.variantQueryXref, true); - putNestedIfNotEmpty(beanParams, "variantQuery.biotype",commandOptions.variantQueryBiotype, true); - putNestedIfNotEmpty(beanParams, "variantQuery.proteinSubstitution",commandOptions.variantQueryProteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "variantQuery.conservation",commandOptions.variantQueryConservation, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyMaf",commandOptions.variantQueryPopulationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyAlt",commandOptions.variantQueryPopulationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyRef",commandOptions.variantQueryPopulationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "variantQuery.transcriptFlag",commandOptions.variantQueryTranscriptFlag, true); - putNestedIfNotEmpty(beanParams, "variantQuery.functionalScore",commandOptions.variantQueryFunctionalScore, true); - putNestedIfNotEmpty(beanParams, "variantQuery.clinical",commandOptions.variantQueryClinical, true); - putNestedIfNotEmpty(beanParams, "variantQuery.clinicalSignificance",commandOptions.variantQueryClinicalSignificance, true); - putNestedIfNotNull(beanParams, "variantQuery.clinicalConfirmedStatus",commandOptions.variantQueryClinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "variantQuery.sampleData",commandOptions.variantQuerySampleData, true); - putNestedIfNotEmpty(beanParams, "variantQuery.fileData",commandOptions.variantQueryFileData, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotNull(beanParams, "indexOverwrite",commandOptions.indexOverwrite, true); - putNestedIfNotEmpty(beanParams, "indexId",commandOptions.indexId, true); - putNestedIfNotEmpty(beanParams, "indexDescription",commandOptions.indexDescription, true); - putNestedIfNotNull(beanParams, "batchSize",commandOptions.batchSize, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "variantQuery.id", commandOptions.variantQueryId, true); + putNestedIfNotEmpty(beanParams, "variantQuery.region", commandOptions.variantQueryRegion, true); + putNestedIfNotEmpty(beanParams, "variantQuery.gene", commandOptions.variantQueryGene, true); + putNestedIfNotEmpty(beanParams, "variantQuery.type", commandOptions.variantQueryType, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panel", commandOptions.variantQueryPanel, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelModeOfInheritance", commandOptions.variantQueryPanelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelConfidence", commandOptions.variantQueryPanelConfidence, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelRoleInCancer", commandOptions.variantQueryPanelRoleInCancer, true); + putNestedIfNotNull(beanParams, "variantQuery.panelIntersection", commandOptions.variantQueryPanelIntersection, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelFeatureType", commandOptions.variantQueryPanelFeatureType, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsRef", commandOptions.variantQueryCohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsAlt", commandOptions.variantQueryCohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsMaf", commandOptions.variantQueryCohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "variantQuery.ct", commandOptions.variantQueryCt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.xref", commandOptions.variantQueryXref, true); + putNestedIfNotEmpty(beanParams, "variantQuery.biotype", commandOptions.variantQueryBiotype, true); + putNestedIfNotEmpty(beanParams, "variantQuery.proteinSubstitution", commandOptions.variantQueryProteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "variantQuery.conservation", commandOptions.variantQueryConservation, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyMaf", commandOptions.variantQueryPopulationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyAlt", commandOptions.variantQueryPopulationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyRef", commandOptions.variantQueryPopulationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "variantQuery.transcriptFlag", commandOptions.variantQueryTranscriptFlag, true); + putNestedIfNotEmpty(beanParams, "variantQuery.functionalScore", commandOptions.variantQueryFunctionalScore, true); + putNestedIfNotEmpty(beanParams, "variantQuery.clinical", commandOptions.variantQueryClinical, true); + putNestedIfNotEmpty(beanParams, "variantQuery.clinicalSignificance", commandOptions.variantQueryClinicalSignificance, true); + putNestedIfNotNull(beanParams, "variantQuery.clinicalConfirmedStatus", commandOptions.variantQueryClinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "variantQuery.sampleData", commandOptions.variantQuerySampleData, true); + putNestedIfNotEmpty(beanParams, "variantQuery.fileData", commandOptions.variantQueryFileData, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotNull(beanParams, "indexOverwrite", commandOptions.indexOverwrite, true); + putNestedIfNotEmpty(beanParams, "indexId", commandOptions.indexId, true); + putNestedIfNotEmpty(beanParams, "indexDescription", commandOptions.indexDescription, true); + putNestedIfNotNull(beanParams, "batchSize", commandOptions.batchSize, true); sampleVariantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1815,11 +1816,11 @@ private RestResponse runStatsExport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsExportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohorts",commandOptions.cohorts, true); - putNestedIfNotEmpty(beanParams, "output",commandOptions.output, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "outputFileFormat",commandOptions.outputFileFormat, true); + putNestedIfNotNull(beanParams, "cohorts", commandOptions.cohorts, true); + putNestedIfNotEmpty(beanParams, "output", commandOptions.output, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "outputFileFormat", commandOptions.outputFileFormat, true); variantStatsExportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1858,14 +1859,14 @@ private RestResponse runStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); variantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java index b240857de8b..287c2223a00 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java @@ -128,8 +128,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); cohortAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -163,7 +163,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -200,16 +200,16 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); cohortCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -285,16 +285,16 @@ private RestResponse generate() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortGenerateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.bodyId, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.bodyCreationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.bodyModificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.bodyId, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.bodyCreationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.bodyModificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); cohortGenerateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -411,16 +411,16 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); cohortUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java index 00dbbbc130f..d2fde239e71 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java @@ -117,8 +117,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); panelAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -153,17 +153,17 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "author",commandOptions.author, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.version",commandOptions.sourceVersion, true); - putNestedIfNotEmpty(beanParams, "source.author",commandOptions.sourceAuthor, true); - putNestedIfNotEmpty(beanParams, "source.project",commandOptions.sourceProject, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "author", commandOptions.author, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.version", commandOptions.sourceVersion, true); + putNestedIfNotEmpty(beanParams, "source.author", commandOptions.sourceAuthor, true); + putNestedIfNotEmpty(beanParams, "source.project", commandOptions.sourceProject, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); panelCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -234,8 +234,8 @@ private RestResponse importPanels() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelImportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "source",commandOptions.source, true); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "source", commandOptions.source, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); panelImportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -355,17 +355,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "author",commandOptions.author, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.version",commandOptions.sourceVersion, true); - putNestedIfNotEmpty(beanParams, "source.author",commandOptions.sourceAuthor, true); - putNestedIfNotEmpty(beanParams, "source.project",commandOptions.sourceProject, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "author", commandOptions.author, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.version", commandOptions.sourceVersion, true); + putNestedIfNotEmpty(beanParams, "source.author", commandOptions.sourceAuthor, true); + putNestedIfNotEmpty(beanParams, "source.project", commandOptions.sourceProject, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); panelUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java index 5e93767f51c..dc5a1878e40 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java @@ -126,10 +126,10 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); familyAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -163,7 +163,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -199,16 +199,16 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "expectedSize",commandOptions.expectedSize, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "expectedSize", commandOptions.expectedSize, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); familyCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -362,17 +362,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "expectedSize",commandOptions.expectedSize, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "expectedSize", commandOptions.expectedSize, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); familyUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java index f9f6f5fedb8..873a16baac3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java @@ -197,9 +197,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); fileAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -233,7 +233,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -273,27 +273,27 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotNull(beanParams, "format",commandOptions.format, true); - putNestedIfNotNull(beanParams, "bioformat",commandOptions.bioformat, true); - putNestedIfNotNull(beanParams, "sampleIds",commandOptions.sampleIds, true); - putNestedIfNotEmpty(beanParams, "software.name",commandOptions.softwareName, true); - putNestedIfNotEmpty(beanParams, "software.version",commandOptions.softwareVersion, true); - putNestedIfNotEmpty(beanParams, "software.repository",commandOptions.softwareRepository, true); - putNestedIfNotEmpty(beanParams, "software.commit",commandOptions.softwareCommit, true); - putNestedIfNotEmpty(beanParams, "software.website",commandOptions.softwareWebsite, true); - putNestedIfNotNull(beanParams, "software.params",commandOptions.softwareParams, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "jobId",commandOptions.jobId, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "format", commandOptions.format, true); + putNestedIfNotNull(beanParams, "bioformat", commandOptions.bioformat, true); + putNestedIfNotNull(beanParams, "sampleIds", commandOptions.sampleIds, true); + putNestedIfNotEmpty(beanParams, "software.name", commandOptions.softwareName, true); + putNestedIfNotEmpty(beanParams, "software.version", commandOptions.softwareVersion, true); + putNestedIfNotEmpty(beanParams, "software.repository", commandOptions.softwareRepository, true); + putNestedIfNotEmpty(beanParams, "software.commit", commandOptions.softwareCommit, true); + putNestedIfNotEmpty(beanParams, "software.website", commandOptions.softwareWebsite, true); + putNestedMapIfNotEmpty(beanParams, "software.params", commandOptions.softwareParams, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "jobId", commandOptions.jobId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); fileCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -371,8 +371,8 @@ private RestResponse fetch() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileFetch.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "url",commandOptions.url, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "url", commandOptions.url, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); fileFetch = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -412,16 +412,16 @@ private RestResponse link() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileLinkParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "uri",commandOptions.uri, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "virtualFileName",commandOptions.virtualFileName, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "internal.sampleMap",commandOptions.internalSampleMap, true); + putNestedIfNotEmpty(beanParams, "uri", commandOptions.uri, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "virtualFileName", commandOptions.virtualFileName, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "internal.sampleMap", commandOptions.internalSampleMap, true); fileLinkParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -460,12 +460,12 @@ private RestResponse runLink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileLinkToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "uri",commandOptions.uri, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "virtualFileName",commandOptions.virtualFileName, true); - putNestedIfNotNull(beanParams, "parents",commandOptions.parents, true); - putNestedIfNotNull(beanParams, "skipPostLink",commandOptions.skipPostLink, true); + putNestedIfNotNull(beanParams, "uri", commandOptions.uri, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "virtualFileName", commandOptions.virtualFileName, true); + putNestedIfNotNull(beanParams, "parents", commandOptions.parents, true); + putNestedIfNotNull(beanParams, "skipPostLink", commandOptions.skipPostLink, true); fileLinkToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -504,8 +504,8 @@ private RestResponse runPostlink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PostLinkToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "files",commandOptions.files, true); - putNestedIfNotNull(beanParams, "batchSize",commandOptions.batchSize, true); + putNestedIfNotNull(beanParams, "files", commandOptions.files, true); + putNestedIfNotNull(beanParams, "batchSize", commandOptions.batchSize, true); postLinkToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -669,37 +669,37 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "sampleIds",commandOptions.sampleIds, true); - putNestedIfNotNull(beanParams, "format",commandOptions.format, true); - putNestedIfNotNull(beanParams, "bioformat",commandOptions.bioformat, true); - putNestedIfNotEmpty(beanParams, "software.name",commandOptions.softwareName, true); - putNestedIfNotEmpty(beanParams, "software.version",commandOptions.softwareVersion, true); - putNestedIfNotEmpty(beanParams, "software.repository",commandOptions.softwareRepository, true); - putNestedIfNotEmpty(beanParams, "software.commit",commandOptions.softwareCommit, true); - putNestedIfNotEmpty(beanParams, "software.website",commandOptions.softwareWebsite, true); - putNestedIfNotNull(beanParams, "software.params",commandOptions.softwareParams, true); - putNestedIfNotNull(beanParams, "experiment.technology",commandOptions.experimentTechnology, true); - putNestedIfNotNull(beanParams, "experiment.method",commandOptions.experimentMethod, true); - putNestedIfNotNull(beanParams, "experiment.nucleicAcidType",commandOptions.experimentNucleicAcidType, true); - putNestedIfNotEmpty(beanParams, "experiment.manufacturer",commandOptions.experimentManufacturer, true); - putNestedIfNotEmpty(beanParams, "experiment.platform",commandOptions.experimentPlatform, true); - putNestedIfNotEmpty(beanParams, "experiment.library",commandOptions.experimentLibrary, true); - putNestedIfNotEmpty(beanParams, "experiment.date",commandOptions.experimentDate, true); - putNestedIfNotEmpty(beanParams, "experiment.center",commandOptions.experimentCenter, true); - putNestedIfNotEmpty(beanParams, "experiment.lab",commandOptions.experimentLab, true); - putNestedIfNotEmpty(beanParams, "experiment.responsible",commandOptions.experimentResponsible, true); - putNestedIfNotEmpty(beanParams, "experiment.description",commandOptions.experimentDescription, true); - putNestedIfNotNull(beanParams, "experiment.attributes",commandOptions.experimentAttributes, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "stats",commandOptions.stats, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "sampleIds", commandOptions.sampleIds, true); + putNestedIfNotNull(beanParams, "format", commandOptions.format, true); + putNestedIfNotNull(beanParams, "bioformat", commandOptions.bioformat, true); + putNestedIfNotEmpty(beanParams, "software.name", commandOptions.softwareName, true); + putNestedIfNotEmpty(beanParams, "software.version", commandOptions.softwareVersion, true); + putNestedIfNotEmpty(beanParams, "software.repository", commandOptions.softwareRepository, true); + putNestedIfNotEmpty(beanParams, "software.commit", commandOptions.softwareCommit, true); + putNestedIfNotEmpty(beanParams, "software.website", commandOptions.softwareWebsite, true); + putNestedMapIfNotEmpty(beanParams, "software.params", commandOptions.softwareParams, true); + putNestedIfNotNull(beanParams, "experiment.technology", commandOptions.experimentTechnology, true); + putNestedIfNotNull(beanParams, "experiment.method", commandOptions.experimentMethod, true); + putNestedIfNotNull(beanParams, "experiment.nucleicAcidType", commandOptions.experimentNucleicAcidType, true); + putNestedIfNotEmpty(beanParams, "experiment.manufacturer", commandOptions.experimentManufacturer, true); + putNestedIfNotEmpty(beanParams, "experiment.platform", commandOptions.experimentPlatform, true); + putNestedIfNotEmpty(beanParams, "experiment.library", commandOptions.experimentLibrary, true); + putNestedIfNotEmpty(beanParams, "experiment.date", commandOptions.experimentDate, true); + putNestedIfNotEmpty(beanParams, "experiment.center", commandOptions.experimentCenter, true); + putNestedIfNotEmpty(beanParams, "experiment.lab", commandOptions.experimentLab, true); + putNestedIfNotEmpty(beanParams, "experiment.responsible", commandOptions.experimentResponsible, true); + putNestedIfNotEmpty(beanParams, "experiment.description", commandOptions.experimentDescription, true); + putNestedMapIfNotEmpty(beanParams, "experiment.attributes", commandOptions.experimentAttributes, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedMapIfNotEmpty(beanParams, "stats", commandOptions.stats, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); fileUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -820,7 +820,7 @@ private RestResponse move() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileMoveParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); fileMoveParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java index 495341543d6..dcf24216d8d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java @@ -135,9 +135,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); individualAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -171,7 +171,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -207,42 +207,42 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "father.id",commandOptions.fatherId, true); - putNestedIfNotEmpty(beanParams, "father.uuid",commandOptions.fatherUuid, true); - putNestedIfNotEmpty(beanParams, "mother.id",commandOptions.motherId, true); - putNestedIfNotEmpty(beanParams, "mother.uuid",commandOptions.motherUuid, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "location.address",commandOptions.locationAddress, true); - putNestedIfNotEmpty(beanParams, "location.postalCode",commandOptions.locationPostalCode, true); - putNestedIfNotEmpty(beanParams, "location.city",commandOptions.locationCity, true); - putNestedIfNotEmpty(beanParams, "location.state",commandOptions.locationState, true); - putNestedIfNotEmpty(beanParams, "location.country",commandOptions.locationCountry, true); - putNestedIfNotEmpty(beanParams, "sex.id",commandOptions.sexId, true); - putNestedIfNotEmpty(beanParams, "sex.name",commandOptions.sexName, true); - putNestedIfNotEmpty(beanParams, "sex.description",commandOptions.sexDescription, true); - putNestedIfNotEmpty(beanParams, "sex.source",commandOptions.sexSource, true); - putNestedIfNotEmpty(beanParams, "sex.url",commandOptions.sexUrl, true); - putNestedIfNotNull(beanParams, "sex.attributes",commandOptions.sexAttributes, true); - putNestedIfNotEmpty(beanParams, "ethnicity.id",commandOptions.ethnicityId, true); - putNestedIfNotEmpty(beanParams, "ethnicity.name",commandOptions.ethnicityName, true); - putNestedIfNotEmpty(beanParams, "ethnicity.description",commandOptions.ethnicityDescription, true); - putNestedIfNotEmpty(beanParams, "ethnicity.source",commandOptions.ethnicitySource, true); - putNestedIfNotEmpty(beanParams, "ethnicity.url",commandOptions.ethnicityUrl, true); - putNestedIfNotNull(beanParams, "ethnicity.attributes",commandOptions.ethnicityAttributes, true); - putNestedIfNotNull(beanParams, "parentalConsanguinity",commandOptions.parentalConsanguinity, true); - putNestedIfNotEmpty(beanParams, "population.name",commandOptions.populationName, true); - putNestedIfNotEmpty(beanParams, "population.subpopulation",commandOptions.populationSubpopulation, true); - putNestedIfNotEmpty(beanParams, "population.description",commandOptions.populationDescription, true); - putNestedIfNotEmpty(beanParams, "dateOfBirth",commandOptions.dateOfBirth, true); - putNestedIfNotNull(beanParams, "karyotypicSex",commandOptions.karyotypicSex, true); - putNestedIfNotNull(beanParams, "lifeStatus",commandOptions.lifeStatus, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "father.id", commandOptions.fatherId, true); + putNestedIfNotEmpty(beanParams, "father.uuid", commandOptions.fatherUuid, true); + putNestedIfNotEmpty(beanParams, "mother.id", commandOptions.motherId, true); + putNestedIfNotEmpty(beanParams, "mother.uuid", commandOptions.motherUuid, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "location.address", commandOptions.locationAddress, true); + putNestedIfNotEmpty(beanParams, "location.postalCode", commandOptions.locationPostalCode, true); + putNestedIfNotEmpty(beanParams, "location.city", commandOptions.locationCity, true); + putNestedIfNotEmpty(beanParams, "location.state", commandOptions.locationState, true); + putNestedIfNotEmpty(beanParams, "location.country", commandOptions.locationCountry, true); + putNestedIfNotEmpty(beanParams, "sex.id", commandOptions.sexId, true); + putNestedIfNotEmpty(beanParams, "sex.name", commandOptions.sexName, true); + putNestedIfNotEmpty(beanParams, "sex.description", commandOptions.sexDescription, true); + putNestedIfNotEmpty(beanParams, "sex.source", commandOptions.sexSource, true); + putNestedIfNotEmpty(beanParams, "sex.url", commandOptions.sexUrl, true); + putNestedMapIfNotEmpty(beanParams, "sex.attributes", commandOptions.sexAttributes, true); + putNestedIfNotEmpty(beanParams, "ethnicity.id", commandOptions.ethnicityId, true); + putNestedIfNotEmpty(beanParams, "ethnicity.name", commandOptions.ethnicityName, true); + putNestedIfNotEmpty(beanParams, "ethnicity.description", commandOptions.ethnicityDescription, true); + putNestedIfNotEmpty(beanParams, "ethnicity.source", commandOptions.ethnicitySource, true); + putNestedIfNotEmpty(beanParams, "ethnicity.url", commandOptions.ethnicityUrl, true); + putNestedMapIfNotEmpty(beanParams, "ethnicity.attributes", commandOptions.ethnicityAttributes, true); + putNestedIfNotNull(beanParams, "parentalConsanguinity", commandOptions.parentalConsanguinity, true); + putNestedIfNotEmpty(beanParams, "population.name", commandOptions.populationName, true); + putNestedIfNotEmpty(beanParams, "population.subpopulation", commandOptions.populationSubpopulation, true); + putNestedIfNotEmpty(beanParams, "population.description", commandOptions.populationDescription, true); + putNestedIfNotEmpty(beanParams, "dateOfBirth", commandOptions.dateOfBirth, true); + putNestedIfNotNull(beanParams, "karyotypicSex", commandOptions.karyotypicSex, true); + putNestedIfNotNull(beanParams, "lifeStatus", commandOptions.lifeStatus, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); individualCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -411,43 +411,43 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "father.id",commandOptions.fatherId, true); - putNestedIfNotEmpty(beanParams, "father.uuid",commandOptions.fatherUuid, true); - putNestedIfNotEmpty(beanParams, "mother.id",commandOptions.motherId, true); - putNestedIfNotEmpty(beanParams, "mother.uuid",commandOptions.motherUuid, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "parentalConsanguinity",commandOptions.parentalConsanguinity, true); - putNestedIfNotEmpty(beanParams, "location.address",commandOptions.locationAddress, true); - putNestedIfNotEmpty(beanParams, "location.postalCode",commandOptions.locationPostalCode, true); - putNestedIfNotEmpty(beanParams, "location.city",commandOptions.locationCity, true); - putNestedIfNotEmpty(beanParams, "location.state",commandOptions.locationState, true); - putNestedIfNotEmpty(beanParams, "location.country",commandOptions.locationCountry, true); - putNestedIfNotEmpty(beanParams, "sex.id",commandOptions.sexId, true); - putNestedIfNotEmpty(beanParams, "sex.name",commandOptions.sexName, true); - putNestedIfNotEmpty(beanParams, "sex.description",commandOptions.sexDescription, true); - putNestedIfNotEmpty(beanParams, "sex.source",commandOptions.sexSource, true); - putNestedIfNotEmpty(beanParams, "sex.url",commandOptions.sexUrl, true); - putNestedIfNotNull(beanParams, "sex.attributes",commandOptions.sexAttributes, true); - putNestedIfNotEmpty(beanParams, "ethnicity.id",commandOptions.ethnicityId, true); - putNestedIfNotEmpty(beanParams, "ethnicity.name",commandOptions.ethnicityName, true); - putNestedIfNotEmpty(beanParams, "ethnicity.description",commandOptions.ethnicityDescription, true); - putNestedIfNotEmpty(beanParams, "ethnicity.source",commandOptions.ethnicitySource, true); - putNestedIfNotEmpty(beanParams, "ethnicity.url",commandOptions.ethnicityUrl, true); - putNestedIfNotNull(beanParams, "ethnicity.attributes",commandOptions.ethnicityAttributes, true); - putNestedIfNotEmpty(beanParams, "population.name",commandOptions.populationName, true); - putNestedIfNotEmpty(beanParams, "population.subpopulation",commandOptions.populationSubpopulation, true); - putNestedIfNotEmpty(beanParams, "population.description",commandOptions.populationDescription, true); - putNestedIfNotEmpty(beanParams, "dateOfBirth",commandOptions.dateOfBirth, true); - putNestedIfNotNull(beanParams, "karyotypicSex",commandOptions.karyotypicSex, true); - putNestedIfNotNull(beanParams, "lifeStatus",commandOptions.lifeStatus, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "father.id", commandOptions.fatherId, true); + putNestedIfNotEmpty(beanParams, "father.uuid", commandOptions.fatherUuid, true); + putNestedIfNotEmpty(beanParams, "mother.id", commandOptions.motherId, true); + putNestedIfNotEmpty(beanParams, "mother.uuid", commandOptions.motherUuid, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "parentalConsanguinity", commandOptions.parentalConsanguinity, true); + putNestedIfNotEmpty(beanParams, "location.address", commandOptions.locationAddress, true); + putNestedIfNotEmpty(beanParams, "location.postalCode", commandOptions.locationPostalCode, true); + putNestedIfNotEmpty(beanParams, "location.city", commandOptions.locationCity, true); + putNestedIfNotEmpty(beanParams, "location.state", commandOptions.locationState, true); + putNestedIfNotEmpty(beanParams, "location.country", commandOptions.locationCountry, true); + putNestedIfNotEmpty(beanParams, "sex.id", commandOptions.sexId, true); + putNestedIfNotEmpty(beanParams, "sex.name", commandOptions.sexName, true); + putNestedIfNotEmpty(beanParams, "sex.description", commandOptions.sexDescription, true); + putNestedIfNotEmpty(beanParams, "sex.source", commandOptions.sexSource, true); + putNestedIfNotEmpty(beanParams, "sex.url", commandOptions.sexUrl, true); + putNestedMapIfNotEmpty(beanParams, "sex.attributes", commandOptions.sexAttributes, true); + putNestedIfNotEmpty(beanParams, "ethnicity.id", commandOptions.ethnicityId, true); + putNestedIfNotEmpty(beanParams, "ethnicity.name", commandOptions.ethnicityName, true); + putNestedIfNotEmpty(beanParams, "ethnicity.description", commandOptions.ethnicityDescription, true); + putNestedIfNotEmpty(beanParams, "ethnicity.source", commandOptions.ethnicitySource, true); + putNestedIfNotEmpty(beanParams, "ethnicity.url", commandOptions.ethnicityUrl, true); + putNestedMapIfNotEmpty(beanParams, "ethnicity.attributes", commandOptions.ethnicityAttributes, true); + putNestedIfNotEmpty(beanParams, "population.name", commandOptions.populationName, true); + putNestedIfNotEmpty(beanParams, "population.subpopulation", commandOptions.populationSubpopulation, true); + putNestedIfNotEmpty(beanParams, "population.description", commandOptions.populationDescription, true); + putNestedIfNotEmpty(beanParams, "dateOfBirth", commandOptions.dateOfBirth, true); + putNestedIfNotNull(beanParams, "karyotypicSex", commandOptions.karyotypicSex, true); + putNestedIfNotNull(beanParams, "lifeStatus", commandOptions.lifeStatus, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); individualUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index cf28e7983bf..ffc4c7a61a1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -137,8 +137,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "job",commandOptions.job, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "job", commandOptions.job, true); jobAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -170,25 +170,25 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "tool.id",commandOptions.toolId, true); - putNestedIfNotEmpty(beanParams, "tool.description",commandOptions.toolDescription, true); - putNestedIfNotNull(beanParams, "tool.scope",commandOptions.toolScope, true); - putNestedIfNotNull(beanParams, "tool.type",commandOptions.toolType, true); - putNestedIfNotNull(beanParams, "tool.resource",commandOptions.toolResource, true); - putNestedIfNotNull(beanParams, "priority",commandOptions.priority, true); - putNestedIfNotEmpty(beanParams, "commandLine",commandOptions.commandLine, true); - putNestedIfNotNull(beanParams, "params",commandOptions.params, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "outDir.path",commandOptions.outDirPath, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "result.id",commandOptions.resultId, true); - putNestedIfNotNull(beanParams, "result.attributes",commandOptions.resultAttributes, true); - putNestedIfNotEmpty(beanParams, "stdout.path",commandOptions.stdoutPath, true); - putNestedIfNotEmpty(beanParams, "stderr.path",commandOptions.stderrPath, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "tool.id", commandOptions.toolId, true); + putNestedIfNotEmpty(beanParams, "tool.description", commandOptions.toolDescription, true); + putNestedIfNotNull(beanParams, "tool.scope", commandOptions.toolScope, true); + putNestedIfNotNull(beanParams, "tool.type", commandOptions.toolType, true); + putNestedIfNotNull(beanParams, "tool.resource", commandOptions.toolResource, true); + putNestedIfNotNull(beanParams, "priority", commandOptions.priority, true); + putNestedIfNotEmpty(beanParams, "commandLine", commandOptions.commandLine, true); + putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "outDir.path", commandOptions.outDirPath, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "result.id", commandOptions.resultId, true); + putNestedMapIfNotEmpty(beanParams, "result.attributes", commandOptions.resultAttributes, true); + putNestedIfNotEmpty(beanParams, "stdout.path", commandOptions.stdoutPath, true); + putNestedIfNotEmpty(beanParams, "stderr.path", commandOptions.stderrPath, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); jobCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -257,9 +257,9 @@ private RestResponse retry() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobRetryParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "job",commandOptions.job, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); - putNestedIfNotNull(beanParams, "params",commandOptions.params, true); + putNestedIfNotEmpty(beanParams, "job", commandOptions.job, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); + putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); jobRetryParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -392,10 +392,10 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visited",commandOptions.visited, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visited", commandOptions.visited, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); jobUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java index 72f9bdd0e8c..f1e0c47ad70 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java @@ -17,8 +17,8 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; @@ -36,19 +36,13 @@ import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.server.generator.models.RestCategory; -import org.opencb.opencga.server.generator.models.RestEndpoint; -import org.opencb.opencga.server.generator.models.RestParameter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.Date; -import java.util.List; +import java.util.*; /** * Created on 27/05/16. @@ -266,18 +260,22 @@ public RestResponse refreshToken(AuthenticationResponse return res; } - public Object putNestedIfNotNull(ObjectMap map, String key, Object value, boolean parents) { + public void putNestedIfNotNull(ObjectMap map, String key, Object value, boolean parents) { if (value != null) { map.putNested(key, value, parents); } - return null; } - public Object putNestedIfNotEmpty(ObjectMap map, String key, String value, boolean parents) { + public void putNestedIfNotEmpty(ObjectMap map, String key, String value, boolean parents) { if (StringUtils.isNotEmpty(value)) { map.putNested(key, value, parents); } - return null; + } + + public void putNestedMapIfNotEmpty(ObjectMap map, String key, Map value, boolean parents) { + if (MapUtils.isNotEmpty(value)) { + map.putNested(key, value, parents); + } } public boolean checkExpiredSession(String[] args) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 6135886595d..35f0c49c312 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -37,6 +37,8 @@ import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.operations.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; @@ -149,6 +151,9 @@ public void execute() throws Exception { case "variant-secondary-index-delete": queryResponse = deleteVariantSecondaryIndex(); break; + case "variant-setup": + queryResponse = setupVariant(); + break; case "variant-stats-delete": queryResponse = deleteVariantStats(); break; @@ -189,10 +194,10 @@ private RestResponse configureCellbase() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CellBaseConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "url",commandOptions.url, true); - putNestedIfNotEmpty(beanParams, "version",commandOptions.version, true); - putNestedIfNotEmpty(beanParams, "dataRelease",commandOptions.dataRelease, true); - putNestedIfNotEmpty(beanParams, "apiKey",commandOptions.apiKey, true); + putNestedIfNotEmpty(beanParams, "url", commandOptions.url, true); + putNestedIfNotEmpty(beanParams, "version", commandOptions.version, true); + putNestedIfNotEmpty(beanParams, "dataRelease", commandOptions.dataRelease, true); + putNestedIfNotEmpty(beanParams, "apiKey", commandOptions.apiKey, true); cellBaseConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -231,8 +236,8 @@ private RestResponse aggregateVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAggregateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantAggregateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -291,15 +296,15 @@ private RestResponse indexVariantAnnotation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "create",commandOptions.create, true); - putNestedIfNotEmpty(beanParams, "load",commandOptions.load, true); - putNestedIfNotEmpty(beanParams, "customName",commandOptions.customName, true); - putNestedIfNotNull(beanParams, "sampleIndexAnnotation",commandOptions.sampleIndexAnnotation, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "create", commandOptions.create, true); + putNestedIfNotEmpty(beanParams, "load", commandOptions.load, true); + putNestedIfNotEmpty(beanParams, "customName", commandOptions.customName, true); + putNestedIfNotNull(beanParams, "sampleIndexAnnotation", commandOptions.sampleIndexAnnotation, true); variantAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -335,7 +340,7 @@ private RestResponse saveVariantAnnotation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAnnotationSaveParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "annotationId",commandOptions.annotationId, true); + putNestedIfNotEmpty(beanParams, "annotationId", commandOptions.annotationId, true); variantAnnotationSaveParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -368,7 +373,7 @@ private RestResponse configureVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantConfigureParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "configuration",commandOptions.configuration, true); + putNestedMapIfNotEmpty(beanParams, "configuration", commandOptions.configuration, true); variantConfigureParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -407,9 +412,9 @@ private RestResponse deleteVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFileDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotNull(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); variantFileDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -448,9 +453,9 @@ private RestResponse aggregateVariantFamily() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAggregateFamilyParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "gapsGenotype",commandOptions.gapsGenotype, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "gapsGenotype", commandOptions.gapsGenotype, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantAggregateFamilyParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -489,10 +494,10 @@ private RestResponse indexVariantFamily() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFamilyIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotNull(beanParams, "updateIndex",commandOptions.updateIndex, true); - putNestedIfNotNull(beanParams, "skipIncompleteFamilies",commandOptions.skipIncompleteFamilies, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "updateIndex", commandOptions.updateIndex, true); + putNestedIfNotNull(beanParams, "skipIncompleteFamilies", commandOptions.skipIncompleteFamilies, true); variantFamilyIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -531,36 +536,36 @@ private RestResponse indexVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "transform",commandOptions.transform, true); - putNestedIfNotNull(beanParams, "gvcf",commandOptions.gvcf, true); - putNestedIfNotNull(beanParams, "normalizationSkip",commandOptions.normalizationSkip, true); - putNestedIfNotEmpty(beanParams, "referenceGenome",commandOptions.referenceGenome, true); - putNestedIfNotEmpty(beanParams, "failOnMalformedLines",commandOptions.failOnMalformedLines, true); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "load",commandOptions.load, true); - putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); - putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); - putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "loadArchive",commandOptions.loadArchive, true); - putNestedIfNotEmpty(beanParams, "loadHomRef",commandOptions.loadHomRef, true); - putNestedIfNotEmpty(beanParams, "postLoadCheck",commandOptions.postLoadCheck, true); - putNestedIfNotEmpty(beanParams, "includeGenotypes",commandOptions.includeGenotypes, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotEmpty(beanParams, "merge",commandOptions.merge, true); - putNestedIfNotEmpty(beanParams, "deduplicationPolicy",commandOptions.deduplicationPolicy, true); - putNestedIfNotNull(beanParams, "calculateStats",commandOptions.calculateStats, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexSearch",commandOptions.indexSearch, true); - putNestedIfNotNull(beanParams, "skipIndexedFiles",commandOptions.skipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "transform", commandOptions.transform, true); + putNestedIfNotNull(beanParams, "gvcf", commandOptions.gvcf, true); + putNestedIfNotNull(beanParams, "normalizationSkip", commandOptions.normalizationSkip, true); + putNestedIfNotEmpty(beanParams, "referenceGenome", commandOptions.referenceGenome, true); + putNestedIfNotEmpty(beanParams, "failOnMalformedLines", commandOptions.failOnMalformedLines, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotNull(beanParams, "load", commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload", commandOptions.forceReload, true); + putNestedIfNotEmpty(beanParams, "loadSplitData", commandOptions.loadSplitData, true); + putNestedIfNotNull(beanParams, "loadMultiFileData", commandOptions.loadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "loadSampleIndex", commandOptions.loadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "loadArchive", commandOptions.loadArchive, true); + putNestedIfNotEmpty(beanParams, "loadHomRef", commandOptions.loadHomRef, true); + putNestedIfNotEmpty(beanParams, "postLoadCheck", commandOptions.postLoadCheck, true); + putNestedIfNotEmpty(beanParams, "includeGenotypes", commandOptions.includeGenotypes, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotEmpty(beanParams, "merge", commandOptions.merge, true); + putNestedIfNotEmpty(beanParams, "deduplicationPolicy", commandOptions.deduplicationPolicy, true); + putNestedIfNotNull(beanParams, "calculateStats", commandOptions.calculateStats, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexSearch", commandOptions.indexSearch, true); + putNestedIfNotNull(beanParams, "skipIndexedFiles", commandOptions.skipIndexedFiles, true); variantIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -599,41 +604,41 @@ private RestResponse launcherVariantIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFileIndexJobLauncherParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "directory",commandOptions.directory, true); - putNestedIfNotNull(beanParams, "resumeFailed",commandOptions.resumeFailed, true); - putNestedIfNotNull(beanParams, "ignoreFailed",commandOptions.ignoreFailed, true); - putNestedIfNotNull(beanParams, "maxJobs",commandOptions.maxJobs, true); - putNestedIfNotEmpty(beanParams, "indexParams.file",commandOptions.indexParamsFile, true); - putNestedIfNotNull(beanParams, "indexParams.resume",commandOptions.indexParamsResume, true); - putNestedIfNotEmpty(beanParams, "indexParams.outdir",commandOptions.indexParamsOutdir, true); - putNestedIfNotNull(beanParams, "indexParams.transform",commandOptions.indexParamsTransform, true); - putNestedIfNotNull(beanParams, "indexParams.gvcf",commandOptions.indexParamsGvcf, true); - putNestedIfNotNull(beanParams, "indexParams.normalizationSkip",commandOptions.indexParamsNormalizationSkip, true); - putNestedIfNotEmpty(beanParams, "indexParams.referenceGenome",commandOptions.indexParamsReferenceGenome, true); - putNestedIfNotEmpty(beanParams, "indexParams.failOnMalformedLines",commandOptions.indexParamsFailOnMalformedLines, true); - putNestedIfNotNull(beanParams, "indexParams.family",commandOptions.indexParamsFamily, true); - putNestedIfNotNull(beanParams, "indexParams.somatic",commandOptions.indexParamsSomatic, true); - putNestedIfNotNull(beanParams, "indexParams.load",commandOptions.indexParamsLoad, true); - putNestedIfNotNull(beanParams, "indexParams.forceReload",commandOptions.indexParamsForceReload, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData",commandOptions.indexParamsLoadSplitData, true); - putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData",commandOptions.indexParamsLoadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex",commandOptions.indexParamsLoadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadArchive",commandOptions.indexParamsLoadArchive, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadHomRef",commandOptions.indexParamsLoadHomRef, true); - putNestedIfNotEmpty(beanParams, "indexParams.postLoadCheck",commandOptions.indexParamsPostLoadCheck, true); - putNestedIfNotEmpty(beanParams, "indexParams.includeGenotypes",commandOptions.indexParamsIncludeGenotypes, true); - putNestedIfNotEmpty(beanParams, "indexParams.includeSampleData",commandOptions.indexParamsIncludeSampleData, true); - putNestedIfNotEmpty(beanParams, "indexParams.merge",commandOptions.indexParamsMerge, true); - putNestedIfNotEmpty(beanParams, "indexParams.deduplicationPolicy",commandOptions.indexParamsDeduplicationPolicy, true); - putNestedIfNotNull(beanParams, "indexParams.calculateStats",commandOptions.indexParamsCalculateStats, true); - putNestedIfNotNull(beanParams, "indexParams.aggregated",commandOptions.indexParamsAggregated, true); - putNestedIfNotEmpty(beanParams, "indexParams.aggregationMappingFile",commandOptions.indexParamsAggregationMappingFile, true); - putNestedIfNotNull(beanParams, "indexParams.annotate",commandOptions.indexParamsAnnotate, true); - putNestedIfNotEmpty(beanParams, "indexParams.annotator",commandOptions.indexParamsAnnotator, true); - putNestedIfNotNull(beanParams, "indexParams.overwriteAnnotations",commandOptions.indexParamsOverwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexParams.indexSearch",commandOptions.indexParamsIndexSearch, true); - putNestedIfNotNull(beanParams, "indexParams.skipIndexedFiles",commandOptions.indexParamsSkipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "directory", commandOptions.directory, true); + putNestedIfNotNull(beanParams, "resumeFailed", commandOptions.resumeFailed, true); + putNestedIfNotNull(beanParams, "ignoreFailed", commandOptions.ignoreFailed, true); + putNestedIfNotNull(beanParams, "maxJobs", commandOptions.maxJobs, true); + putNestedIfNotEmpty(beanParams, "indexParams.file", commandOptions.indexParamsFile, true); + putNestedIfNotNull(beanParams, "indexParams.resume", commandOptions.indexParamsResume, true); + putNestedIfNotEmpty(beanParams, "indexParams.outdir", commandOptions.indexParamsOutdir, true); + putNestedIfNotNull(beanParams, "indexParams.transform", commandOptions.indexParamsTransform, true); + putNestedIfNotNull(beanParams, "indexParams.gvcf", commandOptions.indexParamsGvcf, true); + putNestedIfNotNull(beanParams, "indexParams.normalizationSkip", commandOptions.indexParamsNormalizationSkip, true); + putNestedIfNotEmpty(beanParams, "indexParams.referenceGenome", commandOptions.indexParamsReferenceGenome, true); + putNestedIfNotEmpty(beanParams, "indexParams.failOnMalformedLines", commandOptions.indexParamsFailOnMalformedLines, true); + putNestedIfNotNull(beanParams, "indexParams.family", commandOptions.indexParamsFamily, true); + putNestedIfNotNull(beanParams, "indexParams.somatic", commandOptions.indexParamsSomatic, true); + putNestedIfNotNull(beanParams, "indexParams.load", commandOptions.indexParamsLoad, true); + putNestedIfNotNull(beanParams, "indexParams.forceReload", commandOptions.indexParamsForceReload, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData", commandOptions.indexParamsLoadSplitData, true); + putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData", commandOptions.indexParamsLoadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex", commandOptions.indexParamsLoadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadArchive", commandOptions.indexParamsLoadArchive, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadHomRef", commandOptions.indexParamsLoadHomRef, true); + putNestedIfNotEmpty(beanParams, "indexParams.postLoadCheck", commandOptions.indexParamsPostLoadCheck, true); + putNestedIfNotEmpty(beanParams, "indexParams.includeGenotypes", commandOptions.indexParamsIncludeGenotypes, true); + putNestedIfNotEmpty(beanParams, "indexParams.includeSampleData", commandOptions.indexParamsIncludeSampleData, true); + putNestedIfNotEmpty(beanParams, "indexParams.merge", commandOptions.indexParamsMerge, true); + putNestedIfNotEmpty(beanParams, "indexParams.deduplicationPolicy", commandOptions.indexParamsDeduplicationPolicy, true); + putNestedIfNotNull(beanParams, "indexParams.calculateStats", commandOptions.indexParamsCalculateStats, true); + putNestedIfNotNull(beanParams, "indexParams.aggregated", commandOptions.indexParamsAggregated, true); + putNestedIfNotEmpty(beanParams, "indexParams.aggregationMappingFile", commandOptions.indexParamsAggregationMappingFile, true); + putNestedIfNotNull(beanParams, "indexParams.annotate", commandOptions.indexParamsAnnotate, true); + putNestedIfNotEmpty(beanParams, "indexParams.annotator", commandOptions.indexParamsAnnotator, true); + putNestedIfNotNull(beanParams, "indexParams.overwriteAnnotations", commandOptions.indexParamsOverwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexParams.indexSearch", commandOptions.indexParamsIndexSearch, true); + putNestedIfNotNull(beanParams, "indexParams.skipIndexedFiles", commandOptions.indexParamsSkipIndexedFiles, true); variantFileIndexJobLauncherParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -669,9 +674,9 @@ private RestResponse runVariantJulie() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JulieParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohorts",commandOptions.cohorts, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "cohorts", commandOptions.cohorts, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); julieParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -706,8 +711,8 @@ private RestResponse repairVariantMetadata() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStorageMetadataRepairToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "studies",commandOptions.studies, true); - putNestedIfNotNull(beanParams, "samplesBatchSize",commandOptions.samplesBatchSize, true); + putNestedIfNotNull(beanParams, "studies", commandOptions.studies, true); + putNestedIfNotNull(beanParams, "samplesBatchSize", commandOptions.samplesBatchSize, true); variantStorageMetadataRepairToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -746,8 +751,8 @@ private RestResponse synchronizeVariantMetadata() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStorageMetadataSynchronizeParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotNull(beanParams, "files",commandOptions.files, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotNull(beanParams, "files", commandOptions.files, true); variantStorageMetadataSynchronizeParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -782,9 +787,9 @@ private RestResponse pruneVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantPruneParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "project",commandOptions.project, true); - putNestedIfNotNull(beanParams, "dryRun",commandOptions.dryRun, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "project", commandOptions.project, true); + putNestedIfNotNull(beanParams, "dryRun", commandOptions.dryRun, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantPruneParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -823,9 +828,9 @@ private RestResponse deleteVariantSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSampleDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantSampleDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -864,11 +869,11 @@ private RestResponse indexVariantSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondarySampleIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "buildIndex",commandOptions.buildIndex, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotNull(beanParams, "familyIndex",commandOptions.familyIndex, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "buildIndex", commandOptions.buildIndex, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotNull(beanParams, "familyIndex", commandOptions.familyIndex, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondarySampleIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -957,12 +962,12 @@ private RestResponse indexVariantScore() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantScoreIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "scoreName",commandOptions.scoreName, true); - putNestedIfNotEmpty(beanParams, "cohort1",commandOptions.cohort1, true); - putNestedIfNotEmpty(beanParams, "cohort2",commandOptions.cohort2, true); - putNestedIfNotEmpty(beanParams, "input",commandOptions.input, true); - putNestedIfNotEmpty(beanParams, "inputColumns",commandOptions.inputColumns, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "scoreName", commandOptions.scoreName, true); + putNestedIfNotEmpty(beanParams, "cohort1", commandOptions.cohort1, true); + putNestedIfNotEmpty(beanParams, "cohort2", commandOptions.cohort2, true); + putNestedIfNotEmpty(beanParams, "input", commandOptions.input, true); + putNestedIfNotEmpty(beanParams, "inputColumns", commandOptions.inputColumns, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantScoreIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1002,9 +1007,9 @@ private RestResponse variantSecondaryAnnotationIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondaryAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondaryAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1043,11 +1048,11 @@ private RestResponse variantSecondarySampleIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondarySampleIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "buildIndex",commandOptions.buildIndex, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotNull(beanParams, "familyIndex",commandOptions.familyIndex, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "buildIndex", commandOptions.buildIndex, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotNull(beanParams, "familyIndex", commandOptions.familyIndex, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondarySampleIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1113,9 +1118,9 @@ private RestResponse secondaryIndexVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondaryAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondaryAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1146,6 +1151,45 @@ private RestResponse deleteVariantSecondaryIndex() throws Exception { return openCGAClient.getVariantOperationClient().deleteVariantSecondaryIndex(queryParams); } + private RestResponse setupVariant() throws Exception { + logger.debug("Executing setupVariant in Operations - Variant Storage command line"); + + OperationsVariantStorageCommandOptions.SetupVariantCommandOptions commandOptions = operationsVariantStorageCommandOptions.setupVariantCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + VariantSetupParams variantSetupParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/operation/variant/setup")); + return res; + } else if (commandOptions.jsonFile != null) { + variantSetupParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), VariantSetupParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "expectedSamples", commandOptions.expectedSamples, true); + putNestedIfNotNull(beanParams, "expectedFiles", commandOptions.expectedFiles, true); + putNestedIfNotNull(beanParams, "fileType", commandOptions.fileType, true); + putNestedIfNotEmpty(beanParams, "averageFileSize", commandOptions.averageFileSize, true); + putNestedIfNotNull(beanParams, "variantsPerSample", commandOptions.variantsPerSample, true); + putNestedIfNotNull(beanParams, "averageSamplesPerFile", commandOptions.averageSamplesPerFile, true); + putNestedIfNotNull(beanParams, "dataDistribution", commandOptions.dataDistribution, true); + putNestedIfNotNull(beanParams, "normalizeExtensions", commandOptions.normalizeExtensions, true); + + variantSetupParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), VariantSetupParams.class); + } + return openCGAClient.getVariantOperationClient().setupVariant(variantSetupParams, queryParams); + } + private RestResponse deleteVariantStats() throws Exception { logger.debug("Executing deleteVariantStats in Operations - Variant Storage command line"); @@ -1176,8 +1220,8 @@ private RestResponse deleteVariantStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); variantStatsDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1216,12 +1260,12 @@ private RestResponse indexVariantStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "overwriteStats",commandOptions.overwriteStats, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "overwriteStats", commandOptions.overwriteStats, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); variantStatsIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1260,7 +1304,7 @@ private RestResponse deleteVariantStudy() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStudyDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantStudyDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index c0ce6156d34..1294b1397e7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -12,6 +12,7 @@ import org.opencb.opencga.app.cli.main.options.OrganizationsCommandOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.UpdateAction; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; @@ -126,12 +127,12 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), OrganizationCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "configuration.defaultUserExpirationDate",commandOptions.configurationDefaultUserExpirationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "configuration.defaultUserExpirationDate", commandOptions.configurationDefaultUserExpirationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -162,10 +163,10 @@ private RestResponse createNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); - putNestedIfNotNull(beanParams, "valueType",commandOptions.valueType, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); noteCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -214,6 +215,7 @@ private RestResponse updateNotes() throws Exception { ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("tagsAction", commandOptions.tagsAction); queryParams.putIfNotNull("includeResult", commandOptions.includeResult); @@ -228,8 +230,8 @@ private RestResponse updateNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); noteUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -261,7 +263,7 @@ private RestResponse userUpdateStatus() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserStatusUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "status",commandOptions.status, true); + putNestedIfNotEmpty(beanParams, "status", commandOptions.status, true); userStatusUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -293,14 +295,13 @@ private RestResponse updateUser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), OrganizationUserUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotNull(beanParams, "quota.diskUsage",commandOptions.quotaDiskUsage, true); - putNestedIfNotNull(beanParams, "quota.cpuUsage",commandOptions.quotaCpuUsage, true); - putNestedIfNotNull(beanParams, "quota.maxDisk",commandOptions.quotaMaxDisk, true); - putNestedIfNotNull(beanParams, "quota.maxCpu",commandOptions.quotaMaxCpu, true); - putNestedIfNotEmpty(beanParams, "account.expirationDate",commandOptions.accountExpirationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotNull(beanParams, "quota.diskUsage", commandOptions.quotaDiskUsage, true); + putNestedIfNotNull(beanParams, "quota.cpuUsage", commandOptions.quotaCpuUsage, true); + putNestedIfNotNull(beanParams, "quota.maxDisk", commandOptions.quotaMaxDisk, true); + putNestedIfNotNull(beanParams, "quota.maxCpu", commandOptions.quotaMaxCpu, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -332,11 +333,11 @@ private RestResponse updateConfiguration() throws Exc .readValue(new java.io.File(commandOptions.jsonFile), OrganizationConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "defaultUserExpirationDate",commandOptions.defaultUserExpirationDate, true); - putNestedIfNotNull(beanParams, "optimizations.simplifyPermissions",commandOptions.optimizationsSimplifyPermissions, true); - putNestedIfNotEmpty(beanParams, "token.algorithm",commandOptions.tokenAlgorithm, true); - putNestedIfNotEmpty(beanParams, "token.secretKey",commandOptions.tokenSecretKey, true); - putNestedIfNotNull(beanParams, "token.expiration",commandOptions.tokenExpiration, true); + putNestedIfNotEmpty(beanParams, "defaultUserExpirationDate", commandOptions.defaultUserExpirationDate, true); + putNestedIfNotNull(beanParams, "optimizations.simplifyPermissions", commandOptions.optimizationsSimplifyPermissions, true); + putNestedIfNotEmpty(beanParams, "token.algorithm", commandOptions.tokenAlgorithm, true); + putNestedIfNotEmpty(beanParams, "token.secretKey", commandOptions.tokenSecretKey, true); + putNestedIfNotNull(beanParams, "token.expiration", commandOptions.tokenExpiration, true); organizationConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -380,12 +381,12 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), OrganizationUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "owner",commandOptions.owner, true); - putNestedIfNotNull(beanParams, "admins",commandOptions.admins, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "owner", commandOptions.owner, true); + putNestedIfNotNull(beanParams, "admins", commandOptions.admins, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java index df4a22211a3..929d89e1e4e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java @@ -104,19 +104,19 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ProjectCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "organism.scientificName",commandOptions.organismScientificName, true); - putNestedIfNotEmpty(beanParams, "organism.commonName",commandOptions.organismCommonName, true); - putNestedIfNotEmpty(beanParams, "organism.assembly",commandOptions.organismAssembly, true); - putNestedIfNotEmpty(beanParams, "cellbase.url",commandOptions.cellbaseUrl, true); - putNestedIfNotEmpty(beanParams, "cellbase.version",commandOptions.cellbaseVersion, true); - putNestedIfNotEmpty(beanParams, "cellbase.dataRelease",commandOptions.cellbaseDataRelease, true); - putNestedIfNotEmpty(beanParams, "cellbase.apiKey",commandOptions.cellbaseApiKey, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "organism.scientificName", commandOptions.organismScientificName, true); + putNestedIfNotEmpty(beanParams, "organism.commonName", commandOptions.organismCommonName, true); + putNestedIfNotEmpty(beanParams, "organism.assembly", commandOptions.organismAssembly, true); + putNestedIfNotEmpty(beanParams, "cellbase.url", commandOptions.cellbaseUrl, true); + putNestedIfNotEmpty(beanParams, "cellbase.version", commandOptions.cellbaseVersion, true); + putNestedIfNotEmpty(beanParams, "cellbase.dataRelease", commandOptions.cellbaseDataRelease, true); + putNestedIfNotEmpty(beanParams, "cellbase.apiKey", commandOptions.cellbaseApiKey, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); projectCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -208,14 +208,14 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ProjectUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "organism.scientificName",commandOptions.organismScientificName, true); - putNestedIfNotEmpty(beanParams, "organism.commonName",commandOptions.organismCommonName, true); - putNestedIfNotEmpty(beanParams, "organism.assembly",commandOptions.organismAssembly, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "organism.scientificName", commandOptions.organismScientificName, true); + putNestedIfNotEmpty(beanParams, "organism.commonName", commandOptions.organismCommonName, true); + putNestedIfNotEmpty(beanParams, "organism.assembly", commandOptions.organismAssembly, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); projectUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java index 2aaf0238931..c8e02456a4a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java @@ -133,12 +133,12 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); sampleAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -172,7 +172,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -207,32 +207,32 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "individualId",commandOptions.individualId, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.description",commandOptions.sourceDescription, true); - putNestedIfNotEmpty(beanParams, "source.source",commandOptions.sourceSource, true); - putNestedIfNotEmpty(beanParams, "source.url",commandOptions.sourceUrl, true); - putNestedIfNotEmpty(beanParams, "processing.preparationMethod",commandOptions.processingPreparationMethod, true); - putNestedIfNotEmpty(beanParams, "processing.extractionMethod",commandOptions.processingExtractionMethod, true); - putNestedIfNotEmpty(beanParams, "processing.labSampleId",commandOptions.processingLabSampleId, true); - putNestedIfNotEmpty(beanParams, "processing.quantity",commandOptions.processingQuantity, true); - putNestedIfNotEmpty(beanParams, "processing.date",commandOptions.processingDate, true); - putNestedIfNotNull(beanParams, "processing.attributes",commandOptions.processingAttributes, true); - putNestedIfNotEmpty(beanParams, "collection.type",commandOptions.collectionType, true); - putNestedIfNotEmpty(beanParams, "collection.quantity",commandOptions.collectionQuantity, true); - putNestedIfNotEmpty(beanParams, "collection.method",commandOptions.collectionMethod, true); - putNestedIfNotEmpty(beanParams, "collection.date",commandOptions.collectionDate, true); - putNestedIfNotNull(beanParams, "collection.attributes",commandOptions.collectionAttributes, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "individualId", commandOptions.individualId, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.description", commandOptions.sourceDescription, true); + putNestedIfNotEmpty(beanParams, "source.source", commandOptions.sourceSource, true); + putNestedIfNotEmpty(beanParams, "source.url", commandOptions.sourceUrl, true); + putNestedIfNotEmpty(beanParams, "processing.preparationMethod", commandOptions.processingPreparationMethod, true); + putNestedIfNotEmpty(beanParams, "processing.extractionMethod", commandOptions.processingExtractionMethod, true); + putNestedIfNotEmpty(beanParams, "processing.labSampleId", commandOptions.processingLabSampleId, true); + putNestedIfNotEmpty(beanParams, "processing.quantity", commandOptions.processingQuantity, true); + putNestedIfNotEmpty(beanParams, "processing.date", commandOptions.processingDate, true); + putNestedMapIfNotEmpty(beanParams, "processing.attributes", commandOptions.processingAttributes, true); + putNestedIfNotEmpty(beanParams, "collection.type", commandOptions.collectionType, true); + putNestedIfNotEmpty(beanParams, "collection.quantity", commandOptions.collectionQuantity, true); + putNestedIfNotEmpty(beanParams, "collection.method", commandOptions.collectionMethod, true); + putNestedIfNotEmpty(beanParams, "collection.date", commandOptions.collectionDate, true); + putNestedMapIfNotEmpty(beanParams, "collection.attributes", commandOptions.collectionAttributes, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); sampleCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -444,33 +444,33 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "individualId",commandOptions.individualId, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.description",commandOptions.sourceDescription, true); - putNestedIfNotEmpty(beanParams, "source.source",commandOptions.sourceSource, true); - putNestedIfNotEmpty(beanParams, "source.url",commandOptions.sourceUrl, true); - putNestedIfNotEmpty(beanParams, "processing.preparationMethod",commandOptions.processingPreparationMethod, true); - putNestedIfNotEmpty(beanParams, "processing.extractionMethod",commandOptions.processingExtractionMethod, true); - putNestedIfNotEmpty(beanParams, "processing.labSampleId",commandOptions.processingLabSampleId, true); - putNestedIfNotEmpty(beanParams, "processing.quantity",commandOptions.processingQuantity, true); - putNestedIfNotEmpty(beanParams, "processing.date",commandOptions.processingDate, true); - putNestedIfNotNull(beanParams, "processing.attributes",commandOptions.processingAttributes, true); - putNestedIfNotEmpty(beanParams, "collection.type",commandOptions.collectionType, true); - putNestedIfNotEmpty(beanParams, "collection.quantity",commandOptions.collectionQuantity, true); - putNestedIfNotEmpty(beanParams, "collection.method",commandOptions.collectionMethod, true); - putNestedIfNotEmpty(beanParams, "collection.date",commandOptions.collectionDate, true); - putNestedIfNotNull(beanParams, "collection.attributes",commandOptions.collectionAttributes, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "individualId", commandOptions.individualId, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.description", commandOptions.sourceDescription, true); + putNestedIfNotEmpty(beanParams, "source.source", commandOptions.sourceSource, true); + putNestedIfNotEmpty(beanParams, "source.url", commandOptions.sourceUrl, true); + putNestedIfNotEmpty(beanParams, "processing.preparationMethod", commandOptions.processingPreparationMethod, true); + putNestedIfNotEmpty(beanParams, "processing.extractionMethod", commandOptions.processingExtractionMethod, true); + putNestedIfNotEmpty(beanParams, "processing.labSampleId", commandOptions.processingLabSampleId, true); + putNestedIfNotEmpty(beanParams, "processing.quantity", commandOptions.processingQuantity, true); + putNestedIfNotEmpty(beanParams, "processing.date", commandOptions.processingDate, true); + putNestedMapIfNotEmpty(beanParams, "processing.attributes", commandOptions.processingAttributes, true); + putNestedIfNotEmpty(beanParams, "collection.type", commandOptions.collectionType, true); + putNestedIfNotEmpty(beanParams, "collection.quantity", commandOptions.collectionQuantity, true); + putNestedIfNotEmpty(beanParams, "collection.method", commandOptions.collectionMethod, true); + putNestedIfNotEmpty(beanParams, "collection.date", commandOptions.collectionDate, true); + putNestedMapIfNotEmpty(beanParams, "collection.attributes", commandOptions.collectionAttributes, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); sampleUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 2e8bd7b0040..0c18639ed52 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -178,9 +178,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotEmpty(beanParams, "template",commandOptions.template, true); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotEmpty(beanParams, "template", commandOptions.template, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); studyAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -212,18 +212,18 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "alias",commandOptions.alias, true); - putNestedIfNotEmpty(beanParams, "type.id",commandOptions.typeId, true); - putNestedIfNotEmpty(beanParams, "type.description",commandOptions.typeDescription, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "alias", commandOptions.alias, true); + putNestedIfNotEmpty(beanParams, "type.id", commandOptions.typeId, true); + putNestedIfNotEmpty(beanParams, "type.description", commandOptions.typeDescription, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); studyCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -336,8 +336,8 @@ private RestResponse updateGroups() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "users",commandOptions.users, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "users", commandOptions.users, true); groupCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -366,7 +366,7 @@ private RestResponse updateGroupsUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "users",commandOptions.users, true); + putNestedIfNotNull(beanParams, "users", commandOptions.users, true); groupUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -397,10 +397,10 @@ private RestResponse createNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); - putNestedIfNotNull(beanParams, "valueType",commandOptions.valueType, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); noteCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -448,6 +448,7 @@ private RestResponse updateNotes() throws Exception { ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("tagsAction", commandOptions.tagsAction); queryParams.putIfNotNull("includeResult", commandOptions.includeResult); @@ -462,8 +463,8 @@ private RestResponse updateNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visibility",commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); noteUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -499,10 +500,10 @@ private RestResponse updatePermissionRules() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PermissionRule.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "query",commandOptions.query, true); - putNestedIfNotNull(beanParams, "members",commandOptions.members, true); - putNestedIfNotNull(beanParams, "permissions",commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedMapIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotNull(beanParams, "members", commandOptions.members, true); + putNestedIfNotNull(beanParams, "permissions", commandOptions.permissions, true); permissionRule = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -577,17 +578,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "alias",commandOptions.alias, true); - putNestedIfNotEmpty(beanParams, "type.id",commandOptions.typeId, true); - putNestedIfNotEmpty(beanParams, "type.description",commandOptions.typeDescription, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "alias", commandOptions.alias, true); + putNestedIfNotEmpty(beanParams, "type.id", commandOptions.typeId, true); + putNestedIfNotEmpty(beanParams, "type.description", commandOptions.typeDescription, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); studyUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -627,11 +628,11 @@ private RestResponse updateVariableSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariableSetCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "unique",commandOptions.unique, true); - putNestedIfNotNull(beanParams, "confidential",commandOptions.confidential, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "unique", commandOptions.unique, true); + putNestedIfNotNull(beanParams, "confidential", commandOptions.confidential, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); variableSetCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -660,18 +661,18 @@ private RestResponse updateVariableSetsVariables() throws Exception .readValue(new java.io.File(commandOptions.jsonFile), Variable.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "category",commandOptions.category, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotNull(beanParams, "required",commandOptions.required, true); - putNestedIfNotNull(beanParams, "multiValue",commandOptions.multiValue, true); - putNestedIfNotNull(beanParams, "allowedValues",commandOptions.allowedValues, true); - putNestedIfNotNull(beanParams, "allowedKeys",commandOptions.allowedKeys, true); - putNestedIfNotNull(beanParams, "rank",commandOptions.rank, true); - putNestedIfNotEmpty(beanParams, "dependsOn",commandOptions.dependsOn, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "category", commandOptions.category, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "required", commandOptions.required, true); + putNestedIfNotNull(beanParams, "multiValue", commandOptions.multiValue, true); + putNestedIfNotNull(beanParams, "allowedValues", commandOptions.allowedValues, true); + putNestedIfNotNull(beanParams, "allowedKeys", commandOptions.allowedKeys, true); + putNestedIfNotNull(beanParams, "rank", commandOptions.rank, true); + putNestedIfNotEmpty(beanParams, "dependsOn", commandOptions.dependsOn, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); variable = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java index 4883cedef5c..5f8dfce148c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java @@ -136,11 +136,11 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "organization", commandOptions.organization, true); userCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -178,11 +178,11 @@ private RestResponse password() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PasswordChangeParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "organizationId",commandOptions.organizationId, true); - putNestedIfNotEmpty(beanParams, "user",commandOptions.user, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "newPassword",commandOptions.newPassword, true); - putNestedIfNotEmpty(beanParams, "reset",commandOptions.reset, true); + putNestedIfNotEmpty(beanParams, "organizationId", commandOptions.organizationId, true); + putNestedIfNotEmpty(beanParams, "user", commandOptions.user, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "newPassword", commandOptions.newPassword, true); + putNestedIfNotEmpty(beanParams, "reset", commandOptions.reset, true); passwordChangeParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -253,8 +253,8 @@ private RestResponse updateConfigs() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ConfigUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "configuration",commandOptions.configuration, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedMapIfNotEmpty(beanParams, "configuration", commandOptions.configuration, true); configUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -303,8 +303,8 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); userUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index 7a80b5b62dc..b9dc5e1956a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -241,8 +241,8 @@ public class CreateCommandOptions { @Parameter(names = {"--family-id"}, description = "The body web service id parameter", required = false, arity = 1) public String familyId; - @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) - public Boolean panelLock; + @Parameter(names = {"--panel-locked"}, description = "The body web service panelLocked parameter", required = false, arity = 1) + public Boolean panelLocked; @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; @@ -301,6 +301,9 @@ public class CreateCommandOptions { @Parameter(names = {"--responsible-postcode"}, description = "The body web service postcode parameter", required = false, arity = 1) public String responsiblePostcode; + @Parameter(names = {"--interpretation-name"}, description = "The body web service name parameter", required = false, arity = 1) + public String interpretationName; + @Parameter(names = {"--interpretation-description"}, description = "The body web service description parameter", required = false, arity = 1) public String interpretationDescription; @@ -423,6 +426,9 @@ public class DistinctCommandOptions { @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) public String release; + @Parameter(names = {"--snapshot"}, description = "Snapshot value (Latest version of the entry in the specified release)", required = false, arity = 1) + public Integer snapshot; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) public String status; @@ -455,6 +461,9 @@ public class DistinctInterpretationCommandOptions { @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) public String uuid; + @Parameter(names = {"--name", "-n"}, description = "Comma separated list of Interpretation names up to a maximum of 100", required = false, arity = 1) + public String name; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) public String clinicalAnalysisId; @@ -523,6 +532,9 @@ public class SearchInterpretationCommandOptions { @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) public String uuid; + @Parameter(names = {"--name", "-n"}, description = "Comma separated list of Interpretation names up to a maximum of 100", required = false, arity = 1) + public String name; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) public String clinicalAnalysisId; @@ -667,9 +679,12 @@ public class RunInterpreterExomiserCommandOptions { @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) public Boolean jobDryRun; - @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = false, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + public String exomiserVersion; + } @Parameters(commandNames = {"interpreter-team-run"}, commandDescription ="Run TEAM interpretation analysis") @@ -1737,6 +1752,9 @@ public class SearchCommandOptions { @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) public String release; + @Parameter(names = {"--snapshot"}, description = "Snapshot value (Latest version of the entry in the specified release)", required = false, arity = 1) + public Integer snapshot; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) public String status; @@ -2024,8 +2042,8 @@ public class UpdateCommandOptions { @Parameter(names = {"--disorder-id"}, description = "The body web service id parameter", required = false, arity = 1) public String disorderId; - @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) - public Boolean panelLock; + @Parameter(names = {"--panel-locked"}, description = "The body web service panelLocked parameter", required = false, arity = 1) + public Boolean panelLocked; @Parameter(names = {"--proband-id"}, description = "The body web service id parameter", required = false, arity = 1) public String probandId; @@ -2120,6 +2138,9 @@ public class UpdateCommandOptions { @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; + @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) + public Boolean panelLock; + } @Parameters(commandNames = {"annotation-sets-annotations-update"}, commandDescription ="Update annotations from an annotationSet") @@ -2169,6 +2190,9 @@ public class InfoCommandOptions { @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; + @Parameter(names = {"--version"}, description = "Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if multiple clinical ids are provided", required = false, arity = 1) + public String version; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) public boolean deleted = false; @@ -2204,6 +2228,9 @@ public class CreateInterpretationCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; @@ -2341,6 +2368,9 @@ public class UpdateInterpretationCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 1c83e822d82..44a0ad64dd7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -431,6 +431,9 @@ public class RunExomiserCommandOptions { @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + public String exomiserVersion; + @Parameter(names = {"--clinical-analysis-type"}, description = "Clinical analysis type: SINGLE or FAMILY.", required = false, arity = 1) public String clinicalAnalysisType = "SINGLE"; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index 4d398070124..6f932901d33 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -58,6 +58,7 @@ public class OperationsVariantStorageCommandOptions { public ConfigureVariantSecondarySampleIndexCommandOptions configureVariantSecondarySampleIndexCommandOptions; public SecondaryIndexVariantCommandOptions secondaryIndexVariantCommandOptions; public DeleteVariantSecondaryIndexCommandOptions deleteVariantSecondaryIndexCommandOptions; + public SetupVariantCommandOptions setupVariantCommandOptions; public DeleteVariantStatsCommandOptions deleteVariantStatsCommandOptions; public IndexVariantStatsCommandOptions indexVariantStatsCommandOptions; public DeleteVariantStudyCommandOptions deleteVariantStudyCommandOptions; @@ -92,6 +93,7 @@ public OperationsVariantStorageCommandOptions(CommonCommandOptions commonCommand this.configureVariantSecondarySampleIndexCommandOptions = new ConfigureVariantSecondarySampleIndexCommandOptions(); this.secondaryIndexVariantCommandOptions = new SecondaryIndexVariantCommandOptions(); this.deleteVariantSecondaryIndexCommandOptions = new DeleteVariantSecondaryIndexCommandOptions(); + this.setupVariantCommandOptions = new SetupVariantCommandOptions(); this.deleteVariantStatsCommandOptions = new DeleteVariantStatsCommandOptions(); this.indexVariantStatsCommandOptions = new IndexVariantStatsCommandOptions(); this.deleteVariantStudyCommandOptions = new DeleteVariantStudyCommandOptions(); @@ -380,13 +382,13 @@ public class DeleteVariantCommandOptions { @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--file"}, description = "List of file ids to delete. Use 'all' to remove the whole study", required = false, arity = 1) public String file; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume failed delete operation.", required = false, help = true, arity = 0) public boolean resume = false; - @Parameter(names = {"--force"}, description = "The body web service force parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--force"}, description = "Force delete operation. This would allow deleting partially loaded files.", required = false, help = true, arity = 0) public boolean force = false; } @@ -1360,6 +1362,47 @@ public class DeleteVariantSecondaryIndexCommandOptions { } + @Parameters(commandNames = {"variant-setup"}, commandDescription ="Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation.") + public class SetupVariantCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--expected-samples"}, description = "Expected number of samples that will be loaded. Used to infer some parameters. This number is only used as a hint. If the real number of samples is different, if it grows beyond expectation, or if , the loader should be able to handle it.", required = false, arity = 1) + public Integer expectedSamples; + + @Parameter(names = {"--expected-files"}, description = "Expected number of files that will be loaded. Used to infer some parameters. This number is only used as a hint. If the real number of files is different, the loader should be able to handle it.", required = false, arity = 1) + public Integer expectedFiles; + + @Parameter(names = {"--file-type"}, description = "Main type of the files that will be loaded. If the dataset contains multiple types of files, provide the one that matches most of the files.", required = false, arity = 1) + public String fileType; + + @Parameter(names = {"--average-file-size"}, description = "Average size of the files that will be loaded. This number is only used as a hint. If the real size of the files is different, the loader should be able to handle it. Accepts units. e.g. 435MB, 2GB, 86KB. If not provided, the value will be inferred from the file type.", required = false, arity = 1) + public String averageFileSize; + + @Parameter(names = {"--variants-per-sample"}, description = "Number of variants per sample (non hom_ref variants). This number is only used as a hint. If the real number of variants per sample is different, the loader should be able to handle it. If not provided, the value will be inferred from the file type.", required = false, arity = 1) + public Integer variantsPerSample; + + @Parameter(names = {"--average-samples-per-file"}, description = "Average number of samples per file. This number is only used as a hint. If the real number of samples per file is different, the loader should be able to handle it. If not provided, the value will be inferred from the expectedSamples and expectedFiles and dataDistribution.", required = false, arity = 1) + public Float averageSamplesPerFile; + + @Parameter(names = {"--data-distribution"}, description = "Data distribution of the files. This parameter is used to infer the number of samples per file.", required = false, arity = 1) + public String dataDistribution; + + @Parameter(names = {"--normalize-extensions"}, description = "List of normalization extensions that will be used to normalize the files.", required = false, arity = 1) + public String normalizeExtensions; + + } + @Parameters(commandNames = {"variant-stats-delete"}, commandDescription ="Deletes the VariantStats of a cohort/s from the database") public class DeleteVariantStatsCommandOptions { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java index 68be35b6cac..0f9f2a18c49 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java @@ -214,6 +214,9 @@ public class UpdateNotesCommandOptions { @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) public String id; + @Parameter(names = {"--tags-action"}, description = "Action to be performed if the array of tags is being updated.", required = false, arity = 1) + public String tagsAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; @@ -302,9 +305,6 @@ public class UpdateUserCommandOptions { @Parameter(names = {"--quota-max-cpu"}, description = "The body web service maxCpu parameter", required = false, arity = 1) public Integer quotaMaxCpu; - @Parameter(names = {"--account-expiration-date"}, description = "The body web service expirationDate parameter", required = false, arity = 1) - public String accountExpirationDate; - @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java index 4e5cf838199..d75022de0b9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java @@ -505,6 +505,9 @@ public class UpdateNotesCommandOptions { @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) public String id; + @Parameter(names = {"--tags-action"}, description = "Action to be performed if the array of tags is being updated.", required = false, arity = 1) + public String tagsAction = "ADD"; + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java index c9aeee6c7e2..4952f9d9812 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java @@ -46,10 +46,10 @@ protected final VariantStorageEngine getVariantStorageEngineByProject(String pro * @return List of projects * @throws Exception on error */ - protected final List getVariantStorageProjects(String organizationId) throws Exception { + protected final List getVariantStorageProjects() throws Exception { Set projects = new LinkedHashSet<>(); - for (String studyFqn : getVariantStorageStudies(organizationId)) { + for (String studyFqn : getVariantStorageStudies()) { projects.add(catalogManager.getStudyManager().getProjectFqn(studyFqn)); } @@ -61,7 +61,7 @@ protected final List getVariantStorageProjects(String organizationId) th * @return List of projects * @throws Exception on error */ - protected final List getVariantStorageStudies(String organizationId) throws Exception { + protected final List getVariantStorageStudies() throws Exception { Set studies = new LinkedHashSet<>(); VariantStorageManager variantStorageManager = getVariantStorageManager(); for (Study study : catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), new QueryOptions(QueryOptions.INCLUDE, diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java similarity index 97% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java index ae73bcdc578..e1953673252 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_4.catalog; +package org.opencb.opencga.app.migrations.v2.v2_12_4.catalog; import org.bson.Document; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java similarity index 89% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java index 4efa260d965..d641207a04e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.opencga.app.migrations.StorageMigrationTool; import org.opencb.opencga.catalog.migration.Migration; @@ -11,7 +11,7 @@ public class AddAllelesColumnToPhoenix extends StorageMigrationTool { @Override protected void run() throws Exception { - for (String project : getVariantStorageProjects(organizationId)) { + for (String project : getVariantStorageProjects()) { VariantStorageEngine engine = getVariantStorageEngineByProject(project); if (engine.getStorageEngineId().equals("hadoop")) { logger.info("Adding missing columns (if any) for project " + project); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java similarity index 99% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java index 132e1357f1b..aba2f366b77 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -42,7 +42,7 @@ public class DetectIllegalConcurrentFileLoadingsMigration extends StorageMigrati @Override protected void run() throws Exception { - for (String project : getVariantStorageProjects(organizationId)) { + for (String project : getVariantStorageProjects()) { VariantStorageEngine engine = getVariantStorageEngineByProject(project); if (!engine.getStorageEngineId().equals("hadoop")) { continue; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java new file mode 100644 index 00000000000..372130c0a5e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java @@ -0,0 +1,35 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "add_interpretation_name", description = "Add Interpretation name #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) +public class AddInterpretationName extends MigrationTool { + + @Override + protected void run() throws Exception { + // Add new Interpretation name field + Bson nameDoesNotExistQuery = Filters.exists("name", false); + Bson projection = Projections.include("id"); + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + migrateCollection(collection, nameDoesNotExistQuery, projection, + (document, bulk) -> { + Document updateDocument = new Document() + .append("name", document.get("id")); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); + }); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java new file mode 100644 index 00000000000..775cf2c64d7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java @@ -0,0 +1,112 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.clinical.ClinicalStatus; +import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; + +import java.util.Arrays; +import java.util.List; + +@Migration(id = "add_new_clinical_status", + description = "Add new ClinicalStatus to ClinicalAnalysis and Interpretation, #TASK-5964", + version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240611) +public class AddNewClinicalStatusValues extends MigrationTool { + + @Override + protected void run() throws Exception { + ClinicalAnalysisStudyConfiguration defaultConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + + Bson query = Filters.exists("status.author", false); + Bson projection = Projections.include("status", "id"); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + String clinicalId = document.getString("id"); + Document status = document.get("status", Document.class); + + Document newStatus = fillStatusValues("Clinical Analysis", clinicalId, status, defaultConfiguration.getStatus()); + updateDocument.getSet().put("status", newStatus); + + Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); + + logger.debug("Updating clinical analysis '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); + }); + } + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + String interpretationId = document.getString("id"); + Document status = document.get("status", Document.class); + + Document newStatus = fillStatusValues("Interpretation", interpretationId, status, + defaultConfiguration.getInterpretation().getStatus()); + updateDocument.getSet().put("status", newStatus); + + Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); + + logger.debug("Updating interpretation '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); + }); + } + } + + private Document fillStatusValues(String entity, String id, Document status, List statusValueList) { + ClinicalStatus clinicalStatus = new ClinicalStatus(); + String clinicalStatusId = status != null ? status.getString("id") : null; + if (status == null || StringUtils.isEmpty(clinicalStatusId)) { + logger.warn("Status is empty or does not contain 'id' field. Setting default status value for {} '{}'", entity, id); + for (ClinicalStatusValue clinicalStatusValue : statusValueList) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { + clinicalStatus.setId(clinicalStatusValue.getId()); + clinicalStatus.setDescription(clinicalStatusValue.getDescription()); + clinicalStatus.setType(clinicalStatusValue.getType()); + } + } + } else { + for (ClinicalStatusValue clinicalStatusValue : statusValueList) { + if (clinicalStatusValue.getId().equals(clinicalStatusId)) { + clinicalStatus.setId(clinicalStatusValue.getId()); + clinicalStatus.setDescription(clinicalStatusValue.getDescription()); + clinicalStatus.setType(clinicalStatusValue.getType()); + break; + } + } + if (clinicalStatus.getType() == null) { + logger.warn("Status '{}' not found in the list of available status values. Keeping original status for {} '{}'", + clinicalStatusId, entity, id); + clinicalStatus.setId(clinicalStatusId); + clinicalStatus.setDescription(status.getString("description")); + } + } + clinicalStatus.setDate(TimeUtils.getTime()); + clinicalStatus.setVersion(GitRepositoryState.getInstance().getBuildVersion()); + clinicalStatus.setCommit(GitRepositoryState.getInstance().getCommitId()); + clinicalStatus.setAuthor("opencga"); + + Document document = convertToDocument(clinicalStatus); + return document; + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java new file mode 100644 index 00000000000..302637cf968 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java @@ -0,0 +1,62 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.collections4.CollectionUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Collections; +import java.util.List; + +@Migration(id = "add_version_to_clinicalAnalysis", description = "Add version to Clinical Analysis #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240527) +public class AddVersionToClinicalAnalysisMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // Add missing mandatory "versioning" fields to Clinical Analysis + logger.info("Adding missing 'versioning' fields to Clinical Analysis..."); + Bson versionDoesNotExistQuery = Filters.exists("version", false); + Bson projection = Projections.include("release", "interpretation", "secondaryInterpretations"); + migrateCollection(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, versionDoesNotExistQuery, projection, + (document, bulk) -> { + int version = 1; + Document interpretation = document.get("interpretation", Document.class); + if (interpretation != null) { + int interpretationVersion = interpretation.get("version", Number.class).intValue(); + version = Math.max(version, interpretationVersion + 1); + } + List secondaryInterpretations = document.getList("secondaryInterpretations", Document.class); + if (CollectionUtils.isNotEmpty(secondaryInterpretations)) { + for (Document secondaryInterpretation : secondaryInterpretations) { + int secondaryInterpretationVersion = secondaryInterpretation.get("version", Number.class).intValue(); + version = Math.max(version, secondaryInterpretationVersion + 1); + } + } + int release = document.get("release", Number.class).intValue(); + Document updateDocument = new Document() + .append("version", version) + .append("_releaseFromVersion", Collections.singletonList(release)) + .append("_lastOfVersion", true) + .append("_lastOfRelease", true); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); + }); + + // Recalculate indexes + logger.info("Installing new indexes..."); + catalogManager.installIndexes(organizationId, token); + + // Copy all Clinical Analyses to the archive collection + logger.info("Copying all the data from the main collection to the archive one..."); + copyData(new Document(), OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); + + + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java new file mode 100644 index 00000000000..80692ae910c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java @@ -0,0 +1,114 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Migration(id = "remove_status_name", description = "Remove status name #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240612) +public class RemoveStatusNameMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // Remove from user + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from project + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_PROJECT_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from study + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from sample + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", + "internal.variant.sampleGenotypeIndex.status.name", "internal.variant.sampleGenotypeIndex.familyStatus.name", + "internal.variant.secondarySampleIndex.status.name", "internal.variant.secondarySampleIndex.familyStatus.name", + "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from file + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_FILE_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", + "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name", + "internal.alignment.index.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from individual + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INDIVIDUAL_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from family + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_FAMILY_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from cohort + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_COHORT_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from panel + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, + OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_PANEL_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from job + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from clinical + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from interpretation + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + } + + private void removeStatusField(String collection, List statusList) throws CatalogDBException { + logger.info("Removing status.name fields from collection {}...", collection); + List exists = statusList.stream().map(Filters::exists).collect(Collectors.toList()); + List unsetList = statusList.stream().map(Updates::unset).collect(Collectors.toList()); + getMongoCollection(collection).updateMany(Filters.or(exists), Updates.combine(unsetList)); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java new file mode 100644 index 00000000000..32c10c3fab7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java @@ -0,0 +1,26 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "rename_panel_lock_from_clinical__task_5964", description = "Rename 'panelLock' to 'panelLocked' #TASK-5964", + version = "3.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240710) +public class RenamePanelLockMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = new Document(); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + getMongoCollection(collection).updateMany(query, new Document("$rename", new Document("panelLock", "panelLocked"))); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java new file mode 100644 index 00000000000..35960978d4c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java @@ -0,0 +1,68 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationRuntimeException; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; + +import java.util.Arrays; +import java.util.List; + +@Migration(id = "update_clinical_study_configuration", + description = "Setting new default Clinical Study Configuration status values, #TASK-5964", + version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) +public class UpdateClinicalStudyConfiguration extends MigrationTool { + + @Override + protected void run() throws Exception { + ClinicalAnalysisStudyConfiguration clinicalAnalysisStudyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + Document clinicalConfigurationDocument = convertToDocument(clinicalAnalysisStudyConfiguration); + + Bson query = new Document(); + Bson projection = Projections.include("internal.configuration.clinical", "fqn"); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + Document internal = document.get("internal", Document.class); + if (internal == null) { + throw new MigrationRuntimeException("'internal' field not found in study '" + document.get("fqn") + "'"); + } + Document configuration = internal.get("configuration", Document.class); + if (configuration == null) { + throw new MigrationRuntimeException("'internal.configuration' field not found in study '" + document.get("fqn") + "'"); + } + Document clinicalDocument = configuration.get("clinical", Document.class); + + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + if (clinicalDocument == null) { + logger.warn("Found empty 'internal.configuration.clinical' field in study '{}'. Creating a new one...", document.get("fqn")); + updateDocument.getSet().put("internal.configuration.clinical", clinicalConfigurationDocument); + } else { + Object statusObject = clinicalDocument.get("status"); + if (statusObject instanceof List) { + // The study seems to be already migrated. Skipping... + logger.warn("Study '{}' seems to be already migrated. Skipping...", document.get("fqn")); + return; + } + // Study needs to be migrated + logger.info("Migrating study '{}'", document.get("fqn")); + updateDocument.getSet().put("internal.configuration.clinical.status", clinicalConfigurationDocument.get("status")); + updateDocument.getSet().put("internal.configuration.clinical.flags", clinicalConfigurationDocument.get("flags")); + updateDocument.getSet().put("internal.configuration.clinical.interpretation.status", clinicalConfigurationDocument.get("interpretation", Document.class).get("status")); + } + logger.debug("Updating study '{}': {}", document.get("fqn"), updateDocument.toFinalUpdateDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java new file mode 100644 index 00000000000..f7d6c03be9d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java @@ -0,0 +1,53 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; + +import java.util.LinkedHashSet; +import java.util.List; + +@Migration(id = "variant_setup", description = "Add a dummy variant setup for studies with data", version = "3.2.0", + domain = Migration.MigrationDomain.STORAGE, date = 20240516) +public class VariantSetupMigration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + VariantStorageManager variantStorageManager = getVariantStorageManager(); + List storageStudies = getVariantStorageStudies(); + if (storageStudies.isEmpty()) { + logger.info("No studies with variant storage found on organization '{}'", organizationId); + return; + } + for (String study : storageStudies) { + logger.info("--- Checking study '{}'", study); + if (variantStorageManager.hasVariantSetup(study, token)) { + logger.info("Study '{}' already has a variant setup", study); + continue; + } + + String projectFqn = catalogManager.getStudyManager().getProjectFqn(study); + VariantStorageMetadataManager metadataManager = getVariantStorageEngineByProject(projectFqn).getMetadataManager(); + int studyId = metadataManager.getStudyId(study); + LinkedHashSet indexedFiles = metadataManager.getIndexedFiles(studyId); + if (indexedFiles.isEmpty()) { + logger.info("Study '{}' does not have any indexed files. Skipping variant setup", study); + continue; + } + logger.info("Study '{}' doesn't have a variant setup, but it has {} indexed files. Creating a dummy variant setup", + study, indexedFiles.size()); + logger.info("Creating a dummy variant setup for study '{}'", study); + VariantSetupResult dummy = new VariantSetupResult(); + dummy.setDate(TimeUtils.getTime()); + dummy.setUserId(catalogManager.getUserManager().getUserId(token)); + dummy.setParams(new ObjectMap("executed_from_migration", getId())); + dummy.setStatus(VariantSetupResult.Status.READY); + dummy.setOptions(new ObjectMap()); + catalogManager.getStudyManager().setVariantEngineSetupOptions(study, dummy, token); + } + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java new file mode 100644 index 00000000000..f016e564c77 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java @@ -0,0 +1,45 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; +import java.util.Collections; + +import static org.opencb.opencga.catalog.db.mongodb.UserMongoDBAdaptor.*; + +@Migration(id = "add_archivePasswords_array", + description = "Add password history #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class AddPasswordHistoryMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists(PRIVATE_PASSWORD_ARCHIVE, false); + Bson projection = Projections.include(PRIVATE_PASSWORD); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + String currentPassword = document.getString("_password"); + + Document passwordDoc = new Document() + .append(HASH, currentPassword) + .append(SALT, ""); + Document privatePassword = new Document(); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + updateDocument.getSet().put(PRIVATE_PASSWORD, privatePassword); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java new file mode 100644 index 00000000000..68f2cc00e7c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java @@ -0,0 +1,50 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "move_user_account_to_internal", + description = "Move account to internal.account #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class MoveUserAccountToInternalMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists("account", true); + Bson projection = Projections.include("internal", "account"); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + Document account = document.get("account", Document.class); + Document internal = document.get("internal", Document.class); + internal.put("account", account); + + updateDocument.getSet().put("modificationDate", internal.get("lastModified")); + updateDocument.getSet().put("creationDate", account.get("creationDate")); + account.remove("creationDate"); + + Document password = new Document() + .append("expirationDate", null) + .append("lastModified", internal.get("lastModified")); + account.put("password", password); + account.put("failedAttempts", internal.get("failedAttempts")); + internal.remove("failedAttempts"); + + updateDocument.getSet().put("internal", internal); + updateDocument.getUnset().add("account"); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } +} diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java new file mode 100644 index 00000000000..57ee782a7cc --- /dev/null +++ b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java @@ -0,0 +1,89 @@ +package org.opencb.opencga.app.migrations; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.app.migrations.v3.v3_2_0.VariantSetupMigration; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; + +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; + +@Category(LongTests.class) +public class MigrationsTest { + + public OpenCGATestExternalResource opencga; + + @Test + public void testVariantSetupMigration() throws Exception { + setup("v3.0.0", false); + String studyName = "test@1000G:phase1"; + + VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject("test@1000G").getMetadataManager(); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); + StudyMetadata studyMetadata = metadataManager.createStudy(studyName); + int fileId = metadataManager.registerFile(studyMetadata.getId(), "folder/file.vcf", Arrays.asList("s1", "s2")); + metadataManager.addIndexedFiles(studyMetadata.getId(), Collections.singletonList(fileId)); + + for (Study study : opencga.getCatalogManager().getStudyManager().searchInOrganization("test", new Query(), new QueryOptions(), opencga.getAdminToken()).getResults()) { + Assert.assertNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } + + runMigration(VariantSetupMigration.class); + + for (Study study : opencga.getCatalogManager().getStudyManager().searchInOrganization("test", new Query(), new QueryOptions(), opencga.getAdminToken()).getResults()) { + if (study.getFqn().equals(studyName)) { + Assert.assertNotNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } else { + Assert.assertNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } + } + } + + @After + public void tearDown() throws Exception { + if (opencga != null) { + opencga.after(); + opencga = null; + } + } + + protected void testMigration(Class migration, String dataset) throws Exception { + setup(dataset); + runMigration(migration); + } + + private void setup(String dataset) throws Exception { + setup(dataset, false); + } + + private void setup(String dataset, boolean storageHadoop) throws Exception { + if (opencga != null) { + opencga.after(); + opencga = null; + } + opencga = new OpenCGATestExternalResource(storageHadoop); + opencga.before(); + URL resource = getClass().getResource("/datasets/opencga/" + dataset + "/"); + opencga.restore(resource); + } + + private void runMigration(Class migration) throws CatalogException { + Migration annotation = migration.getAnnotation(Migration.class); + opencga.getCatalogManager().getMigrationManager() + .runManualMigration(annotation.version(), annotation.id(), opencga.getOpencgaHome(), new ObjectMap(), opencga.getAdminToken()); + } + +} \ No newline at end of file diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz new file mode 100644 index 00000000000..2b44b35ae32 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz new file mode 100644 index 00000000000..cb73bece530 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/migration.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/migration.json.gz new file mode 100644 index 00000000000..58bb941a512 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/migration.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz new file mode 100644 index 00000000000..7500b56e11e Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz new file mode 100644 index 00000000000..57cafc8910a Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/project.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/project.json.gz new file mode 100644 index 00000000000..ef669b9c164 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/project.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz new file mode 100644 index 00000000000..e5aad572d01 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/user.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/user.json.gz new file mode 100644 index 00000000000..3425717560d Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/user.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/audit.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/audit.json.gz new file mode 100644 index 00000000000..9e783d0c02b Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/audit.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz new file mode 100644 index 00000000000..fff41de3126 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz new file mode 100644 index 00000000000..72820a3d31c Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz new file mode 100644 index 00000000000..ed972760e0b Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz new file mode 100644 index 00000000000..432b4097641 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz new file mode 100644 index 00000000000..9290eaeeff1 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz new file mode 100644 index 00000000000..41e2ae77fa8 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz new file mode 100644 index 00000000000..f8a2ba4f5fe Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz new file mode 100644 index 00000000000..0ea0384acdc Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz new file mode 100644 index 00000000000..aefbad4adcc Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz differ diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index fdaad5c12fc..df6a3d06d78 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -61,6 +61,11 @@ + + org.mockito + mockito-core + test + org.apache.commons commons-collections4 diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java index f104f8fdf93..6a3a0480c28 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java @@ -39,6 +39,7 @@ import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationProvider; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.user.*; @@ -382,8 +383,9 @@ private List extractUserInformation(List(), attributes); userList.add(user); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java index 65280a1be61..61f9fef8079 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java @@ -17,7 +17,6 @@ package org.opencb.opencga.catalog.auth.authentication; import io.jsonwebtoken.SignatureAlgorithm; -import org.apache.commons.lang3.RandomStringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; @@ -26,6 +25,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.MailUtils; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Email; import org.opencb.opencga.core.models.user.AuthenticationResponse; @@ -125,7 +125,7 @@ public String createNonExpiringToken(String organizationId, String userId, Map user = dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java index 35fe615d161..f79eb85b19d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java @@ -198,9 +198,11 @@ public List getRemoteUserInformation(List userStringList) throws C Map attributes = new HashMap<>(); attributes.put("LDAP_RDN", rdn); - User user = new User(uid, displayName, mail, usersSearch, new Account() - .setAuthentication(new Account.AuthenticationOrigin(originId, false)), new UserInternal(new UserStatus()), - new UserQuota(-1, -1, -1, -1), new ArrayList<>(), new HashMap<>(), new LinkedList<>(), attributes); + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(originId, false)); + User user = new User(uid, displayName, mail, usersSearch, TimeUtils.getTime(), TimeUtils.getTime(), + new UserInternal(new UserStatus(), account), + new UserQuota(-1, -1, -1, -1), new HashMap<>(), new LinkedList<>(), attributes); userList.add(user); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java index c86787e84df..09c142e68e9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java @@ -16,10 +16,8 @@ package org.opencb.opencga.catalog.auth.authorization; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.common.Enums; @@ -80,27 +78,21 @@ > OpenCGAResult> get(List resourceIds, L */ OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resource entry) throws CatalogException; - OpenCGAResult setToMembers(long studyId, List members, - List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setToMembers(long studyId, List members, List aclParams) + throws CatalogException; // Special method only to set acls in study - OpenCGAResult setToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException; - OpenCGAResult addToMembers(long studyId, List members, - List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addToMembers(long studyId, List members, List aclParams) + throws CatalogException; // Special method only to add acls in study - OpenCGAResult addToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException; - OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeFromMembers(List members, List aclParams) throws CatalogException; - OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException; OpenCGAResult setAcls(List resourceIds, AclEntryList aclEntryList, Enums.Resource resource) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java index 5f3367d4d1c..883943f1a4b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java @@ -19,10 +19,8 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.organizations.Organization; @@ -83,8 +81,7 @@ default String getCatalogDatabase(String prefix, String organization) { MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException; - OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) throws CatalogException; void deleteOrganization(Organization organization) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java index a06f3762b4d..ccca18225c9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java @@ -21,6 +21,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.study.Variable; @@ -62,12 +63,10 @@ OpenCGAResult update(Query query, ObjectMap parameters, List variab * @param variableSetId variable set id to identify the annotations that will add a new annotation. * @param variable new variable that will be added. * @return a OpenCGAResult object. - * @throws CatalogDBException if the variable could not be added to an existing annotationSet. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if the variable could not be added to an existing annotationSet, there is any unexpected parameter or + * the operation is not authorized. */ - OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException; // /** // * This method will rename the id of all the annotations corresponding to the variableSetId changing oldName per newName. @@ -88,12 +87,10 @@ OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variab * @param variableSetId variable set id for which the annotationSets have to delete the annotation. * @param annotationName Annotation name. * @return a OpenCGAResult object. - * @throws CatalogDBException when there is an error in the database. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException when there is an error in the database, there is any unexpected parameter or the operation is not + * authorized. */ - OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) throws CatalogException; /** * Makes a groupBy to obtain the different values that every annotation has and the total number of each. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 04f1aeafbe4..3935f24836d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -78,8 +78,10 @@ enum QueryParams implements QueryParam { RESPONSIBLE("responsible", OBJECT, ""), FLAGS("flags", OBJECT, ""), FLAGS_ID("flags.id", TEXT, ""), + VERSION("version", INTEGER, ""), RELEASE("release", INTEGER, ""), - PANEL_LOCK("panelLock", BOOLEAN, ""), + SNAPSHOT("snapshot", INTEGER, ""), + PANEL_LOCKED("panelLocked", BOOLEAN, ""), LOCKED("locked", BOOLEAN, ""), SAMPLE("sample", TEXT_ARRAY, ""), // Alias to search for samples within proband.samples or family.members.samples @@ -235,8 +237,7 @@ default void checkId(long clinicalAnalysisId) throws CatalogDBException, Catalog OpenCGAResult nativeInsert(Map clinicalAnalysis, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList, QueryOptions options) throws CatalogException; OpenCGAResult update(long id, ObjectMap parameters, List variableSetList, List clinicalAuditList, QueryOptions queryOptions) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java index d8b6f82182e..c95d006161d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java @@ -133,8 +133,7 @@ default void checkId(long cohortId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map cohort, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) throws CatalogException; OpenCGAResult get(long cohortId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java index da321435f29..fe9fd976e64 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java @@ -149,7 +149,7 @@ default void checkId(long familyId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map family, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + QueryOptions options) throws CatalogException; OpenCGAResult get(long familyId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 2ffe6fd1ee9..a341840a6bf 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -233,13 +233,10 @@ default void checkId(long fileId) throws CatalogDBException, CatalogParameterExc * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due to any formatting error or the user is not authorised. */ OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List variableSetList, QueryOptions options) throws CatalogException; /*** * Inserts the passed file in the database. @@ -252,13 +249,11 @@ OpenCGAResult insert(long studyId, File file, List existingSamples, List * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due any formatting error or the user is not authorised. */ OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; /*** * Retrieves the file from the database containing the fileId given. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java index d55e1019e0e..b6dbc94b861 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java @@ -163,7 +163,7 @@ default void checkId(long individualId) throws CatalogDBException, CatalogParame OpenCGAResult nativeInsert(Map individual, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(long individualId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java index 8c95202e0de..a49e7fd6262 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java @@ -24,6 +24,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -41,6 +42,7 @@ enum QueryParams implements QueryParam { ID("id", TEXT, ""), UID("uid", INTEGER, ""), UUID("uuid", TEXT, ""), + NAME("name", TEXT, ""), CLINICAL_ANALYSIS_ID("clinicalAnalysisId", TEXT, ""), DESCRIPTION("description", TEXT, ""), INTERNAL_STATUS("internal.status", TEXT, ""), @@ -135,8 +137,7 @@ default void checkId(long interpretationId) throws CatalogDBException, CatalogPa OpenCGAResult nativeInsert(Map interpretation, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList) throws CatalogException; OpenCGAResult update(long uid, ObjectMap parameters, List clinicalAuditList, ParamUtils.SaveInterpretationAs action, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index a6016da7441..ebbc7b62143 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -54,8 +54,7 @@ default void checkId(long jobId) throws CatalogDBException, CatalogParameterExce OpenCGAResult nativeInsert(Map job, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException; default OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException { //return updateStatus(query, new Job.JobStatus(Job.JobStatus.PREPARED)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java index 34d3495407d..eeecd6d55df 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java @@ -6,6 +6,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.response.OpenCGAResult; @@ -16,7 +17,7 @@ public interface NoteDBAdaptor extends DBAdaptor { - OpenCGAResult insert(Note note) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Note note) throws CatalogException; default OpenCGAResult get(long noteUid, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java index d8d8e8a0a51..93e65c63121 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java @@ -5,6 +5,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.response.OpenCGAResult; @@ -98,8 +99,7 @@ public static QueryParams getParam(String key) { // // OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; // - OpenCGAResult insert(Organization organization, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException; OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java index a6ce18d5a2f..63bdd0d5e18 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java @@ -153,11 +153,9 @@ default void checkUid(long panelUid) throws CatalogDBException, CatalogParameter } } - OpenCGAResult insert(long studyUid, List panelList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyUid, List panelList) throws CatalogException; - OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) throws CatalogException; OpenCGAResult get(long panelId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java index fcdfba9034d..1ff667bdbac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java @@ -23,6 +23,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.StudyPermissions; @@ -133,8 +134,7 @@ default void checkId(long projectId) throws CatalogDBException { OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; - OpenCGAResult insert(Project project, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException; OpenCGAResult get(long project, QueryOptions options) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java index b9f0b5b1b36..485ab7b10dd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java @@ -63,7 +63,7 @@ default void checkId(long sampleId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map sample, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(long sampleId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -86,13 +86,11 @@ OpenCGAResult updateProjectRelease(long studyId, int release) */ OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) throws CatalogException; - default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) throws CatalogException { return setRgaIndexes(studyUid, Collections.emptyList(), rgaIndex); } - OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException; enum QueryParams implements QueryParam { ID("id", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index b4e0aea1d2e..0afd3f255d4 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.common.Enums; @@ -249,8 +250,7 @@ OpenCGAResult setUsersToGroup(long studyId, String groupId, List */ OpenCGAResult removeUsersFromGroup(long studyId, String groupId, List members) throws CatalogDBException; - OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException; /** * Delete a group. @@ -273,12 +273,9 @@ OpenCGAResult removeUsersFromAllGroups(long studyId, List users) * @param groupList List containing possible groups that are synced and where the user should be added to. * @param authOrigin Authentication origin of the synced groups. * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException; /** * ADD or REMOVE user to list of provided groups. @@ -288,13 +285,10 @@ OpenCGAResult resyncUserWithSyncedGroups(String user, List groupL * @param groupList List of group ids. * @param action Update action [ADD, REMOVE] * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, ParamUtils.AddRemoveAction action) + throws CatalogException; /** * Create the permission rule to the list of permission rules defined for the entry in the studyId. @@ -381,13 +375,13 @@ default void checkVariableSetExists(String variableSetId, long studyId) throws C OpenCGAResult createVariableSet(long studyId, VariableSet variableSet) throws CatalogDBException; OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult renameFieldVariableSet(long variableSetId, String oldName, String newName, String user) throws CatalogDBException, CatalogAuthorizationException; OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult getVariableSet(long variableSetUid, QueryOptions options) throws CatalogDBException; @@ -411,8 +405,7 @@ OpenCGAResult getVariableSet(long variableSetId, QueryOptions optio OpenCGAResult getVariableSets(Query query, QueryOptions queryOptions, String user) throws CatalogDBException, CatalogAuthorizationException; - OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException; void updateDiskUsage(ClientSession clientSession, long studyId, long size) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index 3099262e917..84efa64aea2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -22,10 +22,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.response.OpenCGAResult; @@ -75,7 +72,7 @@ default void checkIds(List userIds) throws CatalogDBException, CatalogPa void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException; OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -97,9 +94,10 @@ OpenCGAResult delete(String userId, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException; + throws CatalogException; - OpenCGAResult resetPassword(String userId, String email, String newCryptPass) throws CatalogDBException; + OpenCGAResult resetPassword(String userId, String email, String newCryptPass) + throws CatalogException; // Config operations OpenCGAResult setConfig(String userId, String name, Map config) throws CatalogDBException; @@ -118,14 +116,20 @@ enum QueryParams implements QueryParam { NAME("name", TEXT_ARRAY, ""), EMAIL("email", TEXT_ARRAY, ""), ORGANIZATION("organization", TEXT_ARRAY, ""), + CREATION_DATE("creationDate", TEXT_ARRAY, ""), + MODIFICATION_DATE("modificationDate", TEXT_ARRAY, ""), + DEPRECATED_ACCOUNT("account", OBJECT, ""), // Deprecated since 3.2.1 #TASK-6494 TODO: Remove in future releases + DEPRECATED_ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), // Deprecated since 3.2.1 #TASK-6494 INTERNAL("internal", OBJECT, ""), - INTERNAL_FAILED_ATTEMPTS("internal.failedAttempts", INTEGER, ""), INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), - ACCOUNT("account", TEXT_ARRAY, ""), - ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), - ACCOUNT_CREATION_DATE("account.creationDate", TEXT, ""), - ACCOUNT_EXPIRATION_DATE("account.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT("internal.account", TEXT_ARRAY, ""), + INTERNAL_ACCOUNT_AUTHENTICATION_ID("internal.account.authentication.id", TEXT, ""), + INTERNAL_ACCOUNT_FAILED_ATTEMPTS("internal.account.failedAttempts", INTEGER, ""), + INTERNAL_ACCOUNT_CREATION_DATE("internal.account.creationDate", TEXT, ""), + INTERNAL_ACCOUNT_EXPIRATION_DATE("internal.account.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE("internal.account.password.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED("internal.account.password.lastModified", TEXT, ""), QUOTA("quota", OBJECT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java index d1e48be9331..02ce5649729 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java @@ -31,6 +31,7 @@ import org.opencb.opencga.catalog.db.mongodb.converters.AnnotationConverter; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.AnnotationUtils; import org.opencb.opencga.catalog.utils.Constants; @@ -687,8 +688,7 @@ private List getNewAnnotationList(List annotationSetLis return annotationList; } - public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException { long startTime = startQuery(); // We generate the generic document that should be inserted @@ -764,12 +764,9 @@ public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, * @param variableSetId Variable set id. * @param fieldId Field id corresponds with the variable name whose annotations have to be removed. * @return A OpenCGAResult object. - * @throws CatalogDBException if there is any unexpected error. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if there is any unexpected error or parameter, or if the operation is not authorized. */ - public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) throws CatalogException { long startTime = startQuery(); UpdateDocument updateDocument = new UpdateDocument(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index 445c1c6047a..ac6dd7af9fe 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -34,10 +34,8 @@ import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; @@ -578,7 +576,7 @@ public OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resou @Override public OpenCGAResult setToMembers(long studyId, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -598,8 +596,7 @@ public OpenCGAResult setToMembers(long studyId, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -652,7 +649,7 @@ private void setToMembers(List resourceIds, List members, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); addToMembersGroupInStudy(studyId, members, clientSession); @@ -696,8 +693,7 @@ private void addToMembers(List resourceIds, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction((clientSession) -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -724,7 +720,7 @@ private void addToMembersGroupInStudy(long studyId, List members, Client @Override public OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -766,8 +762,7 @@ private void removeFromMembers(ClientSession clientSession, List resourceI } @Override - public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException { if (members == null || members.isEmpty()) { throw new CatalogDBException("Missing 'members' array."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index ec066ba4fc6..7a359e5a7eb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -83,16 +83,23 @@ public class ClinicalAnalysisMongoDBAdaptor extends AnnotationMongoDBAdaptor va try { return runTransaction(clientSession -> transactionalUpdate(clientSession, result.first(), parameters, variableSetList, clinicalAuditList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update clinical analysis {}: {}", clinicalAnalysisId, e.getMessage(), e); throw new CatalogDBException("Could not update clinical analysis " + clinicalAnalysisId + ": " + e.getMessage(), e.getCause()); } @@ -287,61 +294,63 @@ OpenCGAResult transactionalUpdate(ClientSession clientSession, ClinicalAnalysis String clinicalAnalysisId = clinical.getId(); long clinicalAnalysisUid = clinical.getUid(); - DataResult result = updateAnnotationSets(clientSession, clinical.getStudyUid(), clinicalAnalysisUid, parameters, variableSetList, - queryOptions, false); + Query tmpQuery = new Query() + .append(STUDY_UID.key(), clinical.getStudyUid()) + .append(QueryParams.UID.key(), clinicalAnalysisUid); + Bson bsonQuery = parseQuery(tmpQuery); + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, (entryList) -> { + DataResult result = updateAnnotationSets(clientSession, clinical.getStudyUid(), clinicalAnalysisUid, parameters, + variableSetList, queryOptions, false); - // Perform the update - Query query = new Query(QueryParams.UID.key(), clinicalAnalysisUid); - UpdateDocument updateDocument = parseAndValidateUpdateParams(parameters, clinicalAuditList, query, queryOptions); + // Perform the update + UpdateDocument updateDocument = parseAndValidateUpdateParams(parameters, clinicalAuditList, tmpQuery, queryOptions); - Document updateOperation = updateDocument.toFinalUpdateDocument(); - List events = new ArrayList<>(); - if (!updateOperation.isEmpty() || !updateDocument.getNestedUpdateList().isEmpty()) { - DataResult update; + Document updateOperation = updateDocument.toFinalUpdateDocument(); + List events = new ArrayList<>(); + if (!updateOperation.isEmpty() || !updateDocument.getNestedUpdateList().isEmpty()) { + DataResult update; - if (!updateOperation.isEmpty()) { - Bson bsonQuery = Filters.eq(PRIVATE_UID, clinicalAnalysisUid); + if (!updateOperation.isEmpty()) { + logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); + update = clinicalCollection.update(clientSession, bsonQuery, updateOperation, null); - logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); - update = clinicalCollection.update(clientSession, bsonQuery, updateOperation, null); + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + } + if (update.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, clinicalAnalysisId, "Clinical Analysis was already updated")); + } - if (update.getNumMatches() == 0) { - throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); - } - if (update.getNumUpdated() == 0) { - events.add(new Event(Event.Type.WARNING, clinicalAnalysisId, "Clinical Analysis was already updated")); - } + if (parameters.getBoolean(LOCKED.key())) { + // Propagate locked value to Interpretations + logger.debug("Propagating case lock to all the Interpretations"); + dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, + parameters.getBoolean(LOCKED.key())); + } - if (parameters.getBoolean(LOCKED.key())) { - // Propagate locked value to Interpretations - logger.debug("Propagating case lock to all the Interpretations"); - dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, - parameters.getBoolean(LOCKED.key())); + logger.debug("Clinical Analysis {} successfully updated", clinicalAnalysisId); } - logger.debug("Clinical Analysis {} successfully updated", clinicalAnalysisId); - } - - if (!updateDocument.getNestedUpdateList().isEmpty()) { - for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + if (!updateDocument.getNestedUpdateList().isEmpty()) { + // Nested documents are used to update reports + for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + Bson tmpBsonQuery = parseQuery(nestedDocument.getQuery().append(QueryParams.UID.key(), clinicalAnalysisUid)); + logger.debug("Update nested element from Clinical Analysis. Query: {}, Update: {}", + tmpBsonQuery.toBsonDocument(), nestedDocument.getSet()); + update = clinicalCollection.update(clientSession, tmpBsonQuery, nestedDocument.getSet(), null); - Bson bsonQuery = parseQuery(nestedDocument.getQuery().append(QueryParams.UID.key(), clinicalAnalysisUid)); - logger.debug("Update nested element from Clinical Analysis. Query: {}, Update: {}", - bsonQuery.toBsonDocument(), nestedDocument.getSet()); - - update = clinicalCollection.update(clientSession, bsonQuery, nestedDocument.getSet(), null); - - if (update.getNumMatches() == 0) { - throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + } } } - } - } else if (result.getNumUpdated() == 0) { - throw new CatalogDBException("Nothing to update"); - } + } else if (result.getNumUpdated() == 0) { + throw new CatalogDBException("Nothing to update"); + } - return endWrite(tmpStartTime, 1, 1, events); + return endWrite(tmpStartTime, 1, 1, events); + }, null, null); } @Override @@ -359,22 +368,24 @@ OpenCGAResult transactionalUpdate(ClientSession clientSession, Document updateOperation = updateDocument.toFinalUpdateDocument(); if (!updateOperation.isEmpty()) { - logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument); - DataResult update = clinicalCollection.update(clientSession, query, updateOperation, null); - - if (updateDocument.getSet().getBoolean(LOCKED.key(), false)) { - // Propagate locked value to Interpretations - logger.debug("Propagating case lock to all the Interpretations"); - MongoDBIterator iterator = clinicalCollection.iterator(clientSession, query, null, clinicalConverter, - ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); - while (iterator.hasNext()) { - ClinicalAnalysis clinical = iterator.next(); - dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, true); + return versionedMongoDBAdaptor.update(clientSession, query, entryList -> { + logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument); + DataResult update = clinicalCollection.update(clientSession, query, updateOperation, null); + + if (updateDocument.getSet().getBoolean(LOCKED.key(), false)) { + // Propagate locked value to Interpretations + logger.debug("Propagating case lock to all the Interpretations"); + MongoDBIterator iterator = clinicalCollection.iterator(clientSession, query, null, clinicalConverter, + ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); + while (iterator.hasNext()) { + ClinicalAnalysis clinical = iterator.next(); + dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, true); + } } - } - logger.debug("{} clinical analyses successfully updated", update.getNumUpdated()); - return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList()); + logger.debug("{} clinical analyses successfully updated", update.getNumUpdated()); + return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList()); + }, null, null); } else { throw new CatalogDBException("Nothing to update"); } @@ -412,7 +423,7 @@ UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, List delete(ClinicalAnalysis clinicalAnalysis, List privateDelete(clientSession, clinicalAnalysis, clinicalAuditList)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete Clinical Analysis " + clinicalAnalysis.getId() + ": " + e.getMessage(), e.getCause()); @@ -702,7 +713,7 @@ public OpenCGAResult delete(Query query, List c try { result.append(runTransaction(clientSession -> privateDelete(clientSession, clinicalAnalysis, clinicalAuditList))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, clinicalAnalysis.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -770,52 +781,18 @@ OpenCGAResult privateDelete(ClientSession clientSession, ClinicalAnalysis cli } } + // Add Audit to ClinicalAnalysis + transactionalUpdate(clientSession, clinicalAnalysis, new ObjectMap(), Collections.emptyList(), clinicalAuditList, + QueryOptions.empty()); + + // And delete ClinicalAnalysis Query query = new Query() - .append(STUDY_UID.key(), clinicalAnalysis.getStudyUid()) + .append(QueryParams.STUDY_UID.key(), clinicalAnalysis.getStudyUid()) .append(UID.key(), clinicalAnalysis.getUid()); - OpenCGAResult result = nativeGet(clientSession, query, QueryOptions.empty()); - if (result.getNumResults() == 0) { - throw new CatalogDBException("Internal error: Clinical Analysis '" + clinicalAnalysis.getId() + "' not found."); - } - - String clinicalId = result.first().getString(QueryParams.ID.key()); - long clinicalUid = result.first().getLong(PRIVATE_UID); - long studyUid = result.first().getLong(PRIVATE_STUDY_UID); - - logger.debug("Deleting Clinical Analysis {} ({})", clinicalId, clinicalUid); - - // Delete any documents that might have been already deleted with that id - Bson bsonQuery = new Document() - .append(QueryParams.ID.key(), clinicalId) - .append(PRIVATE_STUDY_UID, studyUid); - deletedClinicalCollection.remove(clientSession, bsonQuery, new QueryOptions(MongoDBCollection.MULTI, true)); - - // Set status to DELETED - nestedPut(QueryParams.INTERNAL_STATUS.key(), getMongoDBDocument(new InternalStatus(InternalStatus.DELETED), "status"), - result.first()); - - // Add audit - List auditList = result.first().getList(AUDIT.key(), Document.class); - for (ClinicalAudit clinicalAudit : clinicalAuditList) { - auditList.add(getMongoDBDocument(clinicalAudit, "ClinicalAudit")); - } - result.first().put(AUDIT.key(), auditList); - - // Insert the document in the DELETE collection - deletedClinicalCollection.insert(clientSession, replaceDotsInKeys(result.first()), null); - logger.debug("Inserted Clinical Analysis uid '{}' in DELETE collection", clinicalUid); - - // Remove the document from the main Clinical collection - bsonQuery = parseQuery(new Query(QueryParams.UID.key(), clinicalUid)); - DataResult remove = clinicalCollection.remove(clientSession, bsonQuery, null); - if (remove.getNumMatches() == 0) { - throw new CatalogDBException("Clinical Analysis " + clinicalId + " not found"); - } - if (remove.getNumDeleted() == 0) { - throw new CatalogDBException("Clinical Analysis " + clinicalId + " could not be deleted"); - } + Bson bsonQuery = parseQuery(query); + versionedMongoDBAdaptor.delete(clientSession, bsonQuery); - logger.debug("Clinical Analysis {}({}) deleted", clinicalId, clinicalUid); + logger.debug("Clinical Analysis {}({}) deleted", clinicalAnalysis.getId(), clinicalAnalysis.getUid()); return endWrite(tmpStartTime, 1, 0, 0, 1, Collections.emptyList()); } @@ -968,12 +945,8 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = removeInnerProjections(qOptions, QueryParams.SECONDARY_INTERPRETATIONS.key()); logger.debug("Clinical analysis query : {}", bson.toBsonDocument()); - - if (!query.getBoolean(QueryParams.DELETED.key())) { - return clinicalCollection.iterator(clientSession, bson, null, null, qOptions); - } else { - return deletedClinicalCollection.iterator(clientSession, bson, null, null, qOptions); - } + MongoDBCollection collection = getQueryCollection(query, clinicalCollection, archiveClinicalCollection, deletedClinicalCollection); + return collection.iterator(clientSession, bson, null, null, qOptions); } @Override @@ -1048,8 +1021,7 @@ public OpenCGAResult nativeInsert(Map clinicalAnalysis, String u @Override public OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -1106,7 +1078,7 @@ ClinicalAnalysis insert(ClientSession clientSession, long studyId, ClinicalAnaly clinicalDocument.put(PRIVATE_DUE_DATE, TimeUtils.toDate(clinicalAnalysis.getDueDate())); logger.debug("Inserting ClinicalAnalysis '{}' ({})...", clinicalAnalysis.getId(), clinicalAnalysis.getUid()); - clinicalCollection.insert(clientSession, clinicalDocument, null); + versionedMongoDBAdaptor.insert(clientSession, clinicalDocument); logger.debug("ClinicalAnalysis '{}' successfully inserted", clinicalAnalysis.getId()); return clinicalAnalysis; @@ -1252,7 +1224,7 @@ void updateClinicalAnalysisPanelReferences(ClientSession clientSession, Panel pa Query query = new Query() .append(STUDY_UID.key(), panel.getStudyUid()) .append(PANELS_UID.key(), panel.getUid()) - .append(PANEL_LOCK.key(), false) + .append(PANEL_LOCKED.key(), false) .append(LOCKED.key(), false); QueryOptions include = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( PANELS_UID.key(), @@ -1334,12 +1306,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) queryCopy.remove(ParamConstants.ACL_PARAM); } + if ("all".equalsIgnoreCase(queryCopy.getString(QueryParams.VERSION.key()))) { + queryCopy.put(Constants.ALL_VERSIONS, true); + queryCopy.remove(QueryParams.VERSION.key()); + } + boolean uidVersionQueryFlag = versionedMongoDBAdaptor.generateUidVersionQuery(queryCopy, andBsonList); + for (Map.Entry entry : queryCopy.entrySet()) { String key = entry.getKey().split("\\.")[0]; QueryParams queryParam = QueryParams.getParam(entry.getKey()) != null ? QueryParams.getParam(entry.getKey()) : QueryParams.getParam(key); if (queryParam == null) { - if (Constants.PRIVATE_ANNOTATION_PARAM_TYPES.equals(entry.getKey())) { + if (Constants.ALL_VERSIONS.equals(entry.getKey()) || Constants.PRIVATE_ANNOTATION_PARAM_TYPES.equals(entry.getKey())) { continue; } throw new CatalogDBException("Unexpected parameter " + entry.getKey() + ". The parameter does not exist or cannot be " @@ -1385,7 +1363,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case INTERNAL_STATUS: case INTERNAL_STATUS_ID: // Convert the status to a positive status - queryCopy.put(queryParam.key(), InternalStatus.getPositiveStatus(ClinicalAnalysisStatus.STATUS_LIST, + queryCopy.put(queryParam.key(), InternalStatus.getPositiveStatus(InternalStatus.STATUS_LIST, queryCopy.getString(queryParam.key()))); addAutoOrQuery(INTERNAL_STATUS_ID.key(), queryParam.key(), queryCopy, INTERNAL_STATUS_ID.type(), andBsonList); break; @@ -1395,11 +1373,14 @@ private Bson parseQuery(Query query, Document extraQuery, String user) queryCopy.get(Constants.PRIVATE_ANNOTATION_PARAM_TYPES, ObjectMap.class)); } break; + case SNAPSHOT: + addAutoOrQuery(RELEASE_FROM_VERSION, queryParam.key(), queryCopy, queryParam.type(), andBsonList); + break; // Other parameter that can be queried. case ID: case UUID: case TYPE: - case PANEL_LOCK: + case PANEL_LOCKED: case LOCKED: case FILES_UID: case PROBAND_UID: @@ -1412,6 +1393,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case PRIORITY_ID: case FLAGS_ID: case QUALITY_CONTROL_SUMMARY: + case VERSION: case RELEASE: case COMMENTS_DATE: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); @@ -1425,6 +1407,17 @@ private Bson parseQuery(Query query, Document extraQuery, String user) } } + // If the user doesn't look for a concrete version... + if (!uidVersionQueryFlag && !queryCopy.getBoolean(Constants.ALL_VERSIONS) && !queryCopy.containsKey(QueryParams.VERSION.key())) { + if (queryCopy.containsKey(QueryParams.SNAPSHOT.key())) { + // If the user looks for anything from some release, we will try to find the latest from the release (snapshot) + andBsonList.add(Filters.eq(LAST_OF_RELEASE, true)); + } else { + // Otherwise, we will always look for the latest version + andBsonList.add(Filters.eq(LAST_OF_VERSION, true)); + } + } + if (annotationDocument != null && !annotationDocument.isEmpty()) { andBsonList.add(annotationDocument); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java index 7f25b1ee7b9..6b948a722ba 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java @@ -96,7 +96,7 @@ public OpenCGAResult nativeInsert(Map cohort, String userId) thr @Override public OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -275,7 +275,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, cohort, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update cohort {}: {}", cohort.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohort.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -545,7 +545,7 @@ public OpenCGAResult delete(Cohort cohort) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find cohort " + cohort.getId() + " with uid " + cohort.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohort.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete cohort '" + cohort.getId() + "': " + e.getMessage(), e.getCause()); } @@ -561,7 +561,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String cohortId = cohort.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, cohort))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohortId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohortId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index 9e8c35d00ab..97bfd92a671 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -115,8 +115,7 @@ public OpenCGAResult nativeInsert(Map family, String userId) thr @Override public OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + QueryOptions options) throws CatalogException { try { AtomicReference familyCopy = new AtomicReference<>(); OpenCGAResult result = runTransaction(clientSession -> { @@ -333,7 +332,7 @@ public OpenCGAResult update(long familyUid, ObjectMap parameters, List transactionalUpdate(clientSession, familyDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", familyDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update family " + familyDataResult.first().getId() + ": " + e.getMessage(), e.getCause()); @@ -368,7 +367,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, family, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", family.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, family.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -784,7 +783,7 @@ public OpenCGAResult delete(Family family) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find family " + family.getId() + " with uid " + family.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", family.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete family " + family.getId() + ": " + e.getMessage(), e.getCause()); } @@ -801,7 +800,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String familyId = family.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, family))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", familyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, familyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index f214620bf39..0f8383a9905 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -127,8 +127,7 @@ public OpenCGAResult nativeInsert(Map file, String userId) throw @Override public OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List variableSetList, QueryOptions options) throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -144,7 +143,7 @@ public OpenCGAResult insert(long studyId, File file, List existingSample @Override public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -344,7 +343,7 @@ public OpenCGAResult update(long fileUid, ObjectMap parameters, List transactionalUpdate(clientSession, fileDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", fileDataResult.first().getPath(), e.getMessage(), e); throw new CatalogDBException("Could not update file " + fileDataResult.first().getPath() + ": " + e.getMessage(), e.getCause()); } @@ -370,7 +369,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, file, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", file.getPath(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, file.getPath(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -931,7 +930,7 @@ public OpenCGAResult delete(File file, String status) try { return runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", file.getPath(), e.getMessage(), e); throw new CatalogDBException("Could not delete file " + file.getPath() + ": " + e.getMessage(), e.getCause()); } @@ -962,7 +961,7 @@ public OpenCGAResult delete(Query query, String status) Document fileDocument = iterator.next(); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", fileDocument.getString(QueryParams.PATH.key()), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, fileDocument.getString(QueryParams.ID.key()), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 13dbb40741c..03a6704af17 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -119,7 +119,7 @@ public OpenCGAResult nativeInsert(Map individual, String userId) @Override public OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -329,7 +329,7 @@ public OpenCGAResult update(long individualUid, ObjectMap parameters, List transactionalUpdate(clientSession, individualUid, parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { throw new CatalogDBException("Could not update individual: " + e.getMessage(), e.getCause()); } } @@ -363,7 +363,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, individual, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update individual {}: {}", individual.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individual.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1051,7 +1051,7 @@ public OpenCGAResult delete(Individual individual) throws CatalogDBException, Ca throw new CatalogDBException("Could not find individual " + individual.getId() + " with uid " + individual.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individual.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e); } @@ -1068,7 +1068,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String individualId = individual.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, individual))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individualId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individualId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java index 3d78516dd55..0c833de52ed 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java @@ -40,6 +40,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.InterpretationCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.ClinicalAnalysisManager; import org.opencb.opencga.catalog.utils.Constants; @@ -120,8 +121,7 @@ public OpenCGAResult nativeInsert(Map interpretation, String use @Override public OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting interpretation insert transaction for interpretation id '{}'", interpretation.getId()); @@ -433,7 +433,7 @@ private UpdateDocument parseAndValidateUpdateParams(ClientSession clientSession, String[] booleanParams = {LOCKED.key()}; filterBooleanParams(parameters, document.getSet(), booleanParams); - String[] acceptedParams = {QueryParams.DESCRIPTION.key()}; + String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); if (StringUtils.isNotEmpty(parameters.getString(QueryParams.CREATION_DATE.key()))) { @@ -635,7 +635,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List try { return runTransaction(clientSession -> update(clientSession, interpretation.first(), parameters, clinicalAuditList, action, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not update interpretation " + interpretationId + ": " + e.getMessage(), e.getCause()); } @@ -654,7 +654,7 @@ public OpenCGAResult revert(long id, int previousVersion, List c return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); }); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not delete interpretation " + interpretation.getId() + ": " + e.getMessage(), e.getCause()); } @@ -852,7 +852,7 @@ public OpenCGAResult delete(Query query, List cli return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); })); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, interpretationId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1136,6 +1136,7 @@ protected Bson parseQuery(Query query) throws CatalogDBException { break; // Other parameter that can be queried. case ID: + case NAME: case UUID: case PANELS_UID: case RELEASE: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 53c9a5f157e..d89e0020f93 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -100,8 +100,7 @@ public OpenCGAResult nativeInsert(Map job, String userId) throws } @Override - public OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -228,7 +227,7 @@ public OpenCGAResult update(long jobUid, ObjectMap parameters, QueryOptions quer try { return runTransaction(session -> privateUpdate(session, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update job " + dataResult.first().getId() + ": " + e.getMessage(), e.getCause()); } @@ -254,7 +253,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Job job = iterator.next(); try { result.append(runTransaction(session -> privateUpdate(session, job, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", job.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, job.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -306,7 +305,7 @@ public OpenCGAResult delete(Job job) throws CatalogDBException, CatalogParameter throw new CatalogDBException("Could not find job " + job.getId() + " with uid " + job.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", job.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete job " + job.getId() + ": " + e.getMessage(), e.getCause()); } @@ -322,7 +321,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String jobId = job.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, job))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", jobId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, jobId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java index 7f41e38411d..32c10c362f7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java @@ -60,6 +60,7 @@ public abstract class MongoDBAdaptor extends AbstractDBAdaptor { static final String PRIVATE_PROJECT_UUID = PRIVATE_PROJECT + '.' + PRIVATE_UUID; public static final String PRIVATE_STUDY_UID = "studyUid"; public static final String VERSION = "version"; + public static final String RELEASE = "release"; static final String FILTER_ROUTE_STUDIES = "projects.studies."; static final String FILTER_ROUTE_COHORTS = "projects.studies.cohorts."; @@ -93,16 +94,15 @@ public MongoDBAdaptor(Configuration configuration, Logger logger) { } public interface TransactionBodyWithException { - T execute(ClientSession session) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + T execute(ClientSession session) throws CatalogException; } - protected T runTransaction(TransactionBodyWithException body) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + protected T runTransaction(TransactionBodyWithException body) throws CatalogException { return runTransaction(body, null); } protected T runTransaction(TransactionBodyWithException inputBody, Consumer onException) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { ClientSession session = dbAdaptorFactory.getMongoDataStore().startSession(); try { TransactionBodyWithException body; @@ -124,7 +124,7 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum return session.withTransaction(() -> { try { return body.execute(session); - } catch (CatalogDBException | CatalogAuthorizationException | CatalogParameterException e) { + } catch (CatalogException e) { throw new CatalogDBRuntimeException(e); } }); @@ -147,6 +147,12 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum onException.accept(cause); } throw cause; + } else if (e.getCause() instanceof CatalogAuthenticationException) { + CatalogAuthenticationException cause = (CatalogAuthenticationException) e.getCause(); + if (onException != null) { + onException.accept(cause); + } + throw cause; } else { throw e; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index 621c537cddd..e262c3d3595 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -27,10 +27,8 @@ import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.managers.NoteManager; import org.opencb.opencga.core.api.ParamConstants; @@ -261,7 +259,7 @@ public MetaDBAdaptor getCatalogMetaDBAdaptor(String organizationId) throws Catal @Override public OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { OrganizationMongoDBAdaptorFactory orgFactory = getOrganizationMongoDBAdaptorFactory(organization.getId(), false); if (orgFactory != null && orgFactory.isCatalogDBReady()) { throw new CatalogDBException("Organization '" + organization.getId() + "' already exists."); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java index c32644fee46..0b953919713 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java @@ -14,6 +14,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -51,8 +52,7 @@ public NoteMongoDBAdaptor(MongoDBCollection noteCollection, MongoDBCollection ar } @Override - public OpenCGAResult insert(Note note) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Note note) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting note insert transaction for note id '{}'", note.getId()); @@ -142,7 +142,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, QueryOptions queryOp throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateUpdate(clientSession, uid, parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update note: {}", e.getMessage(), e); throw new CatalogDBException("Could not update note: " + e.getMessage(), e.getCause()); } @@ -234,7 +234,7 @@ private UpdateDocument parseAndValidateUpdateParams(Document note, ObjectMap par filterStringListParams(parameters, document.getSet(), tagsParams); break; case REMOVE: - filterStringListParams(parameters, document.getPull(), tagsParams); + filterStringListParams(parameters, document.getPullAll(), tagsParams); break; case ADD: filterStringListParams(parameters, document.getAddToSet(), tagsParams); @@ -262,7 +262,7 @@ public OpenCGAResult delete(Note note) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateDelete(clientSession, note)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { throw new CatalogDBException("Could not delete note " + note.getId() + ": " + e.getMessage(), e); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java index aa090f498d1..b971f5a5740 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java @@ -17,6 +17,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.OrganizationCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.OrganizationManager; import org.opencb.opencga.catalog.utils.Constants; @@ -52,8 +53,7 @@ public OrganizationMongoDBAdaptor(MongoDBCollection organizationCollection, Conf } @Override - public OpenCGAResult insert(Organization organization, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting organization insert transaction for organization id '{}'", organization.getId()); @@ -124,7 +124,7 @@ public OpenCGAResult update(String organizationId, ObjectMap param try { QueryOptions options = queryOptions != null ? new QueryOptions(queryOptions) : QueryOptions.empty(); return runTransaction(clientSession -> privateUpdate(clientSession, organizationId, parameters, options)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update organization {}: {}", organizationId, e.getMessage(), e); throw new CatalogDBException("Could not update organization " + organizationId + ": " + e.getMessage(), e.getCause()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index b1c1c6704ec..9884bbccc6c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -48,6 +48,7 @@ public class OrganizationMongoDBAdaptorFactory { public static final String INDIVIDUAL_ARCHIVE_COLLECTION = "individual_archive"; public static final String FAMILY_ARCHIVE_COLLECTION = "family_archive"; public static final String PANEL_ARCHIVE_COLLECTION = "panel_archive"; + public static final String CLINICAL_ANALYSIS_ARCHIVE_COLLECTION = "clinical_archive"; public static final String INTERPRETATION_ARCHIVE_COLLECTION = "interpretation_archive"; public static final String DELETED_NOTE_COLLECTION = "note_deleted"; @@ -89,6 +90,7 @@ public class OrganizationMongoDBAdaptorFactory { INDIVIDUAL_ARCHIVE_COLLECTION, FAMILY_ARCHIVE_COLLECTION, PANEL_ARCHIVE_COLLECTION, + CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, INTERPRETATION_ARCHIVE_COLLECTION, DELETED_NOTE_COLLECTION, @@ -119,7 +121,7 @@ public class OrganizationMongoDBAdaptorFactory { private final NoteMongoDBAdaptor notesDBAdaptor; private final OrganizationMongoDBAdaptor organizationDBAdaptor; - private final UserMongoDBAdaptor userDBAdaptor; + private UserMongoDBAdaptor userDBAdaptor; private final ProjectMongoDBAdaptor projectDBAdaptor; private final StudyMongoDBAdaptor studyDBAdaptor; private final IndividualMongoDBAdaptor individualDBAdaptor; @@ -176,6 +178,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa MongoDBCollection individualArchivedCollection = mongoDataStore.getCollection(INDIVIDUAL_ARCHIVE_COLLECTION); MongoDBCollection familyArchivedCollection = mongoDataStore.getCollection(FAMILY_ARCHIVE_COLLECTION); MongoDBCollection panelArchivedCollection = mongoDataStore.getCollection(PANEL_ARCHIVE_COLLECTION); + MongoDBCollection clinicalArchivedCollection = mongoDataStore.getCollection(CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); MongoDBCollection interpretationArchivedCollection = mongoDataStore.getCollection(INTERPRETATION_ARCHIVE_COLLECTION); MongoDBCollection deletedNotesCollection = mongoDataStore.getCollection(DELETED_NOTE_COLLECTION); @@ -211,7 +214,8 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa userDBAdaptor = new UserMongoDBAdaptor(userCollection, deletedUserCollection, configuration, this); cohortDBAdaptor = new CohortMongoDBAdaptor(cohortCollection, deletedCohortCollection, configuration, this); panelDBAdaptor = new PanelMongoDBAdaptor(panelCollection, panelArchivedCollection, deletedPanelCollection, configuration, this); - clinicalDBAdaptor = new ClinicalAnalysisMongoDBAdaptor(clinicalCollection, deletedClinicalCollection, configuration, this); + clinicalDBAdaptor = new ClinicalAnalysisMongoDBAdaptor(clinicalCollection, clinicalArchivedCollection, deletedClinicalCollection, + configuration, this); interpretationDBAdaptor = new InterpretationMongoDBAdaptor(interpretationCollection, interpretationArchivedCollection, deletedInterpretationCollection, configuration, this); // metaDBAdaptor = new MetaMongoDBAdaptor(metaCollection, configuration, this); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java index d82535b9351..fcc8f32fd6f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java @@ -33,6 +33,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -87,8 +88,7 @@ public MongoDBCollection getPanelArchiveCollection() { } @Override - public OpenCGAResult insert(long studyUid, List panelList) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, List panelList) throws CatalogException { if (panelList == null || panelList.isEmpty()) { throw new CatalogDBException("Missing panel list"); } @@ -108,8 +108,7 @@ public OpenCGAResult insert(long studyUid, List panelList) throws Catalog } @Override - public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting insert transaction of panel id '{}'", panel.getId()); @@ -291,7 +290,7 @@ public OpenCGAResult update(long panelUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update panel '" + dataResult.first().getId() + "': " + e.getMessage(), e.getCause()); } @@ -317,7 +316,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Panel panel = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, panel, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", panel.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panel.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -464,7 +463,7 @@ public OpenCGAResult delete(Panel panel) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find panel " + panel.getId() + " with uid " + panel.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panel.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete panel '" + panel.getId() + "': " + e.getMessage(), e.getCause()); } @@ -480,7 +479,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String panelId = panel.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, panel))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panelId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panelId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index 89d90aab932..3a91bc26284 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -32,6 +32,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.ProjectCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -82,8 +83,7 @@ public OpenCGAResult nativeInsert(Map project, String userId) th } @Override - public OpenCGAResult insert(Project project, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting project insert transaction for project id '{}'", project.getId()); @@ -201,7 +201,7 @@ public OpenCGAResult update(long projectUid, ObjectMap parameters, QueryOptions try { return runTransaction(clientSession -> privateUpdate(clientSession, projectDataResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", projectDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update project '" + projectDataResult.first().getId() + "': " + e.getMessage(), e.getCause()); @@ -220,7 +220,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Project project = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, project, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -383,7 +383,7 @@ public OpenCGAResult delete(long id, QueryOptions queryOptions) throws CatalogDB public OpenCGAResult delete(Project project) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateDelete(clientSession, project)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete project '" + project.getId() + "': " + e.getMessage(), e.getCause()); } @@ -412,7 +412,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, project))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index 198c9fce456..d9058193e7d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -39,6 +39,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.SampleCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.IndividualManager; import org.opencb.opencga.catalog.managers.SampleManager; @@ -199,7 +200,7 @@ Sample insert(ClientSession clientSession, long studyUid, Sample sample, List insert(long studyId, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting sample insert transaction for sample id '{}'", sample.getId()); @@ -262,7 +263,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List va try { return runTransaction(clientSession -> privateUpdate(clientSession, documentResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); throw new CatalogDBException("Could not update sample " + sampleId + ": " + e.getMessage(), e.getCause()); } @@ -297,7 +298,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, sampleDocument, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -857,8 +858,7 @@ public OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) } @Override - public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException { ObjectMap params; try { params = new ObjectMap(getDefaultObjectMapper().writeValueAsString(rgaIndex)); @@ -943,7 +943,7 @@ public OpenCGAResult delete(Sample sample) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find sample " + sample.getId() + " with uid " + sample.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sample.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e); } @@ -960,7 +960,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, sample))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java index a86ebb9cc36..c54c80df169 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java @@ -137,6 +137,12 @@ DBIterator iterator(ClientSession session, Query query, QueryOptions options) } protected void insert(ClientSession session, Document document) { + // Versioning private parameters + document.put(VERSION, 1); + document.put(RELEASE_FROM_VERSION, Arrays.asList(document.getInteger(RELEASE))); + document.put(LAST_OF_VERSION, true); + document.put(LAST_OF_RELEASE, true); + String uuid = getClientSessionUuid(session); document.put(PRIVATE_TRANSACTION_ID, uuid); collection.insert(session, document, QueryOptions.empty()); @@ -348,12 +354,17 @@ protected Document revertToVersion(ClientSession clientSession, long uid, int ve return document; } - protected void delete(ClientSession session, Bson query) { + protected void delete(ClientSession session, Bson query) throws CatalogDBException { // Remove any old documents from the "delete" collection matching the criteria deletedCollection.remove(session, query, QueryOptions.empty()); // Remove document from main collection - collection.remove(session, query, QueryOptions.empty()); + DataResult remove = collection.remove(session, query, QueryOptions.empty()); + if (remove.getNumDeleted() == 0) { + logger.error("Delete operation for '{}' could not be performed. Num matches: {}", query.toBsonDocument(), + remove.getNumMatches()); + throw new CatalogDBException("Delete operation could not be performed"); + } // Add versioned documents to "delete" collection InternalStatus internalStatus = new InternalStatus(InternalStatus.DELETED); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index 5c6e0ff2035..c91fd4c3665 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -38,6 +38,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.StudyCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.FqnUtils; @@ -490,8 +491,7 @@ OpenCGAResult removeUsersFromAdminsGroup(ClientSession clientSession, Lis } @Override - public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException { if (users == null || users.size() == 0) { throw new CatalogDBException("Unable to remove users from groups. List of users is empty"); } @@ -546,8 +546,7 @@ public OpenCGAResult syncGroup(long studyId, String groupId, Group.Sync s } @Override - public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogDBException("Missing user field"); } @@ -600,8 +599,7 @@ public OpenCGAResult resyncUserWithSyncedGroups(String user, List @Override public OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + ParamUtils.AddRemoveAction action) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogParameterException("Missing user parameter"); @@ -820,7 +818,7 @@ public OpenCGAResult createVariableSet(long studyId, VariableSet va @Override public OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableNotInVariableSet(variableSet.first(), variable.getId()); @@ -897,9 +895,7 @@ public OpenCGAResult renameFieldVariableSet(long variableSetId, Str @Override public OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - long startTime = startQuery(); - + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableInVariableSet(variableSet.first(), name); @@ -1167,8 +1163,7 @@ public OpenCGAResult getVariableSets(Query query, QueryOptions quer } @Override - public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException { try { return runTransaction(clientSession -> { if (force) { @@ -1341,7 +1336,7 @@ public OpenCGAResult update(long studyUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, studyResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", studyId, e.getMessage(), e); throw new CatalogDBException("Could not update study '" + studyId + "': " + e.getMessage(), e.getCause()); } @@ -1364,7 +1359,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Study study = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, study, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", study.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, study.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1478,7 +1473,7 @@ public OpenCGAResult delete(Study study) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find study " + study.getId() + " with uid " + study.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", study.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete study " + study.getId() + ": " + e.getMessage(), e.getCause()); } @@ -1494,7 +1489,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { String studyId = study.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, study))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", studyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, studyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 19a9a6cd1f3..24c71463e32 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -18,7 +18,9 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.Updates; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -36,13 +38,13 @@ import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.UserConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.project.Project; @@ -54,6 +56,7 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.function.Consumer; @@ -71,7 +74,22 @@ public class UserMongoDBAdaptor extends CatalogMongoDBAdaptor implements UserDBA private final MongoDBCollection deletedUserCollection; private UserConverter userConverter; - private static final String PRIVATE_PASSWORD = "_password"; + // --- Password constants --- + public static final String HASH = "hash"; + public static final String SALT = "salt"; + + public static final String PRIVATE_PASSWORD = "_password"; + + public static final String CURRENT = "current"; + private static final String PRIVATE_PASSWORD_CURRENT = "_password." + CURRENT; + private static final String PRIVATE_PASSWORD_CURRENT_HASH = PRIVATE_PASSWORD_CURRENT + "." + HASH; + private static final String PRIVATE_PASSWORD_CURRENT_SALT = PRIVATE_PASSWORD_CURRENT + "." + SALT; + + public static final String ARCHIVE = "archive"; + public static final String PRIVATE_PASSWORD_ARCHIVE = "_password." + ARCHIVE; + private static final String PRIVATE_PASSWORD_ARCHIVE_HASH = PRIVATE_PASSWORD_ARCHIVE + "." + HASH; + private static final String PRIVATE_PASSWORD_ARCHIVE_SALT = PRIVATE_PASSWORD_ARCHIVE + "." + SALT; + // -------------------------- public UserMongoDBAdaptor(MongoDBCollection userCollection, MongoDBCollection deletedUserCollection, Configuration configuration, OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { @@ -93,8 +111,7 @@ boolean exists(ClientSession clientSession, String userId) throws CatalogDBExcep } @Override - public OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(User user, String password, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -117,7 +134,18 @@ private void insert(ClientSession clientSession, User user, String password) thr Document userDocument = userConverter.convertToStorageType(user); userDocument.append(ID, user.getId()); - userDocument.append(PRIVATE_PASSWORD, encryptPassword(password)); + + Document privatePassword = new Document(); + if (StringUtils.isNotEmpty(password)) { + String salt = PasswordUtils.getStrongRandomSalt(); + String hash = encryptPassword(password, salt); + Document passwordDoc = new Document() + .append(HASH, hash) + .append(SALT, salt); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + } + userDocument.put(PRIVATE_PASSWORD, privatePassword); userCollection.insert(clientSession, userDocument, null); } @@ -130,47 +158,133 @@ public OpenCGAResult get(String userId, QueryOptions options) } @Override - public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException { - Document bson = new Document(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(oldPassword)); - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); - - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); - } - return new OpenCGAResult(result); + public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { + return setPassword(userId, oldPassword, newPassword); } @Override - public void authenticate(String userId, String password) throws CatalogAuthenticationException { - Document bson; - try { - bson = new Document() - .append(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(password)); - } catch (CatalogDBException e) { - throw new CatalogAuthenticationException("Could not encrypt password: " + e.getMessage(), e); + public void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException { + Bson query = Filters.and( + Filters.eq(QueryParams.ID.key(), userId), + // TODO: Deprecated. Remove Filters.or using the deprecated account authentication id + Filters.or( + Filters.eq(DEPRECATED_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA), + Filters.eq(INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA) + ) + ); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult dataResult = userCollection.find(query, projection, QueryOptions.empty()); + if (dataResult.getNumResults() == 0) { + throw new CatalogDBException("User " + userId + " not found"); + } + Document userDocument = dataResult.first(); + Object rootPasswordObject = userDocument.get(PRIVATE_PASSWORD); + Document rootPasswordDoc; + // TODO: Remove this block of code in the future when all users have been migrated + if (rootPasswordObject instanceof String) { + if (ParamConstants.OPENCGA_USER_ID.equals(userId)) { + logger.warn("User {} is using the deprecated password format. Please, migrate your code as soon as possible.", userId); + if (!encryptPassword(password, "").equals(rootPasswordObject)) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); + } + return; + } else { + throw new CatalogDBException("User '" + userId + "' is using the deprecated password format. Please, ask your" + + " administrator to run the pending migrations to fix this issue."); + } + } else { + rootPasswordDoc = (Document) rootPasswordObject; + } + // TODO: End of block of code to remove (and replace using commented code below) +// Document rootPasswordDoc = userDocument.get(PRIVATE_PASSWORD, Document.class); + if (rootPasswordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); } - if (userCollection.count(bson).getNumMatches() == 0) { - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); + Document passwordDoc = rootPasswordDoc.get(CURRENT, Document.class); + if (passwordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); + } + + String salt = passwordDoc.getString(SALT); + String hash = encryptPassword(password, salt); + if (!hash.equals(passwordDoc.getString(HASH))) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); } } @Override - public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogDBException { - Query query = new Query(QueryParams.ID.key(), userId); - query.append(QueryParams.EMAIL.key(), email); - Bson bson = parseQuery(query); + public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogException { + return setPassword(userId, null, newPassword); + } - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); + public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, String newPassword) throws CatalogException { + String prefixErrorMsg = "Could not update the password. "; + return runTransaction(clientSession -> { + // 1. Obtain archived passwords + Bson query = Filters.eq(QueryParams.ID.key(), userId); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult userQueryResult = userCollection.find(clientSession, query, projection, QueryOptions.empty()); + if (userQueryResult.getNumResults() == 0) { + throw new CatalogDBException(prefixErrorMsg + "User " + userId + " not found."); + } + Document userDoc = userQueryResult.first(); + Document passwordDoc = userDoc.get(PRIVATE_PASSWORD, Document.class); + + // 1.1. Check oldPassword + if (StringUtils.isNotEmpty(oldPassword)) { + Document currentPasswordDoc = passwordDoc.get(CURRENT, Document.class); + String currentSalt = currentPasswordDoc.getString(SALT); + String currentHash = encryptPassword(oldPassword, currentSalt); + if (!currentHash.equals(currentPasswordDoc.getString(HASH))) { + throw new CatalogAuthenticationException(prefixErrorMsg + "Please, verify that the current password is correct."); + } + } - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw new CatalogDBException("Bad user or email"); - } - return new OpenCGAResult(result); + // 2. Check new password has not been used before + for (Document document : passwordDoc.getList(ARCHIVE, Document.class)) { + String hashValue = document.getString(HASH); + String saltValue = document.getString(SALT); + String encryptedPassword = encryptPassword(newPassword, saltValue); + if (encryptedPassword.equals(hashValue)) { + throw new CatalogAuthenticationException(prefixErrorMsg + "The new password has already been used." + + " Please, use a different one."); + } + } + + // 3. Generate new salt for current password + String newSalt = PasswordUtils.getStrongRandomSalt(); + String newHash = encryptPassword(newPassword, newSalt); + + // 4. Generate update document + UpdateDocument updateDocument = new UpdateDocument(); + // add to current + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_HASH, newHash); + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_SALT, newSalt); + + // add to archive + Document document = new Document() + .append(HASH, newHash) + .append(SALT, newSalt); + updateDocument.getPush().put(PRIVATE_PASSWORD_ARCHIVE, document); + + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED.key(), TimeUtils.getTime()); + if (configuration.getAccount().getPasswordExpirationDays() > 0) { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + String stringDate = TimeUtils.getTime(date); + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE.key(), stringDate); + } + Document update = updateDocument.toFinalUpdateDocument(); + + logger.debug("Change password: query '{}'; update: '{}'", query.toBsonDocument(), update); + DataResult result = userCollection.update(clientSession, query, update, null); + if (result.getNumUpdated() == 0) { + throw new CatalogAuthenticationException("Could not update the password. Please, verify that the current password is" + + " correct."); + } + return new OpenCGAResult(result); + }, e -> logger.error("User {}: {}", userId, e.getMessage())); } @Override @@ -409,7 +523,7 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog for (Document user : queryResult.getResults()) { ArrayList projects = (ArrayList) user.get("projects"); - if (projects.size() > 0) { + if (CollectionUtils.isNotEmpty(projects)) { List projectsTmp = new ArrayList<>(projects.size()); for (Document project : projects) { Query query1 = new Query(ProjectDBAdaptor.QueryParams.UID.key(), project.get(ProjectDBAdaptor @@ -429,7 +543,7 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException { UpdateDocument document = new UpdateDocument(); - final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), ACCOUNT_EXPIRATION_DATE.key()}; + final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), INTERNAL_ACCOUNT_EXPIRATION_DATE.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); if (parameters.containsKey(QueryParams.INTERNAL_STATUS_ID.key())) { @@ -437,7 +551,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer document.getSet().put(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()); } - final String[] acceptedIntParams = {INTERNAL_FAILED_ATTEMPTS.key()}; + final String[] acceptedIntParams = {INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key()}; filterIntParams(parameters, document.getSet(), acceptedIntParams); final String[] acceptedObjectParams = {QueryParams.QUOTA.key()}; @@ -618,18 +732,10 @@ public void forEach(Query query, Consumer action, QueryOptions o } } - public static void main(String[] args) throws CatalogDBException { - System.out.println(encryptPassword("admin")); - } - - private static String encryptPassword(String password) throws CatalogDBException { + private static String encryptPassword(String password, String salt) throws CatalogDBException { if (StringUtils.isNotEmpty(password)) { - if (password.matches("^[a-fA-F0-9]{40}$")) { - // Password already cyphered - return password; - } try { - return CryptoUtils.sha1(password); + return CryptoUtils.sha1(password + salt); } catch (NoSuchAlgorithmException e) { throw new CatalogDBException("Could not encrypt password", e); } @@ -670,8 +776,8 @@ private Bson parseQuery(Query query) throws CatalogDBException { case EMAIL: case ORGANIZATION: case INTERNAL_STATUS_DATE: - case ACCOUNT_AUTHENTICATION_ID: - case ACCOUNT_CREATION_DATE: + case INTERNAL_ACCOUNT_AUTHENTICATION_ID: + case INTERNAL_ACCOUNT_CREATION_DATE: case TOOL_ID: case TOOL_NAME: case TOOL_ALIAS: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java index 604a6073d0f..837349b6b8a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java @@ -18,32 +18,77 @@ import org.apache.avro.generic.GenericRecord; import org.bson.Document; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserInternal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Created by pfurio on 19/01/16. */ public class UserConverter extends OpenCgaMongoConverter { + protected static Logger logger = LoggerFactory.getLogger(UserConverter.class); + public UserConverter() { super(User.class); getObjectMapper().addMixIn(GenericRecord.class, GenericRecordAvroJsonMixin.class); } + @Override + public Document convertToStorageType(User object) { + Document userDocument = super.convertToStorageType(object); + removeDeprecatedAccountObject(userDocument); + return userDocument; + } + @Override public User convertToDataModelType(Document document) { - // TODO: Remove this piece of code once we are sure User contains the migrated new account type from 1.4.2 - Document account = (Document) document.get("account"); - if (account != null && account.get("authentication") == null) { - String authOrigin = account.getString("authOrigin"); - Document authentication = new Document() - .append("id", authOrigin) - .append("application", false); - account.put("authentication", authentication); + User user = super.convertToDataModelType(document); + + restoreFromDeprecatedAccountObject(user); + addToDeprecatedAccountObject(user); + + return user; + } + + /** + * Remove 'account' object from the User document so it is no longer stored in the database. + * Remove after a few releases. + * + * @param userDocument User document. + */ + @Deprecated + private void removeDeprecatedAccountObject(Document userDocument) { + userDocument.remove(UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key()); + } + + /** + * Restores information from the account object to the corresponding internal.account object. + * Added to maintain backwards compatibility with the deprecated account object in TASK-6494 (v3.2.1) + * Remove after a few releases. + * + * @param user User object. + */ + @Deprecated + private void restoreFromDeprecatedAccountObject(User user) { + if (user.getAccount() != null) { + if (user.getInternal() == null) { + user.setInternal(new UserInternal()); + } + user.getInternal().setAccount(user.getAccount()); + logger.warn("Restoring user account information from deprecated account object to internal.account object. " + + "Please, run 'opencga-admin.sh migration run'."); } + } - return super.convertToDataModelType(document); + private void addToDeprecatedAccountObject(User user) { + // Add account to deprecated place + if (user.getInternal() != null && user.getInternal().getAccount() != null && user.getAccount() == null) { + user.setAccount(user.getInternal().getAccount()); + } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java index 3853e91607f..8ac1b5c385f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java @@ -76,6 +76,11 @@ public static CatalogAuthenticationException accountIsExpired(String userId, Str + " talk to your organization owner/administrator."); } + public static CatalogAuthenticationException passwordExpired(String userId, String expirationDate) { + return new CatalogAuthenticationException("The password for the user account '" + userId + "' expired on " + expirationDate + + ". Please, reset your password or talk to your organization owner/administrator."); + } + public static CatalogAuthenticationException userNotAllowed(String domain) { return new CatalogAuthenticationException(domain + ": User not allowed to access the system."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java index 3d3611c5414..da7566340ac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java @@ -73,68 +73,72 @@ public abstract class AbstractManager { logger = LoggerFactory.getLogger(this.getClass()); } + protected DBAdaptorFactory getCatalogDBAdaptorFactory() { + return catalogDBAdaptorFactory; + } + protected MigrationDBAdaptor getMigrationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getMigrationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getMigrationDBAdaptor(organization); } protected MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogMetaDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogMetaDBAdaptor(organization); } protected OrganizationDBAdaptor getOrganizationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogOrganizationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogOrganizationDBAdaptor(organization); } protected UserDBAdaptor getUserDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogUserDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogUserDBAdaptor(organization); } protected ProjectDBAdaptor getProjectDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogProjectDbAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogProjectDbAdaptor(organization); } protected StudyDBAdaptor getStudyDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogStudyDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogStudyDBAdaptor(organization); } protected FileDBAdaptor getFileDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogFileDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogFileDBAdaptor(organization); } protected SampleDBAdaptor getSampleDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogSampleDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogSampleDBAdaptor(organization); } protected IndividualDBAdaptor getIndividualDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogIndividualDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogIndividualDBAdaptor(organization); } protected JobDBAdaptor getJobDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organization); } protected AuditDBAdaptor getAuditDbAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogAuditDbAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogAuditDbAdaptor(organization); } protected CohortDBAdaptor getCohortDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogCohortDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogCohortDBAdaptor(organization); } protected PanelDBAdaptor getPanelDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogPanelDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogPanelDBAdaptor(organization); } protected FamilyDBAdaptor getFamilyDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogFamilyDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogFamilyDBAdaptor(organization); } protected ClinicalAnalysisDBAdaptor getClinicalAnalysisDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getClinicalAnalysisDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getClinicalAnalysisDBAdaptor(organization); } protected InterpretationDBAdaptor getInterpretationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getInterpretationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getInterpretationDBAdaptor(organization); } protected void fixQueryObject(Query query) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java index 09bc57aabf1..7edf1030496 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java @@ -67,7 +67,7 @@ public OpenCGAResult userSearch(String organizationId, Query query, QueryO query.remove(ParamConstants.USER); } if (query.containsKey(ParamConstants.USER_CREATION_DATE)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); query.remove(ParamConstants.USER_CREATION_DATE); } @@ -134,7 +134,7 @@ public List getOrganizationIds(String token) throws CatalogException { JwtPayload payload = catalogManager.getUserManager().validateToken(token); try { authorizationManager.checkIsOpencgaAdministrator(payload); - List organizationIds = catalogDBAdaptorFactory.getOrganizationIds(); + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); auditManager.audit(ParamConstants.ADMIN_ORGANIZATION, payload.getUserId(), Enums.Action.FETCH_ORGANIZATION_IDS, Enums.Resource.STUDY, "", "", "", "", new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 1409b1e9d4c..030505395d0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -48,7 +48,6 @@ import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; @@ -278,7 +277,7 @@ private void privateInstall(String algorithm, String secretKey, String password, organizationConfiguration, null), QueryOptions.empty(), null); - User user = new User(OPENCGA, new Account().setExpirationDate("")) + User user = new User(OPENCGA) .setEmail(StringUtils.isEmpty(email) ? "opencga@admin.com" : email) .setOrganization(ADMIN_ORGANIZATION); userManager.create(user, password, null); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 452748329d7..20b228aeed8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -25,7 +25,6 @@ import org.opencb.biodata.models.clinical.ClinicalAudit; import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -43,6 +42,7 @@ import org.opencb.opencga.catalog.models.InternalGetDataResult; import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; @@ -51,10 +51,7 @@ import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.clinical.*; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.common.FlagAnnotation; -import org.opencb.opencga.core.models.common.FlagValue; +import org.opencb.opencga.core.models.common.*; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.family.FamilyCreateParams; import org.opencb.opencga.core.models.file.File; @@ -277,7 +274,7 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis List events = new LinkedList<>(); - clinicalAnalysis.setStatus(ParamUtils.defaultObject(clinicalAnalysis.getStatus(), Status::new)); + clinicalAnalysis.setStatus(ParamUtils.defaultObject(clinicalAnalysis.getStatus(), ClinicalStatus::new)); clinicalAnalysis.setInternal(ClinicalAnalysisInternal.init()); clinicalAnalysis.setDisorder(ParamUtils.defaultObject(clinicalAnalysis.getDisorder(), new Disorder("", "", "", Collections.emptyMap(), "", Collections.emptyList()))); @@ -591,10 +588,10 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis validateCustomPriorityParameters(clinicalAnalysis, clinicalConfiguration); validateCustomFlagParameters(clinicalAnalysis, clinicalConfiguration); validateCustomConsentParameters(clinicalAnalysis, clinicalConfiguration); - validateStatusParameter(clinicalAnalysis, clinicalConfiguration); + validateStatusParameter(clinicalAnalysis, clinicalConfiguration, userId, true); if (StringUtils.isNotEmpty(clinicalAnalysis.getStatus().getId())) { - List clinicalStatusValues = clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = clinicalConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (clinicalAnalysis.getStatus().getId().equals(clinicalStatusValue.getId())) { if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -802,17 +799,15 @@ private void fillResponsible(String organizationId, ClinicalResponsible responsi responsible.setEmail(ParamUtils.defaultString(responsible.getEmail(), result.first().getEmail())); } - private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, ClinicalAnalysisStudyConfiguration clinicalConfiguration) - throws CatalogException { + private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, ClinicalAnalysisStudyConfiguration clinicalConfiguration, + String userId, boolean initIfUndefined) throws CatalogException { // Status - if (clinicalConfiguration.getStatus() == null - || CollectionUtils.isEmpty(clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()))) { - throw new CatalogException("Missing status configuration in study for type '" + clinicalAnalysis.getType() - + "'. Please add a proper set of valid statuses."); + if (CollectionUtils.isEmpty(clinicalConfiguration.getStatus())) { + throw new CatalogException("Missing status configuration in study. Please add a proper set of valid statuses."); } if (StringUtils.isNotEmpty(clinicalAnalysis.getStatus().getId())) { Map statusMap = new HashMap<>(); - for (ClinicalStatusValue status : clinicalConfiguration.getStatus().get(clinicalAnalysis.getType())) { + for (ClinicalStatusValue status : clinicalConfiguration.getStatus()) { statusMap.put(status.getId(), status); } if (!statusMap.containsKey(clinicalAnalysis.getStatus().getId())) { @@ -821,10 +816,26 @@ private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, Clinical } ClinicalStatusValue clinicalStatusValue = statusMap.get(clinicalAnalysis.getStatus().getId()); clinicalAnalysis.getStatus().setDescription(clinicalStatusValue.getDescription()); - clinicalAnalysis.getStatus().setDate(TimeUtils.getTime()); + clinicalAnalysis.getStatus().setType(clinicalStatusValue.getType()); } else if (clinicalAnalysis.getStatus().getId() == null) { - clinicalAnalysis.getStatus().setId(""); + if (initIfUndefined) { + // Look for first status of type NOT_STARTED + for (ClinicalStatusValue status : clinicalConfiguration.getStatus()) { + if (status.getType() == ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) { + clinicalAnalysis.getStatus().setId(status.getId()); + clinicalAnalysis.getStatus().setDescription(status.getDescription()); + clinicalAnalysis.getStatus().setType(status.getType()); + break; + } + } + } else { + throw new CatalogException("Missing status id in clinical analysis"); + } } + clinicalAnalysis.getStatus().setDate(TimeUtils.getTime()); + clinicalAnalysis.getStatus().setVersion(GitRepositoryState.getInstance().getBuildVersion()); + clinicalAnalysis.getStatus().setCommit(GitRepositoryState.getInstance().getCommitId()); + clinicalAnalysis.getStatus().setAuthor(userId); } private void validateCustomConsentParameters(ClinicalAnalysis clinicalAnalysis, @@ -879,12 +890,11 @@ private void validateCustomFlagParameters(ClinicalAnalysis clinicalAnalysis, Cli throws CatalogException { // Flag definition if (CollectionUtils.isNotEmpty(clinicalAnalysis.getFlags())) { - if (CollectionUtils.isEmpty(clinicalConfiguration.getFlags().get(clinicalAnalysis.getType()))) { - throw new CatalogException("Missing flags configuration in study for type '" + clinicalAnalysis.getType() - + "'. Please add a proper set of valid priorities."); + if (CollectionUtils.isEmpty(clinicalConfiguration.getFlags())) { + throw new CatalogException("Missing flags configuration. Please add a proper set of valid flags."); } Map supportedFlags = new HashMap<>(); - for (FlagValue flagValue : clinicalConfiguration.getFlags().get(clinicalAnalysis.getType())) { + for (FlagValue flagValue : clinicalConfiguration.getFlags()) { supportedFlags.put(flagValue.getId(), flagValue); } @@ -898,8 +908,8 @@ private void validateCustomFlagParameters(ClinicalAnalysis clinicalAnalysis, Cli flag.setDescription(supportedFlags.get(flag.getId()).getDescription()); flag.setDate(TimeUtils.getTime()); } else { - throw new CatalogException("Flag '" + flag.getId() + "' not supported. Supported flags for Clinical Analyses of " - + "type '" + clinicalAnalysis.getType() + "' are: '" + String.join(", ", supportedFlags.keySet()) + "'."); + throw new CatalogException("Flag '" + flag.getId() + "' not supported. Supported flags for Clinical Analyses are: '" + + String.join(", ", supportedFlags.keySet()) + "'."); } } clinicalAnalysis.setFlags(new ArrayList<>(flagMap.values())); @@ -1393,15 +1403,65 @@ private OpenCGAResult update(String organizationId, Study stud } ClinicalAnalysisStudyConfiguration clinicalConfiguration = study.getInternal().getConfiguration().getClinical(); + // Get the clinical status that are CLOSED and DONE + Set closedStatus = new HashSet<>(); + Set doneStatus = new HashSet<>(); + for (ClinicalStatusValue clinicalStatusValue : clinicalConfiguration.getStatus()) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + closedStatus.add(clinicalStatusValue.getId()); + } else if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.DONE)) { + doneStatus.add(clinicalStatusValue.getId()); + } + } + + // If the current clinical analysis: + // - is locked or panelLocked + // - the user wants to update the locked or panelLocked status + // - the user wants to update the status to/from a done|closed status + boolean adminPermissionsChecked = false; + if (clinicalAnalysis.isLocked() || clinicalAnalysis.isPanelLocked() + || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED + || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.DONE + || updateParamsClone.getLocked() != null + || updateParams.getPanelLocked() != null + || (updateParams.getStatus() != null && (closedStatus.contains(updateParams.getStatus().getId()) + || doneStatus.contains(updateParams.getStatus().getId())))) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.ADMIN); + + // Current status is of type CLOSED + if (clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + // The only allowed action is to remove the CLOSED status + if (updateParams.getStatus() == null || StringUtils.isEmpty(updateParams.getStatus().getId())) { + throw new CatalogException("Cannot update a ClinicalAnalysis with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the ClinicalAnalysis."); + } else if (closedStatus.contains(updateParams.getStatus().getId())) { + // Users should be able to change from one CLOSED status to a different one but we should still control that no further + // modifications are made + if (parameters.size() > 1) { + throw new CatalogException("Cannot update a ClinicalAnalysis with a " + + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status. You need to remove the " + + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able to perform further updates on " + + "the ClinicalAnalysis."); + } else if (clinicalAnalysis.getStatus().getId().equals(updateParams.getStatus().getId())) { + throw new CatalogException("ClinicalAnalysis already have the status '" + clinicalAnalysis.getStatus().getId() + + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } + } + } + + adminPermissionsChecked = true; + } // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets - if (updateParamsClone.getAnnotationSets() != null) { + if (!adminPermissionsChecked && updateParamsClone.getAnnotationSets() != null) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets - if ((parameters.size() == 1 && !parameters.containsKey(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key())) - || parameters.size() > 1) { + if (!adminPermissionsChecked && ((parameters.size() == 1 + && !parameters.containsKey(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1)) { authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE); } @@ -1533,13 +1593,14 @@ private OpenCGAResult update(String organizationId, Study stud parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key(), clinicalAnalysis.getFiles()); } - if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels()) && updateParamsClone.getPanelLock() != null - && updateParamsClone.getPanelLock()) { - throw new CatalogException("Updating the list of panels and setting 'panelLock' to true at the same time is not allowed."); + if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels()) && updateParamsClone.getPanelLocked() != null + && updateParamsClone.getPanelLocked()) { + throw new CatalogException("Updating the list of panels and setting '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' to true at the same time is not allowed."); } if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels())) { - if (clinicalAnalysis.isPanelLock() && (updateParamsClone.getPanelLock() == null || updateParamsClone.getPanelLock())) { + if (clinicalAnalysis.isPanelLocked() && (updateParamsClone.getPanelLocked() == null || updateParamsClone.getPanelLocked())) { throw new CatalogException("Cannot update panels from ClinicalAnalysis '" + clinicalAnalysis.getId() + "'. " + "'panelLocked' field from ClinicalAnalysis is set to true."); } @@ -1556,15 +1617,17 @@ private OpenCGAResult update(String organizationId, Study stud parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), panelResult.getResults()); } - if (updateParamsClone.getPanelLock() != null && updateParamsClone.getPanelLock() && !clinicalAnalysis.isPanelLock()) { + if (updateParamsClone.getPanelLocked() != null && updateParamsClone.getPanelLocked() && !clinicalAnalysis.isPanelLocked()) { // if user wants to set panelLock to true // We need to check if the CA has interpretations. If so, the interpretations should contain at least one of the case panels // in order to set panelLock to true. Otherwise, that action is not allowed. Set panelIds = clinicalAnalysis.getPanels().stream().map(Panel::getId).collect(Collectors.toSet()); String exceptionMsgPrefix = "The interpretation '"; - String exceptionMsgSuffix = "' does not contain any of the case panels. 'panelLock' can only be set to true if all" + String exceptionMsgSuffix = "' does not contain any of the case panels. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' can only be set to true if all" + " all Interpretations contains a non-empty subset of the panels used by the case."; - String alternativeExceptionMsgSuffix = "' is using a panel not defined by the case. 'panelLock' can only be set to true if all" + String alternativeExceptionMsgSuffix = "' is using a panel not defined by the case. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' can only be set to true if all" + " all Interpretations contains a non-empty subset of the panels used by the case."; if (clinicalAnalysis.getInterpretation() != null) { if (CollectionUtils.isEmpty(clinicalAnalysis.getInterpretation().getPanels())) { @@ -1614,8 +1677,8 @@ private OpenCGAResult update(String organizationId, Study stud if (parameters.containsKey(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key())) { Map status = (Map) parameters.get(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key()); - if (!(status instanceof Map) || StringUtils.isEmpty(String.valueOf(status.get("name"))) - || !ClinicalAnalysisStatus.isValid(String.valueOf(status.get("name")))) { + if (!(status instanceof Map) || StringUtils.isEmpty(String.valueOf(status.get("id"))) + || !InternalStatus.isValid(String.valueOf(status.get("id")))) { throw new CatalogException("Missing or invalid status"); } } @@ -1659,12 +1722,12 @@ private OpenCGAResult update(String organizationId, Study stud parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.CONSENT.key(), clinicalAnalysis.getConsent()); } if (parameters.containsKey(ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key())) { - clinicalAnalysis.setStatus(updateParamsClone.getStatus().toStatus()); - validateStatusParameter(clinicalAnalysis, clinicalConfiguration); + clinicalAnalysis.setStatus(updateParamsClone.getStatus().toClinicalStatus()); + validateStatusParameter(clinicalAnalysis, clinicalConfiguration, userId, false); parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key(), clinicalAnalysis.getStatus()); if (StringUtils.isNotEmpty(updateParamsClone.getStatus().getId())) { - List clinicalStatusValues = clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = clinicalConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (updateParamsClone.getStatus().getId().equals(clinicalStatusValue.getId())) { if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -2636,23 +2699,29 @@ public OpenCGAResult configureStudy(String studyStr, ClinicalAnalysisStudyConfig } } - private void validateClinicalStatus(Map> status, String field) + private void validateClinicalStatus(List status, String field) throws CatalogException { - if (status == null) { + if (CollectionUtils.isEmpty(status)) { throw CatalogParameterException.isNull(field); } - for (ClinicalAnalysis.Type value : ClinicalAnalysis.Type.values()) { - if (!status.containsKey(value)) { - throw new CatalogParameterException(field + ": Missing status values for ClinicalAnalysis type '" + value + "'"); + // Ensure there's at least one status id per status type + Map presentMap = new HashMap<>(); + for (ClinicalStatusValue.ClinicalStatusType value : ClinicalStatusValue.ClinicalStatusType.values()) { + // Init map + presentMap.put(value, false); + } + for (ClinicalStatusValue clinicalStatusValue : status) { + if (StringUtils.isEmpty(clinicalStatusValue.getId())) { + throw CatalogParameterException.isNull(field + ".id"); } - List statuses = status.get(value); - for (ClinicalStatusValue clinicalStatusValue : statuses) { - if (StringUtils.isEmpty(clinicalStatusValue.getId())) { - throw CatalogParameterException.isNull(field + ".{" + value + "}.id"); - } - if (clinicalStatusValue.getType() == null) { - throw CatalogParameterException.isNull(field + ".{" + value + "}.type"); - } + if (clinicalStatusValue.getType() == null) { + throw CatalogParameterException.isNull(field + ".type"); + } + presentMap.put(clinicalStatusValue.getType(), true); + } + for (Map.Entry entry : presentMap.entrySet()) { + if (!entry.getValue()) { + throw new CatalogException("Missing status values for ClinicalStatus type '" + entry.getKey() + "'"); } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index 37be3aeeba2..386eb98cafa 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -3258,7 +3258,6 @@ private void checkCanDeleteFile(String organizationId, Study study, String fileI // We need to remove the reference to the transformed files and change their status from TRANSFORMED to NONE next.getInternal().getVariant().getIndex().setTransform(null); next.getInternal().getVariant().getIndex().getStatus().setId(VariantIndexStatus.NONE); - next.getInternal().getVariant().getIndex().getStatus().setName(VariantIndexStatus.NONE); break; case VariantIndexStatus.NONE: case VariantIndexStatus.DELETED: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 95ae214a1d7..417978cef13 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -27,7 +27,6 @@ import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; import org.opencb.biodata.models.clinical.interpretation.InterpretationStats; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -45,6 +44,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.JwtPayload; @@ -69,18 +69,16 @@ public class InterpretationManager extends ResourceManager { public static final QueryOptions INCLUDE_CLINICAL_ANALYSIS = keepFieldsInQueryOptions(ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, - Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key())); + Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key(), + ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key())); public static final QueryOptions INCLUDE_INTERPRETATION_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( InterpretationDBAdaptor.QueryParams.ID.key(), InterpretationDBAdaptor.QueryParams.UID.key(), InterpretationDBAdaptor.QueryParams.UUID.key(), InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), - InterpretationDBAdaptor.QueryParams.LOCKED.key(), + InterpretationDBAdaptor.QueryParams.LOCKED.key(), InterpretationDBAdaptor.QueryParams.STATUS.key(), InterpretationDBAdaptor.QueryParams.VERSION.key(), InterpretationDBAdaptor.QueryParams.STUDY_UID.key())); - public static final QueryOptions INCLUDE_INTERPRETATION_FINDING_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - InterpretationDBAdaptor.QueryParams.ID.key(), InterpretationDBAdaptor.QueryParams.UID.key(), - InterpretationDBAdaptor.QueryParams.UUID.key(), InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), - InterpretationDBAdaptor.QueryParams.VERSION.key(), InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), - InterpretationDBAdaptor.QueryParams.LOCKED.key(), InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS_ID.key(), - InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS_ID.key())); + public static final QueryOptions INCLUDE_INTERPRETATION_FINDING_IDS = keepFieldsInQueryOptions(INCLUDE_INTERPRETATION_IDS, + Arrays.asList(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS_ID.key(), + InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS_ID.key())); protected static Logger logger = LoggerFactory.getLogger(InterpretationManager.class); private UserManager userManager; private StudyManager studyManager; @@ -219,7 +217,7 @@ public OpenCGAResult create(String studyStr, String clinicalAnal try { QueryOptions clinicalOptions = keepFieldsInQueryOptions(ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), - ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key(), + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.AUDIT.key(), ClinicalAnalysisDBAdaptor.QueryParams.INTERPRETATION_ID.key(), ClinicalAnalysisDBAdaptor.QueryParams.SECONDARY_INTERPRETATIONS_ID.key())); @@ -277,6 +275,7 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio } interpretation.setId(clinicalAnalysis.getId() + "." + count); + interpretation.setName(ParamUtils.defaultString(interpretation.getName(), interpretation.getId())); interpretation.setClinicalAnalysisId(clinicalAnalysis.getId()); @@ -290,7 +289,7 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio interpretation.setPrimaryFindings(ParamUtils.defaultObject(interpretation.getPrimaryFindings(), Collections.emptyList())); interpretation.setSecondaryFindings(ParamUtils.defaultObject(interpretation.getSecondaryFindings(), Collections.emptyList())); interpretation.setComments(ParamUtils.defaultObject(interpretation.getComments(), Collections.emptyList())); - interpretation.setStatus(ParamUtils.defaultObject(interpretation.getStatus(), Status::new)); + interpretation.setStatus(ParamUtils.defaultObject(interpretation.getStatus(), ClinicalStatus::new)); interpretation.setRelease(studyManager.getCurrentRelease(study)); interpretation.setVersion(1); interpretation.setAttributes(ParamUtils.defaultObject(interpretation.getAttributes(), Collections.emptyMap())); @@ -300,7 +299,7 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio if (CollectionUtils.isEmpty(interpretation.getPanels())) { interpretation.setPanels(clinicalAnalysis.getPanels()); } else { - if (clinicalAnalysis.isPanelLock()) { + if (clinicalAnalysis.isPanelLocked()) { // Check the panels are the same provided in the Clinical Analysis Map clinicalPanelIds = clinicalAnalysis.getPanels().stream() .collect(Collectors.toMap(DiseasePanel::getId, panel -> panel)); @@ -308,7 +307,8 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio List panelList = new ArrayList<>(clinicalPanelIds.size()); for (Panel panel : interpretation.getPanels()) { if (!clinicalPanelIds.containsKey(panel.getId())) { - throw new CatalogException("'panelLock' from ClinicalAnalysis is set to True. Please, leave list of panels empty" + throw new CatalogException("'" + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + + "' from ClinicalAnalysis is set to True. Please, leave list of panels empty" + " so they can be inherited or pass at least a subset of the panels defined in the Clinical Analysis."); } panelList.add(clinicalPanelIds.get(panel.getId())); @@ -331,9 +331,9 @@ void validateNewInterpretation(String organizationId, Study study, Interpretatio } // Validate status - validateStatusParameter(interpretation, clinicalAnalysis.getType(), interpretationConfiguration); + validateStatusParameter(interpretation, interpretationConfiguration, userId, true); if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { - List clinicalStatusValues = interpretationConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = interpretationConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (interpretation.getStatus().getId().equals(clinicalStatusValue.getId()) && clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -413,6 +413,16 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); auditManager.initAuditBatch(operationId); + InterpretationStudyConfiguration interpretationConfiguration = + study.getInternal().getConfiguration().getClinical().getInterpretation(); + ClinicalStatusValue initStatus = null; + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { + if (status.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { + initStatus = status; + break; + } + } + OpenCGAResult result = OpenCGAResult.empty(); for (String interpretationStr : interpretationList) { String interpretationId = interpretationStr; @@ -457,13 +467,13 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly actionMap.put(InterpretationDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - InterpretationUpdateParams params = new InterpretationUpdateParams("", new ClinicalAnalystParam(), + InterpretationUpdateParams params = new InterpretationUpdateParams("", "", new ClinicalAnalystParam(), InterpretationMethod.init(), null, null, Collections.emptyList(), Collections.emptyList(), clinicalAnalysis.getPanels() != null ? clinicalAnalysis.getPanels().stream() .map(p -> new PanelReferenceParam().setId(p.getId())).collect(Collectors.toList()) : null, - Collections.emptyList(), new StatusParam(), false, new ObjectMap()); + Collections.emptyList(), new StatusParam(initStatus.getId()), false, new ObjectMap()); ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.CLEAR_INTERPRETATION, "Clear interpretation '" + interpretationId + "'", TimeUtils.getTime()); @@ -629,89 +639,89 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly // } // } - public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, - ParamUtils.SaveInterpretationAs as, QueryOptions options, String token) - throws CatalogException { - return update(studyStr, query, updateParams, as, false, options, token); - } - - public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, - ParamUtils.SaveInterpretationAs as, boolean ignoreException, QueryOptions options, - String token) throws CatalogException { - options = ParamUtils.defaultObject(options, QueryOptions::new); - - JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); - CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); - String organizationId = studyFqn.getOrganizationId(); - String userId = tokenPayload.getUserId(organizationId); - Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); - - String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); - - ObjectMap updateMap; - try { - updateMap = updateParams != null ? updateParams.getUpdateMap() : null; - } catch (JsonProcessingException e) { - throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); - } - - ObjectMap auditParams = new ObjectMap() - .append("study", studyStr) - .append("query", query) - .append("updateParams", updateMap) - .append("as", as) - .append("ignoreException", ignoreException) - .append("options", options) - .append("token", token); - - Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); - fixQueryObject(organizationId, study, finalQuery, userId); - - DBIterator iterator; - try { - finalQuery.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = getInterpretationDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_INTERPRETATION_FINDING_IDS, - userId); - } catch (CatalogException e) { - auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, "", "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - throw e; - } - - auditManager.initAuditBatch(operationId); - OpenCGAResult result = OpenCGAResult.empty(); - while (iterator.hasNext()) { - Interpretation interpretation = iterator.next(); - try { - List clinicalAuditList = new ArrayList<>(); - clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_INTERPRETATION, - "Update interpretation '" + interpretation.getId() + "'", TimeUtils.getTime())); - if (as != null) { - clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, - "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); - } - OpenCGAResult writeResult = update(organizationId, study, interpretation, updateParams, clinicalAuditList, as, options, - userId); - auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), - interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - - result.append(writeResult); - } catch (CatalogException e) { - Event event = new Event(Event.Type.ERROR, interpretation.getId(), e.getMessage()); - result.getEvents().add(event); - result.setNumErrors(result.getNumErrors() + 1); - - logger.error("Cannot update interpretation {}: {}", interpretation.getId(), e.getMessage(), e); - auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), - interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - } - } - auditManager.finishAuditBatch(organizationId, operationId); - - return endResult(result, ignoreException); - } +// public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, +// ParamUtils.SaveInterpretationAs as, QueryOptions options, String token) +// throws CatalogException { +// return update(studyStr, query, updateParams, as, false, options, token); +// } +// +// public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, +// ParamUtils.SaveInterpretationAs as, boolean ignoreException, QueryOptions options, +// String token) throws CatalogException { +// options = ParamUtils.defaultObject(options, QueryOptions::new); +// +// JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); +// CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); +// String organizationId = studyFqn.getOrganizationId(); +// String userId = tokenPayload.getUserId(organizationId); +// Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); +// +// String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); +// +// ObjectMap updateMap; +// try { +// updateMap = updateParams != null ? updateParams.getUpdateMap() : null; +// } catch (JsonProcessingException e) { +// throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); +// } +// +// ObjectMap auditParams = new ObjectMap() +// .append("study", studyStr) +// .append("query", query) +// .append("updateParams", updateMap) +// .append("as", as) +// .append("ignoreException", ignoreException) +// .append("options", options) +// .append("token", token); +// +// Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); +// fixQueryObject(organizationId, study, finalQuery, userId); +// +// DBIterator iterator; +// try { +// finalQuery.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); +// iterator = getInterpretationDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_INTERPRETATION_FINDING_IDS, +// userId); +// } catch (CatalogException e) { +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, "", "", study.getId(), +// study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); +// throw e; +// } +// +// auditManager.initAuditBatch(operationId); +// OpenCGAResult result = OpenCGAResult.empty(); +// while (iterator.hasNext()) { +// Interpretation interpretation = iterator.next(); +// try { +// List clinicalAuditList = new ArrayList<>(); +// clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_INTERPRETATION, +// "Update interpretation '" + interpretation.getId() + "'", TimeUtils.getTime())); +// if (as != null) { +// clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, +// "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); +// } +// OpenCGAResult writeResult = update(organizationId, study, interpretation, updateParams, clinicalAuditList, as, options, +// userId); +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), +// interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, +// new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); +// +// result.append(writeResult); +// } catch (CatalogException e) { +// Event event = new Event(Event.Type.ERROR, interpretation.getId(), e.getMessage()); +// result.getEvents().add(event); +// result.setNumErrors(result.getNumErrors() + 1); +// +// logger.error("Cannot update interpretation {}: {}", interpretation.getId(), e.getMessage(), e); +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), +// interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, +// new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); +// } +// } +// auditManager.finishAuditBatch(organizationId, operationId); +// +// return endResult(result, ignoreException); +// } public OpenCGAResult update(String studyStr, String clinicalAnalysisId, String intepretationId, InterpretationUpdateParams updateParams, ParamUtils.SaveInterpretationAs as, @@ -910,17 +920,89 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation InterpretationStudyConfiguration interpretationConfiguration = study.getInternal().getConfiguration().getClinical().getInterpretation(); + ObjectMap parameters = new ObjectMap(); + if (updateParams != null) { + try { + parameters = updateParams.getUpdateMap(); + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); + } + } else { + throw new CatalogException("Missing interpretation update parameters"); + } + Map actionMap = options.getMap(Constants.ACTIONS); // Check if user has permissions to write clinical analysis ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().internalGet(organizationId, study.getUid(), interpretation.getClinicalAnalysisId(), INCLUDE_CLINICAL_ANALYSIS, userId).first(); - authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, - ClinicalAnalysisPermissions.WRITE); +// if (clinicalAnalysis.isLocked()) { +// throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" +// + " the Interpretation."); +// } + if (clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + throw new CatalogException("Cannot update the Interpretation. Case status is " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } + + InterpretationStudyConfiguration interpretationStudyConfiguration = study.getInternal().getConfiguration().getClinical() + .getInterpretation(); + // Get the interpretation status that are CLOSED and DONE + Set closedStatus = new HashSet<>(); + Set doneStatus = new HashSet<>(); + for (ClinicalStatusValue clinicalStatusValue : interpretationStudyConfiguration.getStatus()) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + closedStatus.add(clinicalStatusValue.getId()); + } else if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.DONE)) { + doneStatus.add(clinicalStatusValue.getId()); + } + } + + // If the current interpretation: + // - is locked + // - the user wants to update the locked status + // - the user wants to update the status to/from a done|closed status + boolean adminPermissionsChecked = false; + if (interpretation.isLocked() + || interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED + || interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.DONE + || updateParams.getLocked() != null + || (updateParams.getStatus() != null && (closedStatus.contains(updateParams.getStatus().getId()) + || doneStatus.contains(updateParams.getStatus().getId())))) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.ADMIN); + + // Current status is of type CLOSED + if (interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + // The only allowed action is to remove the CLOSED status + if (updateParams.getStatus() == null || StringUtils.isEmpty(updateParams.getStatus().getId())) { + throw new CatalogException("Cannot update a Interpretation with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the Interpretation."); + } else if (closedStatus.contains(updateParams.getStatus().getId())) { + // Users should be able to change from one CLOSED status to a different one but we should still control that no further + // modifications are made + if (parameters.size() > 1) { + throw new CatalogException("Cannot update a Interpretation with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the Interpretation."); + } else if (interpretation.getStatus().getId().equals(updateParams.getStatus().getId())) { + throw new CatalogException("Interpretation already have the status '" + interpretation.getStatus().getId() + + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } + } + } - if (clinicalAnalysis.isLocked()) { - throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" - + " the Interpretation."); + adminPermissionsChecked = true; + } + + if (!adminPermissionsChecked) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.WRITE); + } + + if (clinicalAnalysis.isLocked() && updateParams.getLocked() != null && !updateParams.getLocked()) { + throw new CatalogException("Could not unlock the Interpretation. Case is locked so unlocking the Interpretation is not" + + " allowed."); } List events = new ArrayList<>(); @@ -932,20 +1014,11 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation ParamUtils.checkDateFormat(updateParams.getModificationDate(), InterpretationDBAdaptor.QueryParams.MODIFICATION_DATE.key()); } - ObjectMap parameters = new ObjectMap(); - if (updateParams != null) { - try { - parameters = updateParams.getUpdateMap(); - } catch (JsonProcessingException e) { - throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); - } - } - - if (!parameters.isEmpty() && interpretation.isLocked() - && parameters.getBoolean(InterpretationDBAdaptor.QueryParams.LOCKED.key(), true)) { - throw new CatalogException("Could not update the Interpretation. Interpretation '" + interpretation.getId() - + " is locked. Please, unlock it first."); - } +// if (!parameters.isEmpty() && interpretation.isLocked() +// && parameters.getBoolean(InterpretationDBAdaptor.QueryParams.LOCKED.key(), true)) { +// throw new CatalogException("Could not update the Interpretation. Interpretation '" + interpretation.getId() +// + " is locked. Please, unlock it first."); +// } if (updateParams != null && updateParams.getComments() != null && !updateParams.getComments().isEmpty()) { List comments = new ArrayList<>(updateParams.getComments().size()); @@ -982,15 +1055,15 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation } if (updateParams != null && CollectionUtils.isNotEmpty(updateParams.getPanels())) { - if (clinicalAnalysis.isPanelLock()) { - throw new CatalogException("Updating panels from Interpretation is not allowed. 'panelLock' from ClinicalAnalysis is set " - + "to True."); + if (clinicalAnalysis.isPanelLocked()) { + throw new CatalogException("Updating panels from Interpretation is not allowed. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' from ClinicalAnalysis is set to True."); } // Validate and get panels List panelIds = updateParams.getPanels().stream().map(PanelReferenceParam::getId).collect(Collectors.toList()); Query query = new Query(PanelDBAdaptor.QueryParams.ID.key(), panelIds); - OpenCGAResult panelResult = + OpenCGAResult panelResult = getPanelDBAdaptor(organizationId).get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); if (panelResult.getNumResults() < panelIds.size()) { throw new CatalogException("Some panels were not found or user doesn't have permissions to see them"); @@ -1068,12 +1141,12 @@ private OpenCGAResult update(String organizationId, Study study, Interpretation } if (parameters.containsKey(InterpretationDBAdaptor.QueryParams.STATUS.key())) { - interpretation.setStatus(updateParams.getStatus().toStatus()); - validateStatusParameter(interpretation, clinicalAnalysis.getType(), interpretationConfiguration); + interpretation.setStatus(updateParams.getStatus().toClinicalStatus()); + validateStatusParameter(interpretation, interpretationConfiguration, userId, false); parameters.put(InterpretationDBAdaptor.QueryParams.STATUS.key(), interpretation.getStatus()); if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { - List clinicalStatusValues = interpretationConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = interpretationConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (interpretation.getStatus().getId().equals(clinicalStatusValue.getId()) && clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -1128,8 +1201,9 @@ public OpenCGAResult revert(String studyStr, String clinicalAnal throw new CatalogException("Could not revert the Interpretation. Case is locked so no further modifications can be made to" + " the Interpretation."); } - if (clinicalAnalysis.isPanelLock()) { - throw new CatalogException("Could not revert the Interpretation. 'panelLock' is set to True, so no further modifications" + if (clinicalAnalysis.isPanelLocked()) { + throw new CatalogException("Could not revert the Interpretation. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' is set to True, so no further modifications" + " can be made to the Interpretation."); } @@ -1515,17 +1589,15 @@ protected void fixQueryObject(String organizationId, Study study, Query query, S } } - private void validateStatusParameter(Interpretation interpretation, ClinicalAnalysis.Type type, - InterpretationStudyConfiguration interpretationConfiguration) throws CatalogException { + private void validateStatusParameter(Interpretation interpretation, InterpretationStudyConfiguration interpretationConfiguration, + String userId, boolean initIfUndefined) throws CatalogException { // Status - if (interpretationConfiguration.getStatus() == null - || CollectionUtils.isEmpty(interpretationConfiguration.getStatus().get(type))) { - throw new CatalogException("Missing status configuration in study for type '" + type - + "'. Please add a proper set of valid statuses."); + if (CollectionUtils.isEmpty(interpretationConfiguration.getStatus())) { + throw new CatalogException("Missing status configuration in study. Please add a proper set of valid statuses."); } if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { Map statusMap = new HashMap<>(); - for (ClinicalStatusValue status : interpretationConfiguration.getStatus().get(type)) { + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { statusMap.put(status.getId(), status); } if (!statusMap.containsKey(interpretation.getStatus().getId())) { @@ -1534,7 +1606,25 @@ private void validateStatusParameter(Interpretation interpretation, ClinicalAnal } ClinicalStatusValue clinicalStatusValue = statusMap.get(interpretation.getStatus().getId()); interpretation.getStatus().setDescription(clinicalStatusValue.getDescription()); - interpretation.getStatus().setDate(TimeUtils.getTime()); + interpretation.getStatus().setType(clinicalStatusValue.getType()); + } else { + if (initIfUndefined) { + // Look for first status of type NOT_STARTED + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { + if (status.getType() == ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) { + interpretation.getStatus().setId(status.getId()); + interpretation.getStatus().setDescription(status.getDescription()); + interpretation.getStatus().setType(status.getType()); + break; + } + } + } else { + throw new CatalogException("Missing status id in Interpretation"); + } } + interpretation.getStatus().setDate(TimeUtils.getTime()); + interpretation.getStatus().setVersion(GitRepositoryState.getInstance().getBuildVersion()); + interpretation.getStatus().setCommit(GitRepositoryState.getInstance().getCommitId()); + interpretation.getStatus().setAuthor(userId); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index b8b9ba55bae..fe7ffc18e44 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -421,7 +421,7 @@ public OpenCGAResult kill(String studyStr, String jobId, String token) thro } ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_KILL_JOB_REQUESTED.key(), true); - OpenCGAResult update = catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, QueryOptions.empty()); auditManager.audit(organizationId, userId, Enums.Action.KILL_JOB, Enums.Resource.JOB, jobId, jobUuid, study.getId(), @@ -816,7 +816,7 @@ public DBIterator iterator(String studyId, Query query, QueryOptions option public OpenCGAResult countInOrganization(String organizationId, Query query, String token) throws CatalogException { JwtPayload jwtPayload = userManager.validateToken(token); authorizationManager.checkIsOpencgaAdministrator(jwtPayload); - return catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organizationId).count(query); + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).count(query); } @Override diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java index b3c7139bd30..f32c8422dec 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java @@ -54,7 +54,7 @@ private OpenCGAResult internalGet(String organizationId, long studyUid, St } else { query.put(NoteDBAdaptor.QueryParams.ID.key(), noteId); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(query, QueryOptions.empty()); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, QueryOptions.empty()); if (result.getNumResults() == 0) { throw CatalogException.notFound("note", Collections.singletonList(noteId)); } @@ -88,7 +88,7 @@ public OpenCGAResult searchOrganizationNote(Query query, QueryOptions opti queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; @@ -131,7 +131,7 @@ public OpenCGAResult searchStudyNote(String studyStr, Query query, QueryOp queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; @@ -209,11 +209,11 @@ public OpenCGAResult createStudyNote(String studyStr, NoteCreateParams not private OpenCGAResult create(Note note, QueryOptions options, JwtPayload tokenPayload) throws CatalogException { String organizationId = tokenPayload.getOrganization(); validateNewNote(note, tokenPayload.getUserId()); - OpenCGAResult insert = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).insert(note); + OpenCGAResult insert = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).insert(note); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created note Query query = new Query(NoteDBAdaptor.QueryParams.UID.key(), note.getUid()); - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(query, options); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } return insert; @@ -302,11 +302,11 @@ private OpenCGAResult update(long noteUid, NoteUpdateParams noteUpdatePara // Write who's performing the update updateMap.put(NoteDBAdaptor.QueryParams.USER_ID.key(), tokenPayload.getUserId()); - OpenCGAResult update = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, - QueryOptions.empty()); + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, + options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated note - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); update.setResults(result.getResults()); } return update; @@ -379,7 +379,7 @@ public OpenCGAResult deleteStudyNote(String studyStr, String noteId, Query } private OpenCGAResult delete(Note note, QueryOptions options, JwtPayload jwtPayload) throws CatalogException { - OpenCGAResult delete = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(jwtPayload.getOrganization()).delete(note); + OpenCGAResult delete = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(jwtPayload.getOrganization()).delete(note); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { delete.setResults(Collections.singletonList(note)); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java index 90b83a92728..20ec12a7f2b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java @@ -185,7 +185,7 @@ public OpenCGAResult create(OrganizationCreateParams organizationC organization = organizationCreateParams.toOrganization(); validateOrganizationForCreation(organization, userId); - queryResult = catalogDBAdaptorFactory.createOrganization(organization, options, userId); + queryResult = getCatalogDBAdaptorFactory().createOrganization(organization, options, userId); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { OpenCGAResult result = getOrganizationDBAdaptor(organization.getId()).get(options); organization = result.first(); @@ -211,7 +211,7 @@ public OpenCGAResult create(OrganizationCreateParams organizationC auditManager.auditCreate(ParamConstants.ADMIN_ORGANIZATION, userId, Enums.Resource.ORGANIZATION, organization.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); try { - catalogDBAdaptorFactory.deleteOrganization(organization); + getCatalogDBAdaptorFactory().deleteOrganization(organization); } catch (Exception e1) { logger.error("Error deleting organization from catalog after failing creating the folder in the filesystem", e1); throw e; @@ -327,9 +327,6 @@ public OpenCGAResult updateUser(@Nullable String organizationId, String us throw new CatalogException("Max CPU cannot be negative"); } } - if (updateParams.getAccount() != null && StringUtils.isNotEmpty(updateParams.getAccount().getExpirationDate())) { - ParamUtils.checkDateIsNotExpired(updateParams.getAccount().getExpirationDate(), "expirationDate"); - } ObjectMap updateMap; try { @@ -337,6 +334,12 @@ public OpenCGAResult updateUser(@Nullable String organizationId, String us } catch (JsonProcessingException e) { throw new CatalogException("Could not parse OrganizationUserUpdateParams object: " + e.getMessage(), e); } + + if (updateParams.getInternal() != null && updateParams.getInternal().getAccount() != null + && StringUtils.isNotEmpty(updateParams.getInternal().getAccount().getExpirationDate())) { + ParamUtils.checkDateIsNotExpired(updateParams.getInternal().getAccount().getExpirationDate(), "expirationDate"); + } + OpenCGAResult updateResult = getUserDBAdaptor(myOrganizationId).update(userId, updateMap); auditManager.auditUpdate(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, userId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); @@ -592,7 +595,7 @@ Set getOrganizationOwnerAndAdmins(String organizationId) throws CatalogE public List getOrganizationIds(String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); authorizationManager.checkIsOpencgaAdministrator(tokenPayload, "get all organization ids"); - return catalogDBAdaptorFactory.getOrganizationIds(); + return getCatalogDBAdaptorFactory().getOrganizationIds(); } private void privatizeResults(OpenCGAResult result) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index beda16bfa2e..92591852817 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -227,13 +227,13 @@ private void validateProjectForCreation(String organizationId, Project project) CellBaseConfiguration cellBaseConfiguration = ParamUtils.defaultObject(project.getCellbase(), new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, defaultDataRelease, ParamConstants.CELLBASE_APIKEY)); - cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, project.getOrganism().getScientificName(), + cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, + project.getOrganism().getScientificName(), project.getOrganism().getAssembly(), true); project.setCellbase(cellBaseConfiguration); } catch (IOException e) { throw new CatalogParameterException(e); } - project.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.PROJECT)); if (project.getStudies() != null && !project.getStudies().isEmpty()) { throw new CatalogParameterException("Creating project and studies in a single transaction is forbidden"); @@ -562,7 +562,7 @@ public void importReleases(String organizationId, String owner, String inputDirS } OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(owner, new QueryOptions(QueryOptions.INCLUDE, - Collections.singletonList(UserDBAdaptor.QueryParams.ACCOUNT.key()))); + Collections.singletonList(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT.key()))); if (userDataResult.getNumResults() == 0) { throw new CatalogException("User " + owner + " not found"); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 848cb8e5e2d..70f933f6430 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -1820,6 +1820,40 @@ public void setVariantEngineConfigurationOptions(String studyStr, ObjectMap opti getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); } + public void setVariantEngineSetupOptions(String studyStr, VariantSetupResult variantSetupResult, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), + StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key())), + tokenPayload); + + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + StudyVariantEngineConfiguration configuration = study.getInternal().getConfiguration().getVariantEngine(); + if (configuration == null) { + configuration = new StudyVariantEngineConfiguration(); + } + if (configuration.getOptions() == null) { + configuration.setOptions(new ObjectMap()); + } + VariantSetupResult prevSetupValue = configuration.getSetup(); + if (prevSetupValue != null && prevSetupValue.getOptions() != null) { + // Variant setup was already executed. + // Remove the options from the previous execution before adding the new ones + // Check that both key/value matches, to avoid removing options that might have been modified manually + for (Map.Entry entry : prevSetupValue.getOptions().entrySet()) { + configuration.getOptions().remove(entry.getKey(), entry.getValue()); + } + } + configuration.getOptions().putAll(variantSetupResult.getOptions()); + configuration.setSetup(variantSetupResult); + + ObjectMap parameters = new ObjectMap(StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key(), configuration); + getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); + } + public void setVariantEngineConfigurationSampleIndex(String studyStr, SampleIndexConfiguration sampleIndexConfiguration, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index 4c9f03d12e1..7e8e8cdffeb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -31,15 +31,18 @@ import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.CatalogIOManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupUpdateParams; @@ -60,8 +63,8 @@ */ public class UserManager extends AbstractManager { - static final QueryOptions INCLUDE_ACCOUNT_AND_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.ACCOUNT.key(), UserDBAdaptor.QueryParams.INTERNAL.key())); + static final QueryOptions INCLUDE_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.ID.key(), + UserDBAdaptor.QueryParams.INTERNAL.key(), UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key())); protected static Logger logger = LoggerFactory.getLogger(UserManager.class); private final CatalogIOManager catalogIOManager; private final AuthenticationFactory authenticationFactory; @@ -84,6 +87,9 @@ public void changePassword(String organizationId, String userId, String oldPassw if (oldPassword.equals(newPassword)) { throw new CatalogException("New password is the same as the old password."); } + if (!PasswordUtils.isStrongPassword(newPassword)) { + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); + } getUserDBAdaptor(organizationId).checkId(userId); String authOrigin = getAuthenticationOriginId(organizationId, userId); @@ -127,36 +133,54 @@ public OpenCGAResult create(User user, String password, String token) thro // Initialise fields ParamUtils.checkObj(user, "User"); ParamUtils.checkValidUserId(user.getId()); - ParamUtils.checkParameter(user.getName(), "name"); + user.setName(ParamUtils.defaultString(user.getName(), user.getId())); user.setEmail(ParamUtils.defaultString(user.getEmail(), "")); if (StringUtils.isNotEmpty(user.getEmail())) { checkEmail(user.getEmail()); } - user.setAccount(ParamUtils.defaultObject(user.getAccount(), Account::new)); - user.getAccount().setCreationDate(TimeUtils.getTime()); - if (StringUtils.isEmpty(user.getAccount().getExpirationDate())) { - // By default, user accounts will be valid for 1 year when they are created. - user.getAccount().setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); - Date date = TimeUtils.add1YeartoDate(new Date()); - user.getAccount().setExpirationDate(TimeUtils.getTime(date)); - } else { - // Validate expiration date is not over - ParamUtils.checkDateIsNotExpired(user.getAccount().getExpirationDate(), "account.expirationDate"); - } - user.setInternal(new UserInternal(new UserStatus(UserStatus.READY))); + user.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(user.getCreationDate(), + UserDBAdaptor.QueryParams.CREATION_DATE.key())); + user.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(user.getModificationDate(), + UserDBAdaptor.QueryParams.MODIFICATION_DATE.key())); + + user.setInternal(ParamUtils.defaultObject(user.getInternal(), UserInternal::new)); + user.getInternal().setStatus(new UserStatus(InternalStatus.READY)); user.setQuota(ParamUtils.defaultObject(user.getQuota(), UserQuota::new)); user.setProjects(ParamUtils.defaultObject(user.getProjects(), Collections::emptyList)); user.setConfigs(ParamUtils.defaultObject(user.getConfigs(), HashMap::new)); user.setFilters(ParamUtils.defaultObject(user.getFilters(), LinkedList::new)); user.setAttributes(ParamUtils.defaultObject(user.getAttributes(), Collections::emptyMap)); + // Init account + user.getInternal().setAccount(ParamUtils.defaultObject(user.getInternal().getAccount(), Account::new)); + Account account = user.getInternal().getAccount(); + account.setPassword(ParamUtils.defaultObject(account.getPassword(), Password::new)); + if (StringUtils.isEmpty(account.getExpirationDate())) { + account.setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); + } else { + // Validate expiration date is not over + ParamUtils.checkDateIsNotExpired(account.getExpirationDate(), UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key()); + } if (StringUtils.isEmpty(password)) { Map authOriginMap = authenticationFactory.getOrganizationAuthenticationManagers(organizationId); - if (!authOriginMap.containsKey(user.getAccount().getAuthentication().getId())) { - throw new CatalogException("Unknown authentication origin id '" + user.getAccount().getAuthentication() + "'"); + if (!authOriginMap.containsKey(account.getAuthentication().getId())) { + throw new CatalogException("Unknown authentication origin id '" + account.getAuthentication() + "'"); } } else { - user.getAccount().setAuthentication(new Account.AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, false)); + account.setAuthentication(new Account.AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, false)); + } + + // Set password expiration + if (AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId())) { + account.getPassword().setLastModified(TimeUtils.getTime()); + } + if (!AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId()) + || configuration.getAccount().getPasswordExpirationDays() <= 0) { + // User password doesn't expire or it's not managed by OpenCGA + account.getPassword().setExpirationDate(null); + } else { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + account.getPassword().setExpirationDate(TimeUtils.getTime(date)); } if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) || !OPENCGA.equals(user.getId())) { @@ -174,7 +198,7 @@ public OpenCGAResult create(User user, String password, String token) thro try { if (StringUtils.isNotEmpty(password) && !PasswordUtils.isStrongPassword(password)) { - throw new CatalogException("Invalid password. Check password strength for user " + user.getId()); + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); } if (user.getProjects() != null && !user.getProjects().isEmpty()) { throw new CatalogException("Creating user and projects in a single transaction is forbidden"); @@ -210,7 +234,6 @@ public OpenCGAResult create(User user, String password, String token) thro public OpenCGAResult create(String id, String name, String email, String password, String organization, Long quota, String token) throws CatalogException { User user = new User(id, name, email, organization, new UserInternal(new UserStatus())) - .setAccount(new Account("", "", null)) .setQuota(new UserQuota().setMaxDisk(quota != null ? quota : -1)); return create(user, password, token); } @@ -241,7 +264,8 @@ public OpenCGAResult search(@Nullable String organizationId, Query query, // Fix query params if (query.containsKey(ParamConstants.USER_AUTHENTICATION_ORIGIN)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_AUTHENTICATION_ID.key(), query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), + query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); query.remove(ParamConstants.USER_AUTHENTICATION_ORIGIN); } @@ -266,11 +290,12 @@ public JwtPayload validateToken(String token) throws CatalogException { authOrigin = CatalogAuthenticationManager.OPENCGA; } else { OpenCGAResult userResult = getUserDBAdaptor(jwtPayload.getOrganization()).get(jwtPayload.getUserId(), - INCLUDE_ACCOUNT_AND_INTERNAL); + INCLUDE_INTERNAL); if (userResult.getNumResults() == 0) { throw new CatalogException("User '" + jwtPayload.getUserId() + "' could not be found."); } - authOrigin = userResult.first().getAccount().getAuthentication().getId(); + User user = userResult.first(); + authOrigin = user.getInternal().getAccount().getAuthentication().getId(); } authenticationFactory.validateToken(jwtPayload.getOrganization(), authOrigin, token); @@ -475,8 +500,10 @@ public void importRemoteEntities(String organizationId, String authOrigin, List< } } else { for (String applicationId : idList) { - User application = new User(applicationId, new Account() - .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true))) + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true)); + User application = new User(applicationId) + .setInternal(new UserInternal(new UserStatus(UserStatus.READY), account)) .setEmail("mail@mail.co.uk"); application.setOrganization(organizationId); create(application, null, token); @@ -782,7 +809,7 @@ public AuthenticationResponse login(String organizationId, String username, Stri if (OPENCGA.equals(username)) { organizationId = ParamConstants.ADMIN_ORGANIZATION; } else { - List organizationIds = catalogDBAdaptorFactory.getOrganizationIds(); + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); if (organizationIds.size() == 2) { organizationId = organizationIds.stream().filter(s -> !ParamConstants.ADMIN_ORGANIZATION.equals(s)).findFirst().get(); } else { @@ -791,38 +818,56 @@ public AuthenticationResponse login(String organizationId, String username, Stri } } - OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_ACCOUNT_AND_INTERNAL); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { User user = userOpenCGAResult.first(); // Only local OPENCGA users that are not superadmins can be automatically banned or their accounts be expired boolean userCanBeBanned = !ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) - && CatalogAuthenticationManager.OPENCGA.equals(user.getAccount().getAuthentication().getId()); + && CatalogAuthenticationManager.OPENCGA.equals(user.getInternal().getAccount().getAuthentication().getId()); // We check if (userCanBeBanned) { // Check user is not banned, suspended or has an expired account if (UserStatus.BANNED.equals(user.getInternal().getStatus().getId())) { throw CatalogAuthenticationException.userIsBanned(username); } - Date date = TimeUtils.toDate(user.getAccount().getExpirationDate()); - if (date == null) { - throw new CatalogException("Unexpected null expiration date for user '" + username + "'."); + if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { + throw CatalogAuthenticationException.userIsSuspended(username); } - if (date.before(new Date())) { - throw CatalogAuthenticationException.accountIsExpired(username, user.getAccount().getExpirationDate()); + Account account2 = user.getInternal().getAccount(); + if (account2.getPassword().getExpirationDate() != null) { + Account account1 = user.getInternal().getAccount(); + Date passwordExpirationDate = TimeUtils.toDate(account1.getPassword().getExpirationDate()); + if (passwordExpirationDate == null) { + throw new CatalogException("Unexpected null 'passwordExpirationDate' for user '" + username + "'."); + } + if (passwordExpirationDate.before(new Date())) { + Account account = user.getInternal().getAccount(); + throw CatalogAuthenticationException.passwordExpired(username, account.getPassword().getExpirationDate()); + } + } + if (user.getInternal().getAccount().getExpirationDate() != null) { + Date date = TimeUtils.toDate(user.getInternal().getAccount().getExpirationDate()); + if (date == null) { + throw new CatalogException("Unexpected null 'expirationDate' for user '" + username + "'."); + } + if (date.before(new Date())) { + throw CatalogAuthenticationException.accountIsExpired(username, + user.getInternal().getAccount().getExpirationDate()); + } } } - if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { - throw CatalogAuthenticationException.userIsSuspended(username); - } - authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); + User user1 = userOpenCGAResult.first(); + authId = user1.getInternal().getAccount().getAuthentication().getId(); try { response = authenticationFactory.authenticate(organizationId, authId, username, password); } catch (CatalogAuthenticationException e) { if (userCanBeBanned) { // We can only lock the account if it is not the root user - int failedAttempts = userOpenCGAResult.first().getInternal().getFailedAttempts(); - ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), failedAttempts + 1); - if (failedAttempts >= (configuration.getMaxLoginAttempts() - 1)) { + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + int failedAttempts = userInternal.getAccount().getFailedAttempts(); + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), + failedAttempts + 1); + if (failedAttempts >= (configuration.getAccount().getMaxLoginAttempts() - 1)) { // Ban the account updateParams.append(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), UserStatus.BANNED); } @@ -835,10 +880,13 @@ public AuthenticationResponse login(String organizationId, String username, Stri } // If it was a local user and the counter of failed attempts was greater than 0, we reset it - if (userCanBeBanned && userOpenCGAResult.first().getInternal().getFailedAttempts() > 0) { - // Reset login failed attempts counter - ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); - getUserDBAdaptor(organizationId).update(username, updateParams); + if (userCanBeBanned) { + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + if (userInternal.getAccount().getFailedAttempts() > 0) { + // Reset login failed attempts counter + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); + getUserDBAdaptor(organizationId).update(username, updateParams); + } } } else { // We attempt to login the user with the different authentication managers @@ -989,7 +1037,7 @@ public OpenCGAResult changeStatus(String organizationId, String userId, St // Update user status and reset failed attempts to 0 ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), status); if (UserStatus.READY.equals(status)) { - updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); + updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); } OpenCGAResult result = getUserDBAdaptor(userIdOrganization).update(userId, updateParams); @@ -1056,9 +1104,10 @@ public String getNonExpiringToken(String organizationId, String userId, Map userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_ACCOUNT_AND_INTERNAL); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { - String authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); + User user1 = userOpenCGAResult.first(); + String authId = user1.getInternal().getAccount().getAuthentication().getId(); return authenticationFactory.getOrganizationAuthenticationManager(organizationId, authId); } else { throw new CatalogException("User '" + user + "' not found."); @@ -1489,7 +1538,8 @@ private String getAuthenticationOriginId(String organizationId, String userId) t if (user == null || user.getNumResults() == 0) { throw new CatalogException(userId + " user not found"); } - return user.first().getAccount().getAuthentication().getId(); + User user1 = user.first(); + return user1.getInternal().getAccount().getAuthentication().getId(); } /** @@ -1523,4 +1573,22 @@ private String getValidUserId(String userId, JwtPayload payload) throws CatalogE } } + public String getUserId(String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + return tokenPayload.getUserId(); + } + + public String getUserIdContextProject(String projectStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + return tokenPayload.getUserId(organizationId); + } + + public String getUserIdContextStudy(String studyStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + return tokenPayload.getUserId(organizationId); + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java index fa8fd7b9ee0..aacf4949226 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java @@ -668,7 +668,8 @@ private MigrationRun run(String organizationId, Class r + "-" + RandomStringUtils.randomAlphanumeric(5); String logFile = startMigrationLogger(jobId, Paths.get(configuration.getJobDir()).resolve(path)); logger.info("------------------------------------------------------"); - logger.info("Executing migration '{}' for version '{}'", annotation.id(), annotation.version()); + logger.info("Executing migration '{}' for version '{}' in organization '{}'", annotation.id(), annotation.version(), + organizationId); logger.info(" {}", annotation.description()); logger.info("------------------------------------------------------"); @@ -707,6 +708,7 @@ private MigrationRun run(String organizationId, Class r logger.info("Migration '{}' finished with status {} : {}", annotation.id(), status, TimeUtils.durationToString(stopWatch)); } logger.info("------------------------------------------------------"); + logger.info(""); } catch (Exception e) { migrationRun.setStatus(MigrationRun.MigrationStatus.ERROR); String message; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java new file mode 100644 index 00000000000..01d1d59d4a2 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.catalog.migration; + +public class MigrationRuntimeException extends IllegalArgumentException { + + public MigrationRuntimeException(String message) { + super(message); + } + + public MigrationRuntimeException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java index 219e1178652..1a57bb6a239 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java @@ -3,6 +3,8 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.IndexOptions; +import com.mongodb.client.model.InsertOneModel; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.WriteModel; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -174,6 +176,25 @@ protected final void migrateCollection(MongoCollection inputCollection } } + protected void copyData(Bson query, String sourceCol, String targetCol) throws CatalogDBException { + MongoCollection sourceMongoCollection = getMongoCollection(sourceCol); + MongoCollection targetMongoCollection = getMongoCollection(targetCol); + copyData(query, sourceMongoCollection, targetMongoCollection); + } + + protected void copyData(Bson query, MongoCollection sourceCol, MongoCollection targetCol) { + // Move data to the new collection + logger.info("Copying data from {} to {}", sourceCol.getNamespace(), targetCol.getNamespace()); + migrateCollection(sourceCol, targetCol, query, Projections.exclude("_id"), + (document, bulk) -> bulk.add(new InsertOneModel<>(document))); + } + + protected void moveData(Bson query, MongoCollection sourceCol, MongoCollection targetCol) { + copyData(query, sourceCol, targetCol); + // Remove data from the source collection + sourceCol.deleteMany(query); + } + protected final void createIndex(String collection, Document index) throws CatalogDBException { createIndex(getMongoCollection(collection), index, new IndexOptions().background(true)); } diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 557a660716a..fe7f43506d5 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -207,35 +207,36 @@ {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"uuid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"uid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"type": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"panelLock": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"type": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"panelLocked": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"id": 1, "version": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"clinicalAnalysisId": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"method.name": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java index 4cae5cd565b..8882260fabc 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java @@ -26,9 +26,8 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.IndividualPopulation; @@ -187,7 +186,7 @@ public void testModifyIndividualNegativeFatherId() throws Exception { } @Test - public void testAvoidDuplicatedSamples() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void testAvoidDuplicatedSamples() throws CatalogException { dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), Collections.emptyList(), QueryOptions.empty()); Sample sample1 = getSample(studyUid, "sample1"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java index 9c6d82a4c79..a382282764d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java @@ -60,7 +60,7 @@ private Job getNewJob(String id) { } @Test - public void createJobTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void createJobTest() throws CatalogException { Job job = getNewJob("jobName1"); System.out.println(catalogJobDBAdaptor.insert(studyUid, job, null)); @@ -120,7 +120,7 @@ public void getJobTest() throws CatalogException { } @Test - public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void testSortResultsPriorityAndCreationDate() throws CatalogException { Date startDate = TimeUtils.getDate(); // Create 100 jobs @@ -176,7 +176,7 @@ public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, // } @Test - public void getJobsOrderedByDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void getJobsOrderedByDate() throws CatalogException { // Job with current date Job job1 = getNewJob("job1"); @@ -250,7 +250,7 @@ public void updateInputAndOutputFiles() throws Exception { } @Test - public void groupByStatus() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void groupByStatus() throws CatalogException { for (int i = 0; i < 10; i++) { Enums.ExecutionStatus status = new Enums.ExecutionStatus(); if (i < 5) { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java index 48ee2b0cfc0..252085c353b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java @@ -19,18 +19,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; /** * Contains two methods mainly for testing purposes. One to create a dump of the current testing OpenCGA installation and a second one to @@ -38,6 +37,7 @@ */ public class MongoBackupUtils { + private static final String TEMPORAL_FOLDER_HERE = "TEMPORAL_FOLDER_HERE"; private static Logger logger = LoggerFactory.getLogger(MongoBackupUtils.class); public static void dump(CatalogManager catalogManager, Path opencgaHome) throws CatalogDBException { @@ -89,12 +89,12 @@ public static void dump(CatalogManager catalogManager, Path opencgaHome) throws } public static void restore(CatalogManager catalogManager, Path opencgaHome) - throws CatalogDBException, IOException, CatalogIOException, URISyntaxException { - StopWatch stopWatch = StopWatch.createStarted(); + throws Exception { try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory())) { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) .getMongoDataStore().getMongoClient(); + MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); Map databaseNames = new HashMap<>(); try (MongoCursor mongoIterator = dumpDatabase.getCollection("summary") @@ -107,25 +107,67 @@ public static void restore(CatalogManager catalogManager, Path opencgaHome) } } - List organizationIds = dbAdaptorFactory.getOrganizationIds(); - for (String organizationId : organizationIds) { + restore(catalogManager, opencgaHome, dumpDatabase, databaseNames.keySet()); + } + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome, URL restoreFolder) + throws Exception { + /* + dataset=v3.0.0 + mongo test_dump --eval 'db.getCollectionNames()' --quiet | jq .[] -r | grep -v summary | while read i ; do + org=$(echo $i | cut -d "_" -f 1) ; + collection=$(echo $i | cut -d "_" -f 3) ; + mkdir -p opencga-app/src/test/resources/datasets/catalog/$dataset/mongodb/$org ; + mongo test_dump --quiet --eval "db.getCollection(\"$i\").find().forEach(function(d){ print(tojsononeline(d)); })" | gzip > opencga-app/src/test/resources/datasets/opencga/$dataset/mongodb/$org/$collection.json.gz ; + done + */ + if (restoreFolder.getProtocol().equals("file")) { + restore(catalogManager, opencgaHome, Paths.get(restoreFolder.toURI())); + } else if (restoreFolder.getProtocol().equals("jar")) { + throw new UnsupportedOperationException("Cannot restore from a jar file"); + } + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome, Path restoreFolder) + throws Exception { + + List organizationIds = new ArrayList<>(); + try (Stream stream = Files.list(restoreFolder)) { + stream.forEach(file -> organizationIds.add(file.getFileName().toString())); + } + if (organizationIds.isEmpty()) { + throw new CatalogDBException("No organization found in the restore folder '" + restoreFolder + "'"); + } + restore(catalogManager, opencgaHome, restoreFolder, organizationIds); + } + + private static void restore(CatalogManager catalogManager, Path opencgaHome, Object source, Collection organizationIds) + throws Exception { + logger.info("Restore opencga from source " + source + " for organizations " + organizationIds); + StopWatch stopWatch = StopWatch.createStarted(); + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), + catalogManager.getIoManagerFactory())) { + + for (String existingOrganizationId : dbAdaptorFactory.getOrganizationIds()) { // We need to completely remove databases that were not backed up so tests that attempt to create them again don't fail - if (!databaseNames.containsKey(organizationId)) { - logger.info("Completely removing database for organization '{}'", organizationId); - dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(organizationId).deleteCatalogDB(); + if (!organizationIds.contains(existingOrganizationId)) { + logger.info("Completely removing database for organization '{}'", existingOrganizationId); + dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(existingOrganizationId).deleteCatalogDB(); } } // First restore the main admin database String adminDBName = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) .getMongoDataStore().getDatabaseName(); - restoreDatabase(catalogManager, opencgaHome, ParamConstants.ADMIN_ORGANIZATION, adminDBName, dbAdaptorFactory); + logger.info("Restoring database for organization '{}'", ParamConstants.ADMIN_ORGANIZATION); + restoreDatabase(catalogManager, opencgaHome, ParamConstants.ADMIN_ORGANIZATION, adminDBName, dbAdaptorFactory, source); - for (Map.Entry entry : databaseNames.entrySet()) { - String organizationId = entry.getKey(); + for (String organizationId : organizationIds) { if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { - String databaseName = entry.getValue(); - restoreDatabase(catalogManager, opencgaHome, organizationId, databaseName, dbAdaptorFactory); + logger.info("Restoring database for organization '{}'", organizationId); + String databaseName = catalogManager.getCatalogDatabase(organizationId); + restoreDatabase(catalogManager, opencgaHome, organizationId, databaseName, dbAdaptorFactory, source); } } } @@ -133,10 +175,9 @@ public static void restore(CatalogManager catalogManager, Path opencgaHome) } private static void restoreDatabase(CatalogManager catalogManager, Path opencgaHome, String organizationId, String databaseName, - MongoDBAdaptorFactory dbAdaptorFactory) - throws IOException, CatalogIOException, CatalogDBException, URISyntaxException { + MongoDBAdaptorFactory dbAdaptorFactory, Object source) + throws Exception { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION).getMongoDataStore().getMongoClient(); - MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); Bson emptyBsonQuery = new Document(); MongoDatabase database = mongoClient.getDatabase(databaseName); @@ -152,10 +193,9 @@ private static void restoreDatabase(CatalogManager catalogManager, Path opencgaH for (String collection : OrganizationMongoDBAdaptorFactory.COLLECTIONS_LIST) { MongoCollection dbCollection = database.getCollection(collection); - MongoCollection dumpCollection = dumpDatabase.getCollection(organizationId + "__" + collection); dbCollection.deleteMany(emptyBsonQuery); - try (MongoCursor iterator = dumpCollection.find(emptyBsonQuery).noCursorTimeout(true).iterator()) { + try (CloseableIterator iterator = documentIterator(source, organizationId, collection)) { List documentList = new LinkedList<>(); while (iterator.hasNext()) { Document document = iterator.next(); @@ -171,8 +211,8 @@ private static void restoreDatabase(CatalogManager catalogManager, Path opencgaH // Write actual temporal folder in database String uri = document.getString("uri"); - String temporalFolder = opencgaHome.getFileName().toString(); - String replacedUri = uri.replace("TEMPORAL_FOLDER_HERE", temporalFolder); + String uriPath = uri.substring(uri.indexOf(TEMPORAL_FOLDER_HERE) + TEMPORAL_FOLDER_HERE.length() + 1); + String replacedUri = opencgaHome.resolve(uriPath).toUri().toString(); document.put("uri", replacedUri); if (OrganizationMongoDBAdaptorFactory.FILE_COLLECTION.equals(collection)) { @@ -198,6 +238,72 @@ private static void restoreDatabase(CatalogManager catalogManager, Path opencgaH } } + private static CloseableIterator documentIterator(Object source, String organizationId, String collection) + throws IOException { + if (source instanceof MongoDatabase) { + MongoDatabase dumpDatabase = (MongoDatabase) source; + + MongoCollection dumpCollection = dumpDatabase.getCollection(organizationId + "__" + collection); + logger.info("Restoring {}:{} from database - {}", organizationId, collection, dumpCollection.getNamespace().getFullName()); + return new CloseableIterator<>(dumpCollection.find(new Document()).noCursorTimeout(true).iterator()); + } else if (source instanceof Path) { + Path dir = (Path) source; + java.io.File file = dir.resolve(organizationId).resolve(collection + ".json.gz").toFile(); + if (!file.exists()) { +// logger.info("File {} not found", file); + return new CloseableIterator<>(null, Collections.emptyIterator()); + } + logger.info("Restoring {}:{} from file - {}", organizationId, collection, file); + // Read file lines + Stream stream = new BufferedReader( + new InputStreamReader( + new GZIPInputStream( + Files.newInputStream(file.toPath())))).lines(); + + Iterator iterator = stream + .filter(s -> !s.isEmpty()) +// .peek(System.out::println) + .map(Document::parse) + .iterator(); + return new CloseableIterator<>(stream, iterator); + } else { + throw new IllegalArgumentException("Unknown restore source type " + source.getClass()); + } + } + + private static class CloseableIterator implements Iterator, AutoCloseable { + + private final AutoCloseable closeable; + private final Iterator iterator; + + public CloseableIterator(AutoCloseable closeable, Iterator iterator) { + this.closeable = closeable; + this.iterator = iterator; + } + + public & Closeable> CloseableIterator(CI c) { + this.closeable = c; + this.iterator = c; + } + + @Override + public void close() throws Exception { + if (closeable != null) { + closeable.close(); + } + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public T next() { + return iterator.next(); + } + } + private static void createFile(IOManager ioManager, Document document) throws IOException, CatalogIOException { String type = document.getString("type"); if (File.Type.FILE.name().equals(type)) { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java index 5488bc6a545..fa908630690 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java @@ -27,10 +27,8 @@ import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.SampleManager; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.cohort.Cohort; @@ -175,7 +173,7 @@ public class SampleMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { // } @Test - public void searchByOntology() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void searchByOntology() throws CatalogException { List ontologyList = Arrays.asList( new Phenotype("hpo:123", "One hpo term", "hpo", Phenotype.Status.UNKNOWN), new Phenotype("hpo:456", "Another hpo term", "hpo", Phenotype.Status.UNKNOWN), @@ -387,7 +385,7 @@ public void caseInsensitiveSearchTest() throws Exception { // Test if we can search for samples of an individual @Test - public void getSampleWithIndividual() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void getSampleWithIndividual() throws CatalogException { QueryOptions queryOptions = new QueryOptions(); // We create a new sample with the individual diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index 697a77d894f..f81f28b37a6 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -20,10 +20,8 @@ import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -112,7 +110,7 @@ public void createVariableSetTest() throws CatalogDBException { } @Test - public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void testRemoveFieldFromVariableSet() throws CatalogException { DataResult variableSetDataResult = createExampleVariableSet("VARSET_1", false); DataResult result = catalogStudyDBAdaptor.removeFieldFromVariableSet(5L, variableSetDataResult.first().getUid(), "NAME", orgAdminUserId1); @@ -165,7 +163,7 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void addFieldToVariableSetTest1() throws CatalogException { DataResult varset1 = createExampleVariableSet("VARSET_1", false); createExampleVariableSet("VARSET_2", true); Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, @@ -191,7 +189,7 @@ public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAutho * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest2() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void addFieldToVariableSetTest2() throws CatalogException { Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap()); thrown.expect(CatalogDBException.class); @@ -208,7 +206,7 @@ public void createGroup() throws CatalogDBException { } @Test - public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void removeUsersFromAllGroups() throws CatalogException { catalogStudyDBAdaptor.createGroup(studyUid, new Group("name1", Arrays.asList(normalUserId1, normalUserId2))); catalogStudyDBAdaptor.createGroup(studyUid, new Group("name2", Arrays.asList(normalUserId1, normalUserId2, normalUserId3))); catalogStudyDBAdaptor.createGroup(studyUid, new Group("name3", Arrays.asList(normalUserId1, normalUserId3))); @@ -221,7 +219,7 @@ public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParamet } @Test - public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void resyncUserWithSyncedGroups() throws CatalogException { // We create synced groups and not synced groups in study studyUid Group group = new Group("@notSyncedGroup", Arrays.asList(normalUserId1, normalUserId2, normalUserId3)); catalogStudyDBAdaptor.createGroup(studyUid, group); @@ -293,7 +291,7 @@ public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParam } @Test - public void updateUserToGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void updateUserToGroups() throws CatalogException { // We create synced groups and not synced groups in study studyUid Group group = new Group("@notSyncedGroup", Collections.emptyList()); catalogStudyDBAdaptor.createGroup(studyUid, group); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java index a007c100b0f..8e4571d86ea 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java @@ -100,7 +100,7 @@ public void getUserTest() throws CatalogDBException, CatalogParameterException, } @Test - public void changePasswordTest() throws CatalogDBException, CatalogAuthenticationException { + public void changePasswordTest() throws CatalogException { DataResult result = catalogUserDBAdaptor.changePassword(normalUserId1, TestParamConstants.PASSWORD, "1234"); assertEquals(1, result.getNumUpdated()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java index 9044dd2f7c0..dc857e15965 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java @@ -23,13 +23,17 @@ import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; import org.junit.rules.TestName; +import org.mockito.Mockito; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -50,8 +54,6 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -182,7 +184,7 @@ public void setUp() throws Exception { setUpCatalogManager(catalogManager); } - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException, URISyntaxException { + public void setUpCatalogManager(CatalogManager catalogManager) throws Exception { if (!firstExecutionFinished) { createDummyData(catalogManager); MongoBackupUtils.dump(catalogManager, catalogManagerResource.getOpencgaHome()); @@ -474,4 +476,23 @@ private void createDummyData(CatalogManager catalogManager) throws CatalogExcept } + protected CatalogManager mockCatalogManager() throws CatalogDBException { + CatalogManager spy = Mockito.spy(catalogManager); + UserManager userManager = spy.getUserManager(); + UserManager userManagerSpy = Mockito.spy(userManager); + Mockito.doReturn(userManagerSpy).when(spy).getUserManager(); + MongoDBAdaptorFactory mongoDBAdaptorFactory = mockMongoDBAdaptorFactory(); + Mockito.doReturn(mongoDBAdaptorFactory).when(userManagerSpy).getCatalogDBAdaptorFactory(); + return spy; + } + + protected MongoDBAdaptorFactory mockMongoDBAdaptorFactory() throws CatalogDBException { + MongoDBAdaptorFactory catalogDBAdaptorFactory = (MongoDBAdaptorFactory) catalogManager.getUserManager().getCatalogDBAdaptorFactory(); + MongoDBAdaptorFactory dbAdaptorFactorySpy = Mockito.spy(catalogDBAdaptorFactory); + UserDBAdaptor userDBAdaptor = dbAdaptorFactorySpy.getCatalogUserDBAdaptor(organizationId); + UserDBAdaptor userDBAdaptorSpy = Mockito.spy(userDBAdaptor); + Mockito.doReturn(userDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogUserDBAdaptor(organizationId); + return dbAdaptorFactorySpy; + } + } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java index e8259db2c4b..09c94ee33d3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java @@ -124,6 +124,7 @@ public CatalogManager getCatalogManager() { public CatalogManager resetCatalogManager() throws CatalogException { catalogManager.close(); catalogManager = new CatalogManager(configuration); + adminToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); return catalogManager; } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java index 73cab79045b..8f0bce1401e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java @@ -16,13 +16,10 @@ package org.opencb.opencga.catalog.managers; -import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.common.Status; @@ -34,7 +31,10 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -43,7 +43,6 @@ import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; -import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; import org.opencb.opencga.core.models.common.AnnotationSet; @@ -61,11 +60,10 @@ import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.study.*; -import org.opencb.opencga.core.models.user.*; +import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import javax.naming.NamingException; import java.io.IOException; import java.util.*; import java.util.concurrent.ExecutorService; @@ -73,22 +71,12 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; -import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; @Category(MediumTests.class) public class CatalogManagerTest extends AbstractManagerTest { - @Test - public void createOpencgaUserTest() throws CatalogException { - thrown.expect(CatalogException.class); - thrown.expectMessage("forbidden"); - catalogManager.getUserManager().create(new User().setId(ParamConstants.OPENCGA_USER_ID).setName(orgOwnerUserId) - .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - } - @Test public void createStudyFailMoreThanOneProject() throws CatalogException { catalogManager.getProjectManager().incrementRelease(project1, ownerToken); @@ -102,443 +90,6 @@ public void createStudyFailMoreThanOneProject() throws CatalogException { null, null, null, null, ownerToken); } - @Test - public void testAdminUserExists() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - JwtPayload payload = catalogManager.getUserManager().validateToken(token); - assertEquals(ParamConstants.OPENCGA_USER_ID, payload.getUserId()); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, payload.getOrganization()); - } - - @Test - public void searchUsersTest() throws CatalogException { - OpenCGAResult search = catalogManager.getUserManager().search(organizationId, new Query(), QueryOptions.empty(), opencgaToken); - assertEquals(8, search.getNumResults()); - for (User user : search.getResults()) { - if (noAccessUserId1.equals(user.getId())) { - assertEquals(0, user.getProjects().size()); - } else if (user.getId().startsWith("normalUser")) { - assertEquals(1, user.getProjects().size()); - } else { - assertEquals(2, user.getProjects().size()); - } - } - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), ownerToken); - assertEquals(8, search.getNumResults()); - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken2); - assertEquals(8, search.getNumResults()); - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken1); - assertEquals(8, search.getNumResults()); - - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), - QueryOptions.empty(), studyAdminToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), - QueryOptions.empty(), normalToken1)); - } - - @Test - public void testGetToken() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - Map claims = new HashMap<>(); - claims.put("a", "hola"); - claims.put("ab", "byw"); - // Create a token valid for 1 second - String expiringToken = catalogManager.getUserManager().getToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, 1L, token); - assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); - - String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, token); - assertEquals("opencga", catalogManager.getUserManager().validateToken(nonExpiringToken).getUserId()); - - Thread.sleep(1000); - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage("expired"); - assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); - } - - @Test - public void loginWithoutOrganizationId() throws CatalogException { - String token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - JwtPayload jwtPayload = new JwtPayload(token); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); - - token = catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - jwtPayload = new JwtPayload(token); - assertEquals(organizationId, jwtPayload.getOrganization()); - - // Create a third organization - catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("other").setName("Test"), QueryOptions.empty(), opencgaToken); - token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - jwtPayload = new JwtPayload(token); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); - - thrown.expect(CatalogParameterException.class); - thrown.expectMessage("organization"); - catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD); - } - - @Test - public void testCreateExistingUser() throws Exception { - thrown.expect(CatalogException.class); - thrown.expectMessage(containsString("already exists")); - catalogManager.getUserManager().create(orgOwnerUserId, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, - null, opencgaToken); - } - - @Test - public void testCreateAnonymousUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, - organizationId, null, opencgaToken); - } - - @Test - public void testCreateRegisteredUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, - opencgaToken); - } - - @Test - public void testLogin() throws Exception { - catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD); - - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); - catalogManager.getUserManager().login(organizationId, normalUserId1, "fakePassword"); - } - - @Test - public void refreshTokenTest() throws Exception { - String refreshToken = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getRefreshToken(); - AuthenticationResponse authenticationResponse = catalogManager.getUserManager().refreshToken(refreshToken); - assertNotNull(authenticationResponse); - assertNotNull(authenticationResponse.getToken()); - } - - @Test - public void anonymousUserLoginTest() throws CatalogException { - AuthenticationResponse authResponse = catalogManager.getUserManager().loginAnonymous(organizationId); - assertNotNull(authResponse.getToken()); - - String org2 = "otherOrg"; - catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); - catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); - String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); - Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() - .setId("project") - .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), - INCLUDE_RESULT, owner2Token).first(); - Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); - - try { - catalogManager.getUserManager().loginAnonymous(org2); - fail("Anonymous user should not get a token for that organization as it has not been granted any kind of access"); - } catch (Exception e) { - assertEquals(CatalogAuthenticationException.class, e.getClass()); - assertTrue(e.getMessage().contains("not found")); - } - - catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), owner2Token); - authResponse = catalogManager.getUserManager().loginAnonymous(org2); - assertNotNull(authResponse.getToken()); - - - catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Collections.singletonList("*")), owner2Token); - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().loginAnonymous(org2); - } - - @Test - public void incrementLoginAttemptsTest() throws CatalogException { - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(1, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - - for (int i = 2; i < 5; i++) { - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(i, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - } - - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(5, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); - - CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - assertTrue(incorrect.getMessage().contains("banned")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(5, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); - - CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); - assertTrue(authException.getMessage().contains("banned")); - - // Remove ban from user - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(0, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - - String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); - assertNotNull(token); - } - - @Test - public void changeUserStatusTest() throws CatalogException { - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); - assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); - - catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); - CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); - assertTrue(authException.getMessage().contains("own account")); - - authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); - assertTrue(authException.getMessage().contains("suspend administrators")); - - CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); - assertTrue(incorrect.getMessage().contains("suspended")); - - catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); - String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); - assertNotNull(token); - - CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); - assertTrue(paramException.getMessage().contains("Invalid status")); - - CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); - assertTrue(dbException.getMessage().contains("not exist")); - } - - @Test - public void loginExpiredAccountTest() throws CatalogException { - // Expire account of normalUserId1 - ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); - catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); - - CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); - assertTrue(authException.getMessage().contains("expired")); - - // Ensure it doesn't matter whether opencga account is expired or not - catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); - String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertNotNull(token); - } - - @Test - public void updateUserTest() throws JsonProcessingException, CatalogException { - UserUpdateParams userUpdateParams = new UserUpdateParams() - .setName("newName") - .setEmail("mail@mail.com"); - ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); - User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); - assertEquals(userUpdateParams.getName(), user.getName()); - assertEquals(userUpdateParams.getEmail(), user.getEmail()); - - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); - - userUpdateParams = new UserUpdateParams() - .setEmail("notAnEmail"); - ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); - assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); - } - - @Test - public void testGetUserInfo() throws CatalogException { - // OpenCGA administrator - DataResult user = catalogManager.getUserManager().get(organizationId, - Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), opencgaToken); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - // Organization owner - user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), - new QueryOptions(), ownerToken); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - // Organization administrator - user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), - new QueryOptions(), orgAdminToken1); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - thrown.expect(CatalogAuthorizationException.class); - thrown.expectMessage("organization"); - catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), - studyAdminToken1); - } - - @Test - public void testGetProjectsFromUserInfo() throws CatalogException { - String userId = organizationId; - catalogManager.getUserManager().create(userId, "test", "mail@mail.com", TestParamConstants.PASSWORD, organizationId, null, - opencgaToken); - catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("test")), ownerToken); - String token = catalogManager.getUserManager().login(organizationId, userId, TestParamConstants.PASSWORD).getToken(); - - DataResult user = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(), token); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, normalUserId3, new QueryOptions(), normalToken3); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(), ownerToken); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, new QueryOptions(), studyAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - - user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), normalToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, normalUserId3, new QueryOptions(), normalToken3); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, orgOwnerUserId, new QueryOptions(), ownerToken); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, orgAdminUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, studyAdminUserId1, new QueryOptions(), studyAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - } - - @Test - public void testModifyUser() throws CatalogException, InterruptedException, IOException { - ObjectMap params = new ObjectMap(); - String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); - String newPassword = RandomStringUtils.randomAlphanumeric(10); - String newEmail = "new@email.ac.uk"; - - params.put("name", newName); - - Thread.sleep(10); - - catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); - catalogManager.getUserManager().update(orgOwnerUserId, new ObjectMap("email", newEmail), null, ownerToken); - catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD, newPassword); - - List userList = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(QueryOptions - .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), - UserDBAdaptor.QueryParams.ATTRIBUTES.key())), ownerToken).getResults(); - User userPost = userList.get(0); - System.out.println("userPost = " + userPost); - assertEquals(userPost.getName(), newName); - assertEquals(userPost.getEmail(), newEmail); - - catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); - catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD); - catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD); - - try { - params = new ObjectMap(); - params.put("password", "1234321"); - catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); - fail("Expected exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - - try { - catalogManager.getUserManager().update(orgOwnerUserId, params, null, orgAdminToken1); - fail("Expected exception"); - } catch (CatalogException e) { - System.out.println(e); - } - } - - @Test - public void testUpdateUserConfig() throws CatalogException { - Map map = new HashMap<>(); - map.put("key1", "value1"); - map.put("key2", "value2"); - catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); - - Map config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); - assertEquals(2, config.size()); - assertEquals("value1", config.get("key1")); - assertEquals("value2", config.get("key2")); - - map = new HashMap<>(); - map.put("key2", "value3"); - catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); - config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); - assertEquals(1, config.size()); - assertEquals("value3", config.get("key2")); - - catalogManager.getUserManager().deleteConfig(normalUserId1, "a", normalToken1); - - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1); - } - - private String getAdminToken() throws CatalogException, IOException { - return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - } - - @Test - public void createUserUsingMailAsId() throws CatalogException { - catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); - AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); - assertNotNull(login); - User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); - assertEquals("hello.mail@mymail.org", user.getId()); - } - @Test public void getGroupsTest() throws CatalogException { Group group = new Group("groupId", Arrays.asList(normalUserId2, normalUserId3)).setSyncedFrom(new Group.Sync("ldap", "bio")); @@ -564,82 +115,6 @@ public void getGroupsTest() throws CatalogException { catalogManager.getStudyManager().getCustomGroups(studyFqn, group.getId(), normalToken2); } - @Ignore - @Test - public void importLdapUsers() throws CatalogException, NamingException, IOException { - // Action only for admins - catalogManager.getUserManager().importRemoteEntities(organizationId, "ldap", Arrays.asList("pfurio", "imedina"), false, null, null, - getAdminToken()); - // TODO: Validate the users have been imported - } - - // To make this test work we will need to add a correct user and password to be able to login - @Ignore - @Test - public void loginNotRegisteredUsers() throws CatalogException { - // Action only for admins - Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); - catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); - catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", "view_only"), - ParamUtils.AclAction.SET, ownerToken); - String token = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, "password").getToken(); - - assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - - // We remove the permissions for group ldap - catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", ""), - ParamUtils.AclAction.RESET, this.ownerToken); - - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - } - - @Ignore - @Test - public void syncUsers() throws CatalogException { - // Action only for admins - String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", "bio", "bio", studyFqn, true, token); - DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); - - assertEquals(1, bio.getNumResults()); - assertEquals(0, bio.first().getUserIds().size()); - - catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, studyFqn, "ldap", token); - bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); - - assertEquals(1, bio.getNumResults()); - assertTrue(!bio.first().getUserIds().isEmpty()); - } - - @Ignore - @Test - public void importLdapGroups() throws CatalogException, IOException { - // Action only for admins - String remoteGroup = "bio"; - String internalGroup = "test"; - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); - - DataResult test = catalogManager.getStudyManager().getGroup(studyFqn, "test", ownerToken); - assertEquals(1, test.getNumResults()); - assertEquals("@test", test.first().getId()); - assertTrue(test.first().getUserIds().size() > 0); - -// internalGroup = "test1"; -// try { -// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); -// fail("Should not be possible creating another group containing the same users that belong to a different group"); -// } catch (CatalogException e) { -// System.out.println(e.getMessage()); -// } - - remoteGroup = "bioo"; - internalGroup = "test2"; - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); - } - @Test public void createEmptyGroup() throws CatalogException { catalogManager.getUserManager().create("test", "test", "test@mail.com", TestParamConstants.PASSWORD, organizationId, 100L, opencgaToken); @@ -681,29 +156,6 @@ public void testAssignPermissions() throws CatalogException { assertTrue(acls.stream().map(x -> String.valueOf(x.get("member"))).collect(Collectors.toSet()).contains("@group_cancer_some_thing_else")); } - @Test - public void getUserInfoTest() throws CatalogException { - OpenCGAResult result = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, QueryOptions.empty(), ownerToken); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, QueryOptions.empty(), orgAdminToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, QueryOptions.empty(), studyAdminToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), normalToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(1, result.first().getProjects().size()); - } - /** * Project methods *************************** */ diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index cf267f04b71..6d42dbc6fbd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -25,7 +25,6 @@ import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariantEvidence; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; -import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.variant.avro.VariantAvro; import org.opencb.biodata.models.variant.avro.VariantType; @@ -36,6 +35,7 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult; import org.opencb.opencga.catalog.utils.Constants; @@ -61,9 +61,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.sample.SampleUpdateParams; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; +import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.models.study.configuration.ClinicalConsent; import org.opencb.opencga.core.models.study.configuration.*; import org.opencb.opencga.core.models.user.Account; @@ -159,7 +157,7 @@ private DataResult createDummyFamily() throws CatalogException { private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new Status().setId(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId("READY_FOR_INTERPRETATION")) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); @@ -205,25 +203,21 @@ public void createAndTestStatusIdIsNotNull() throws CatalogException { OpenCGAResult result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, false, INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, result.first().getStatus().getType()); clinicalAnalysis = new ClinicalAnalysis() .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) - .setStatus(new Status(null, null, null, null)) + .setStatus(new ClinicalStatus()) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))) .setFamily(new Family().setId("family") .setMembers(Collections.singletonList(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))))); result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, false, INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, result.first().getStatus().getType()); - result = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(null)), INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); - - result = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("")), INCLUDE_RESULT, ownerToken); - assertEquals("", result.first().getStatus().getId()); + String clinicalId = clinicalAnalysis.getId(); + assertThrows(CatalogException.class, () -> catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalId, + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(null)), INCLUDE_RESULT, ownerToken)); } @Test @@ -259,8 +253,8 @@ public void createMultipleCasesSameFamily() throws CatalogException { public void updateClinicalAnalystsTest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); // Add analysts OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), @@ -312,7 +306,7 @@ public void updateClinicalAnalysisRequest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); assertTrue(StringUtils.isEmpty(case1.getRequest().getId())); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setEmail("mail@mail.com").setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setEmail("mail@mail.com"), TestParamConstants.PASSWORD, opencgaToken); ClinicalRequest request = new ClinicalRequest("requestId", "bla", null, new ClinicalResponsible().setId("u1"), new HashMap<>()); @@ -350,8 +344,8 @@ public void updateClinicalAnalysisResponsible() throws CatalogException { assertEquals(orgOwnerUserId, case1.getResponsible().getId()); catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setEmail("mail@mail.com") - .setOrganization(organizationId) - .setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); + .setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); ClinicalResponsible responsible = new ClinicalResponsible().setId("u1"); @@ -592,6 +586,32 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { } @Test + public void queryByVersionTest() throws CatalogException { + Individual individual = new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2"))); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); + + ClinicalComment comment = new ClinicalComment(orgOwnerUserId, "my comment", new ArrayList<>(), ""); + ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() + .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) + .setDescription("My description").setType(ClinicalAnalysis.Type.SINGLE) + .setQualityControl(new ClinicalAnalysisQualityControl(ClinicalAnalysisQualityControl.QualityControlSummary.LOW, + Collections.singletonList(comment), Collections.emptyList())) + .setProband(individual); + + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, true, INCLUDE_RESULT, ownerToken).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams().setDescription("blabla"), null, ownerToken); + + ClinicalAnalysis case1 = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), null, ownerToken).first(); + assertEquals(2, case1.getVersion()); + assertEquals("blabla", case1.getDescription()); + + Query query = new Query(ParamConstants.CLINICAL_VERSION_PARAM, 1); + case1 = catalogManager.getClinicalAnalysisManager().get(studyFqn, Collections.singletonList(clinicalAnalysis.getId()), query, null, false, ownerToken).first(); + assertEquals(1, case1.getVersion()); + assertEquals(clinicalAnalysis.getDescription(), case1.getDescription()); + } + + @Test public void createAndUpdateClinicalAnalysisWithQualityControl() throws CatalogException, InterruptedException { Individual individual = new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2"))); catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); @@ -644,7 +664,7 @@ public void automaticallyLockCaseTest() throws CatalogException { .setProband(individual); OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); - assertTrue(StringUtils.isEmpty(clinical.first().getStatus().getId())); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, clinical.first().getStatus().getType()); assertFalse(clinical.first().isLocked()); clinical = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), @@ -655,10 +675,11 @@ public void automaticallyLockCaseTest() throws CatalogException { clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical2") .setType(ClinicalAnalysis.Type.SINGLE) - .setStatus(new ClinicalAnalysisStatus().setId("CLOSED")) + .setStatus(new ClinicalStatus().setId("CLOSED")) .setProband(individual); clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals("CLOSED", clinical.first().getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, clinical.first().getStatus().getType()); assertTrue(clinical.first().isLocked()); } @@ -932,6 +953,57 @@ public void updateClinicalAnalysis() throws CatalogException { assertEquals(attributes.get("b"), clinical.getAttributes().get("b")); } + @Test + public void versioningTest() throws CatalogException { + Individual individual = new Individual() + .setId("proband") + .setSamples(Collections.singletonList(new Sample().setId("sample"))); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); + + List findingList = new ArrayList<>(); + VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); + ClinicalVariantEvidence evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); + ClinicalVariant cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + findingList.add(cv1); + variantAvro = new VariantAvro("id2", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); + ClinicalVariant cv2 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + findingList.add(cv2); + + ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() + .setId("Clinical") + .setType(ClinicalAnalysis.Type.SINGLE) + .setProband(individual) + .setInterpretation(new Interpretation() + .setPrimaryFindings(findingList) + ); + ClinicalAnalysis clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken).first(); + assertEquals(1, clinical.getVersion()); + assertEquals(1, clinical.getInterpretation().getVersion()); + + // Update clinical analysis + clinical = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setDescription("my new description"), INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new description", clinical.getDescription()); + assertEquals(2, clinical.getVersion()); + assertEquals(1, clinical.getInterpretation().getVersion()); + + // Update interpretation + Interpretation interpretation = catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), + clinical.getInterpretation().getId(), new InterpretationUpdateParams().setDescription("my new interpretation description"), + null, INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new interpretation description", interpretation.getDescription()); + assertEquals(2, interpretation.getVersion()); + + // Get clinical Analysis + clinical = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("my new description", clinical.getDescription()); + assertEquals(3, clinical.getVersion()); + assertEquals("my new interpretation description", clinical.getInterpretation().getDescription()); + assertEquals(2, clinical.getInterpretation().getVersion()); + } + @Test public void createRepeatedInterpretationPrimaryFindings() throws CatalogException { Individual individual = new Individual() @@ -1323,6 +1395,41 @@ public void updateSecondaryFindings() throws CatalogException { } } + @Test + public void updateStatusTest() throws CatalogException { + ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); + + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + + // Create 2 allowed statuses of type CLOSED + ClinicalAnalysisStudyConfiguration studyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + List statusValueList = new ArrayList<>(); + for (ClinicalStatusValue status : studyConfiguration.getStatus()) { + if (!status.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + statusValueList.add(status); + } + } + // Add two statuses of type CLOSED + statusValueList.add(new ClinicalStatusValue("closed1", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + statusValueList.add(new ClinicalStatusValue("closed2", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + studyConfiguration.setStatus(statusValueList); + catalogManager.getClinicalAnalysisManager().configureStudy(studyFqn, studyConfiguration, studyAdminToken1); + + // Update status to one of the new statuses + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("closed1")), QueryOptions.empty(), studyAdminToken1); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), studyAdminToken1).first(); + assertEquals("closed1", ca.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, ca.getStatus().getType()); + + // Update status to the other new CLOSED status + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("closed2")), QueryOptions.empty(), studyAdminToken1); + assertEquals("closed1", ca.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, ca.getStatus().getType()); + } + @Test public void updateInterpretationComments() throws CatalogException { Individual individual = new Individual() @@ -1593,6 +1700,79 @@ public void updateClinicalAnalysisTest() throws CatalogException { assertEquals("URGENT", ca.getPriority().getId()); } + @Test + public void adminPermissionTest() throws CatalogException { + // Add ADMIN permissions to the user2 + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, + new StudyAclParams(StudyPermissions.Permissions.ADMIN_CLINICAL_ANALYSIS.name(), null), ParamUtils.AclAction.SET, ownerToken); + + DataResult dummyEnvironment = createDummyEnvironment(true, false); + + // Update ClinicalAnalysis with user1 + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description"), INCLUDE_RESULT, normalToken1).first(); + assertEquals("My description", clinicalAnalysis.getDescription()); + + // Update ClinicalAnalysis with user2 + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description 2"), INCLUDE_RESULT, normalToken2).first(); + assertEquals("My description 2", clinicalAnalysis.getDescription()); + + // Set status to CLOSED with user1 - FAIL + assertThrows(CatalogAuthorizationException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, normalToken1) + ); + + // Set status to CLOSED with user2 - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, normalToken2).first(); + assertEquals("CLOSED", clinicalAnalysis.getStatus().getId()); + assertTrue(clinicalAnalysis.isLocked()); + + // Unset status from CLOSED to other with user1 - FAIL + assertThrows(CatalogAuthorizationException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")), INCLUDE_RESULT, normalToken1) + ); + + // Edit CLOSED ClinicalAnalysis from user with ADMIN permission + assertThrows(CatalogException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("new description"), INCLUDE_RESULT, normalToken2) + ); + + // Send CLOSED status and edit description from CLOSED ClinicalAnalysis from user with ADMIN permission + assertThrows(CatalogException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")).setDescription("new description"), + INCLUDE_RESULT, normalToken2) + ); + + // Remove CLOSED status and edit description from CLOSED ClinicalAnalysis from user with ADMIN permission + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")).setDescription("new description"), + INCLUDE_RESULT, normalToken2).first(); + assertEquals("READY_FOR_INTERPRETATION", clinicalAnalysis.getStatus().getId()); + assertEquals("new description", clinicalAnalysis.getDescription()); + + // Set status to CLOSED with study admin - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, studyAdminToken1).first(); + assertEquals("CLOSED", clinicalAnalysis.getStatus().getId()); + + // Unset status from CLOSED to other with study admin - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")).setLocked(false), INCLUDE_RESULT, + studyAdminToken1).first(); + assertEquals("READY_FOR_INTERPRETATION", clinicalAnalysis.getStatus().getId()); + + // Update ClinicalAnalysis with user1 - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description 3"), INCLUDE_RESULT, normalToken1).first(); + assertEquals("My description 3", clinicalAnalysis.getDescription()); + } + @Test public void updateCustomStatusTest() throws CatalogException { Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); @@ -1600,7 +1780,7 @@ public void updateCustomStatusTest() throws CatalogException { DataResult dummyEnvironment = createDummyEnvironment(true, false); - ClinicalStatusValue status = configuration.getStatus().get(dummyEnvironment.first().getType()).get(0); + ClinicalStatusValue status = configuration.getStatus().get(0); ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setStatus(new StatusParam(status.getId())); @@ -1645,9 +1825,9 @@ public void updateCustomFlagTest() throws CatalogException { DataResult dummyEnvironment = createDummyEnvironment(true, false); - FlagValue flag1 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(1); - FlagValue flag2 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(3); - FlagValue flag3 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(4); + FlagValue flag1 = configuration.getFlags().get(1); + FlagValue flag2 = configuration.getFlags().get(3); + FlagValue flag3 = configuration.getFlags().get(4); ObjectMap actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FLAGS.key(), ParamUtils.BasicUpdateAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); @@ -1684,8 +1864,8 @@ public void updateCustomFlagTest() throws CatalogException { } // Set other flags - flag1 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(0); - flag2 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(2); + flag1 = configuration.getFlags().get(0); + flag2 = configuration.getFlags().get(2); actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FLAGS.key(), ParamUtils.BasicUpdateAction.SET); options = new QueryOptions(Constants.ACTIONS, actionMap); @@ -1768,7 +1948,7 @@ public void updateInterpretationCustomStatusTest() throws CatalogException { InterpretationStudyConfiguration configuration = study.getInternal().getConfiguration().getClinical().getInterpretation(); DataResult dummyEnvironment = createDummyEnvironment(true, true); - ClinicalStatusValue status = configuration.getStatus().get(dummyEnvironment.first().getType()).get(0); + ClinicalStatusValue status = configuration.getStatus().get(1); InterpretationUpdateParams updateParams = new InterpretationUpdateParams() .setStatus(new StatusParam(status.getId())); @@ -2581,35 +2761,35 @@ public void searchClinicalAnalysisByStatus() throws CatalogException { createDummyEnvironment(false, false); OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.DONE), + new Query(ParamConstants.STATUS_PARAM, "DONE"), new QueryOptions(QueryOptions.INCLUDE, ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_ID.key()), ownerToken); assertEquals(0, search.getNumResults()); search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.READY_FOR_INTERPRETATION), + new Query(ParamConstants.STATUS_PARAM, "READY_FOR_INTERPRETATION"), new QueryOptions(), ownerToken); assertEquals(2, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals("READY_FOR_INTERPRETATION", result.getStatus().getId()); } catalogManager.getClinicalAnalysisManager().update(studyFqn, search.first().getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(ClinicalAnalysisStatus.REJECTED)), QueryOptions.empty(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("REJECTED")), QueryOptions.empty(), ownerToken); search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.READY_FOR_INTERPRETATION), + new Query(ParamConstants.STATUS_PARAM, "READY_FOR_INTERPRETATION"), new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals("READY_FOR_INTERPRETATION", result.getStatus().getId()); } search = catalogManager.getClinicalAnalysisManager().search(studyFqn, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.REJECTED), + new Query(ParamConstants.STATUS_PARAM, "REJECTED"), new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.REJECTED, result.getStatus().getId()); + assertEquals("REJECTED", result.getStatus().getId()); } } @@ -3131,7 +3311,7 @@ public void createClinicalAnalysisWithPanelsTest() throws CatalogException { assertNotNull(panel.getId()); assertNotNull(panel.getName()); } - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); } @Test @@ -3322,7 +3502,7 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogExc catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, true, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Map actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); @@ -3331,7 +3511,7 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogExc try { catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())) - .setPanelLock(true), + .setPanelLocked(true), updateOptions, ownerToken); fail("Updating panels and setting panellock to true in one call should not be accepted"); } catch (CatalogException e) { @@ -3342,24 +3522,24 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogExc .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), updateOptions, ownerToken); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), updateOptions, ownerToken); result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertTrue(result.first().isPanelLock()); + assertTrue(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))) - .setPanelLock(false), + .setPanelLocked(false), updateOptions, ownerToken); result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(1, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), updateOptions, ownerToken); thrown.expect(CatalogException.class); thrown.expectMessage("panelLock"); @@ -3386,13 +3566,13 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisWithInterpretationTest() catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); thrown.expect(CatalogException.class); thrown.expectMessage("not allowed"); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())) - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); } @@ -3414,7 +3594,7 @@ public void setPanelLockWithInterpretationWithNoPanelsTest() throws CatalogExcep catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), @@ -3426,7 +3606,7 @@ public void setPanelLockWithInterpretationWithNoPanelsTest() throws CatalogExcep thrown.expect(CatalogException.class); thrown.expectMessage("any of the case panels"); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); } @@ -3448,7 +3628,7 @@ public void setPanelLockWithInterpretationWithPanelSubsetTest() throws CatalogEx catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), @@ -3462,15 +3642,15 @@ public void setPanelLockWithInterpretationWithPanelSubsetTest() throws CatalogEx .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))), null, QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertFalse(clinicalAnalysis.isPanelLock()); + assertFalse(clinicalAnalysis.isPanelLocked()); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(1, clinicalAnalysis.getInterpretation().getPanels().size()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(1, clinicalAnalysis.getInterpretation().getPanels().size()); } @@ -3493,7 +3673,7 @@ public void setPanelLockWithInterpretationWithDifferentPanelsTest() throws Catal catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))), @@ -3513,7 +3693,7 @@ public void setPanelLockWithInterpretationWithDifferentPanelsTest() throws Catal thrown.expect(CatalogException.class); thrown.expectMessage("not defined by the case"); catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), + .setPanelLocked(true), QueryOptions.empty(), ownerToken); } @@ -3536,7 +3716,7 @@ public void updatePanelsFromClinicalAnalysisWithPanelLockTest() throws CatalogEx catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); @@ -3544,9 +3724,9 @@ public void updatePanelsFromClinicalAnalysisWithPanelLockTest() throws CatalogEx // Set panelLock to true catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), ownerToken); + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); thrown.expect(CatalogException.class); thrown.expectMessage("panelLock"); @@ -3574,7 +3754,7 @@ public void updatePanelLockWithDifferentPanels() throws CatalogException { catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); @@ -3582,15 +3762,15 @@ public void updatePanelLockWithDifferentPanels() throws CatalogException { // Set panelLock to true catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), ownerToken); + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); // Set panelLock to false catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(false), QueryOptions.empty(), ownerToken); + .setPanelLocked(false), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertFalse(clinicalAnalysis.isPanelLock()); + assertFalse(clinicalAnalysis.isPanelLocked()); Map actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); @@ -3606,9 +3786,9 @@ public void updatePanelLockWithDifferentPanels() throws CatalogException { thrown.expectMessage("panels"); // Set panelLock to true catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), ownerToken); + .setPanelLocked(true), QueryOptions.empty(), ownerToken); clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + assertTrue(clinicalAnalysis.isPanelLocked()); } @Test @@ -3624,14 +3804,14 @@ public void updatePanelsFromInterpretationWithLockedCATest() throws CatalogExcep .setId("analysis") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(proband) - .setPanelLock(true) + .setPanelLocked(true) .setPanels(panels.subList(0, 2)); OpenCGAResult result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertTrue(result.first().isPanelLock()); + assertTrue(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); @@ -3663,7 +3843,7 @@ public void updatePanelsFromInterpretationWithUnlockedCATest() throws CatalogExc catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java index c31ba8e9301..80b630d34ee 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java @@ -1,11 +1,9 @@ package org.opencb.opencga.catalog.managers; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -14,15 +12,19 @@ import org.opencb.opencga.core.models.clinical.*; import org.opencb.opencga.core.models.common.StatusParam; import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.panel.PanelReferenceParam; import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.study.StudyAclParams; +import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import static org.junit.Assert.*; @@ -45,7 +47,7 @@ private DataResult createDummyFamily() throws CatalogException { private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new Status().setId(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId("READY_FOR_INTERPRETATION")) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); @@ -60,25 +62,6 @@ private DataResult createDummyEnvironment(boolean createFamily INCLUDE_RESULT, ownerToken); } - private List registerDummyFiles() throws CatalogException { - List files = new LinkedList<>(); - - String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - vcfFile = getClass().getResource("/biofiles/family.vcf").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - String bamFile = getClass().getResource("/biofiles/HG00096.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, ownerToken).first()); - - return files; - } - @Test public void deleteLockedInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); @@ -118,7 +101,7 @@ public void automaticallyLockInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); - assertTrue(StringUtils.isEmpty(interpretation.getStatus().getId())); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, interpretation.getStatus().getType()); assertFalse(interpretation.isLocked()); interpretation = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), @@ -128,7 +111,7 @@ public void automaticallyLockInterpretationTest() throws CatalogException { interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation() - .setStatus(new Status("REJECTED", "", "", "")), + .setStatus(new ClinicalStatus("REJECTED", "", null, "", "", "", "")), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); assertEquals("REJECTED", interpretation.getStatus().getId()); assertTrue(interpretation.isLocked()); @@ -151,15 +134,34 @@ public void interpretationLockedTest() throws CatalogException { assertFalse(secondaryInterpretation.isLocked()); } + // Add ADMIN permissions to the user2 + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, + new StudyAclParams(StudyPermissions.Permissions.ADMIN_CLINICAL_ANALYSIS.name(), null), ParamUtils.AclAction.SET, ownerToken); + // Try to update interpretation 1 try { catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), - new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), ownerToken); + new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), normalToken1); fail("Interpretation is locked so it should not allow this"); } catch (CatalogException e) { - assertTrue(e.getMessage().contains("locked")); + assertTrue(e.getMessage().contains(ClinicalAnalysisPermissions.ADMIN.name())); } + // Try to update interpretation 1 + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), normalToken2); + Interpretation interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), + QueryOptions.empty(), ownerToken).first(); + assertEquals("blabla", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + + // Try to update interpretation 1 + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("blabla2"), null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("blabla2", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + // Update interpretation 2 catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), ownerToken); @@ -175,15 +177,6 @@ public void interpretationLockedTest() throws CatalogException { assertEquals("bloblo", interpretation2.getDescription()); assertTrue(interpretation2.isLocked()); - // Try to lock again and update interpretation 2 - try { - catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), - new InterpretationUpdateParams().setDescription("blabla").setLocked(true), null, QueryOptions.empty(), ownerToken); - fail("Interpretation was already locked so it should not allow this"); - } catch (CatalogException e) { - assertTrue(e.getMessage().contains("locked")); - } - // Unlock and update interpretation 2 catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), new InterpretationUpdateParams().setDescription("blabla").setLocked(false), null, QueryOptions.empty(), ownerToken); @@ -210,6 +203,29 @@ public void interpretationLockedTest() throws CatalogException { assertTrue(secondaryInterpretation.isLocked()); } + // Try to update the interpretation 1 + try { + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description"), null, QueryOptions.empty(), normalToken1); + fail("Case and Interpretation are locked so it should not allow this"); + } catch (CatalogException e) { + assertTrue(e.getMessage().contains(ClinicalAnalysisPermissions.ADMIN.name()) && e.getMessage().toLowerCase().contains("permission denied")); + } + + // Try to update the interpretation 1 (ADMIN permission) + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description"), null, QueryOptions.empty(), normalToken2); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("new description", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + + // Try to update the interpretation 1 (owner user) + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description2"), null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("new description2", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + // Try to unlock interpretation 1 try { catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), @@ -230,6 +246,43 @@ public void interpretationLockedTest() throws CatalogException { } } + @Test + public void interpretationStatusTest() throws CatalogException { + ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); + + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + + // Create 2 allowed statuses of type CLOSED + ClinicalAnalysisStudyConfiguration studyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + List statusValueList = new ArrayList<>(); + for (ClinicalStatusValue status : studyConfiguration.getInterpretation().getStatus()) { + if (!status.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + statusValueList.add(status); + } + } + // Add two statuses of type CLOSED + statusValueList.add(new ClinicalStatusValue("closed1", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + statusValueList.add(new ClinicalStatusValue("closed2", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + studyConfiguration.getInterpretation().setStatus(statusValueList); + catalogManager.getClinicalAnalysisManager().configureStudy(studyFqn, studyConfiguration, studyAdminToken1); + + // Update status to one of the new statuses + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), + new InterpretationUpdateParams().setStatus(new StatusParam("closed1")), null, QueryOptions.empty(), studyAdminToken1); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), studyAdminToken1).first(); + assertEquals("closed1", interpretation.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, interpretation.getStatus().getType()); + assertTrue(interpretation.isLocked()); + + // Update status to the other new CLOSED status + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), + new InterpretationUpdateParams().setStatus(new StatusParam("closed2")), null, QueryOptions.empty(), studyAdminToken1); + assertEquals("closed1", interpretation.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, interpretation.getStatus().getType()); + assertTrue(interpretation.isLocked()); + } + @Test public void createInterpretationWithSubsetOfPanels() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); @@ -247,7 +300,7 @@ public void createInterpretationWithSubsetOfPanels() throws CatalogException { catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), updateParams, QueryOptions.empty(), ownerToken); updateParams = new ClinicalAnalysisUpdateParams() - .setPanelLock(true); + .setPanelLocked(true); catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), updateParams, QueryOptions.empty(), ownerToken); // Create interpretation with just panel1 diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java index 11e5d29e37a..9d1afcadb2b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java @@ -7,6 +7,9 @@ import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; import org.opencb.opencga.core.models.notes.NoteUpdateParams; @@ -16,6 +19,8 @@ import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import static org.junit.Assert.*; @@ -190,6 +195,45 @@ public void updateStudyNoteTest() throws CatalogException { catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), noteUpdateParams, INCLUDE_RESULT, normalToken1); } + @Test + public void noteTagsUpdateTest() throws CatalogException { + NoteCreateParams noteCreateParams = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setTags(Arrays.asList("tag1", "tag2")) + .setValue("hello"); + Note note = catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(2, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag1", "tag2").toArray(), note.getTags().toArray()); + + QueryOptions queryOptions = new QueryOptions(); + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.ADD); + queryOptions.put(Constants.ACTIONS, actionMap); + queryOptions.put(ParamConstants.INCLUDE_RESULT_PARAM, true); + NoteUpdateParams updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag3", "tag1")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(3, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag1", "tag2", "tag3").toArray(), note.getTags().toArray()); + + // Remove tag1 and tag2 + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.REMOVE); + queryOptions.put(Constants.ACTIONS, actionMap); + updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag1", "tag2")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(1, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag3").toArray(), note.getTags().toArray()); + + // Set new list of tags + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.SET); + queryOptions.put(Constants.ACTIONS, actionMap); + updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag4", "tag5")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(2, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag4", "tag5").toArray(), note.getTags().toArray()); + } + @Test public void getStudyNoteTest() throws CatalogException { NoteCreateParams noteCreateParams1 = new NoteCreateParams() diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java index ca9323b422d..a72d52eace3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java @@ -287,7 +287,7 @@ private void updateAndAssertChanges(String orgId, OrganizationUserUpdateParams u User user = catalogManager.getOrganizationManager().updateUser(orgId, normalUserId1, userUpdateParams, INCLUDE_RESULT, token).first(); assertEquals(userUpdateParams.getName(), user.getName()); assertEquals(userUpdateParams.getEmail(), user.getEmail()); - assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getAccount().getExpirationDate()); + assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getInternal().getAccount().getExpirationDate()); assertEquals(userUpdateParams.getQuota().getCpuUsage(), user.getQuota().getCpuUsage()); assertEquals(userUpdateParams.getQuota().getDiskUsage(), user.getQuota().getDiskUsage()); assertEquals(userUpdateParams.getQuota().getMaxCpu(), user.getQuota().getMaxCpu()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java index 10b54926e6c..43191dc6838 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java @@ -244,7 +244,7 @@ public void panelIncrementVersionWithCasesAndInterpretations() throws CatalogExc ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getInterpretationManager().create(studyFqn, case2.getId(), interpretation5, ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); - catalogManager.getClinicalAnalysisManager().update(studyFqn, case2.getId(), new ClinicalAnalysisUpdateParams().setPanelLock(true), + catalogManager.getClinicalAnalysisManager().update(studyFqn, case2.getId(), new ClinicalAnalysisUpdateParams().setPanelLocked(true), QueryOptions.empty(), ownerToken); // case 3 diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java index 3a0a4e4a808..85e13914da5 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java @@ -38,7 +38,6 @@ import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.GroupUpdateParams; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -89,7 +88,7 @@ public void searchSampleNoPermissions() throws CatalogException { public void searchProjects() throws CatalogException { String org2 = "otherOrg"; catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); - catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setAccount(new Account()).setOrganization(org2), + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java new file mode 100644 index 00000000000..f23295fbd60 --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java @@ -0,0 +1,664 @@ +package org.opencb.opencga.catalog.managers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.PasswordUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; +import org.opencb.opencga.core.models.study.Group; +import org.opencb.opencga.core.models.study.GroupUpdateParams; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.StudyAclParams; +import org.opencb.opencga.core.models.user.*; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.MediumTests; + +import javax.naming.NamingException; +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +@Category(MediumTests.class) +public class UserManagerTest extends AbstractManagerTest { + + @Test + public void createOpencgaUserTest() throws CatalogException { + thrown.expect(CatalogException.class); + thrown.expectMessage("forbidden"); + catalogManager.getUserManager().create(new User().setId(ParamConstants.OPENCGA_USER_ID).setName(orgOwnerUserId) + .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + } + + + @Test + public void testAdminUserExists() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + assertEquals(ParamConstants.OPENCGA_USER_ID, payload.getUserId()); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, payload.getOrganization()); + } + + @Test + public void searchUsersTest() throws CatalogException { + OpenCGAResult search = catalogManager.getUserManager().search(organizationId, new Query(), QueryOptions.empty(), opencgaToken); + assertEquals(8, search.getNumResults()); + for (User user : search.getResults()) { + if (noAccessUserId1.equals(user.getId())) { + assertEquals(0, user.getProjects().size()); + } else if (user.getId().startsWith("normalUser")) { + assertEquals(1, user.getProjects().size()); + } else { + assertEquals(2, user.getProjects().size()); + } + } + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), ownerToken); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken2); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken1); + assertEquals(8, search.getNumResults()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), normalToken1)); + } + + @Test + public void testGetToken() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + Map claims = new HashMap<>(); + claims.put("a", "hola"); + claims.put("ab", "byw"); + // Create a token valid for 1 second + String expiringToken = catalogManager.getUserManager().getToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, 1L, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + + String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(nonExpiringToken).getUserId()); + + Thread.sleep(1000); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("expired"); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + } + + @Test + public void loginWithoutOrganizationId() throws CatalogException { + String token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + JwtPayload jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + token = catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(organizationId, jwtPayload.getOrganization()); + + // Create a third organization + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("other").setName("Test"), QueryOptions.empty(), opencgaToken); + token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + thrown.expect(CatalogParameterException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD); + } + + @Test + public void testCreateExistingUser() throws Exception { + thrown.expect(CatalogException.class); + thrown.expectMessage(containsString("already exists")); + catalogManager.getUserManager().create(orgOwnerUserId, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, + null, opencgaToken); + } + + @Test + public void testCreateAnonymousUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, + organizationId, null, opencgaToken); + } + + @Test + public void testCreateRegisteredUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + } + + @Test + public void testLogin() throws Exception { + catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD); + + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); + catalogManager.getUserManager().login(organizationId, normalUserId1, "fakePassword"); + } + + @Test + public void refreshTokenTest() throws Exception { + String refreshToken = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getRefreshToken(); + AuthenticationResponse authenticationResponse = catalogManager.getUserManager().refreshToken(refreshToken); + assertNotNull(authenticationResponse); + assertNotNull(authenticationResponse.getToken()); + } + + @Test + public void anonymousUserLoginTest() throws CatalogException { + AuthenticationResponse authResponse = catalogManager.getUserManager().loginAnonymous(organizationId); + assertNotNull(authResponse.getToken()); + + String org2 = "otherOrg"; + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); + String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); + Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() + .setId("project") + .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), + INCLUDE_RESULT, owner2Token).first(); + Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); + + try { + catalogManager.getUserManager().loginAnonymous(org2); + fail("Anonymous user should not get a token for that organization as it has not been granted any kind of access"); + } catch (Exception e) { + assertEquals(CatalogAuthenticationException.class, e.getClass()); + assertTrue(e.getMessage().contains("not found")); + } + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + authResponse = catalogManager.getUserManager().loginAnonymous(org2); + assertNotNull(authResponse.getToken()); + + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().loginAnonymous(org2); + } + + @Test + public void incrementLoginAttemptsTest() throws CatalogException { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal3 = user.getInternal(); + assertEquals(1, userInternal3.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + for (int i = 2; i < 5; i++) { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(i, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + } + + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal2 = user.getInternal(); + assertEquals(5, userInternal2.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + assertTrue(incorrect.getMessage().contains("banned")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal1 = user.getInternal(); + assertEquals(5, userInternal1.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("banned")); + + // Remove ban from user + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(0, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void changeUserStatusTest() throws CatalogException { + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); + assertTrue(authException.getMessage().contains("own account")); + + authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(authException.getMessage().contains("suspend administrators")); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); + assertTrue(incorrect.getMessage().contains("suspended")); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); + String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + + CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); + assertTrue(paramException.getMessage().contains("Invalid status")); + + CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(dbException.getMessage().contains("not exist")); + } + + @Test + public void loginExpiredAccountTest() throws CatalogException { + // Expire account of normalUserId1 + ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); + catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("expired")); + + // Ensure it doesn't matter whether opencga account is expired or not + catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); + String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void updateUserTest() throws JsonProcessingException, CatalogException { + UserUpdateParams userUpdateParams = new UserUpdateParams() + .setName("newName") + .setEmail("mail@mail.com"); + ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); + assertEquals(userUpdateParams.getName(), user.getName()); + assertEquals(userUpdateParams.getEmail(), user.getEmail()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); + + userUpdateParams = new UserUpdateParams() + .setEmail("notAnEmail"); + ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); + } + + @Test + public void testGetUserInfo() throws CatalogException { + // OpenCGA administrator + DataResult user = catalogManager.getUserManager().get(organizationId, + Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), opencgaToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization owner + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), ownerToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization administrator + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), orgAdminToken1); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), + studyAdminToken1); + } + + @Test + public void testGetProjectsFromUserInfo() throws CatalogException { + String userId = organizationId; + catalogManager.getUserManager().create(userId, "test", "mail@mail.com", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("test")), ownerToken); + String token = catalogManager.getUserManager().login(organizationId, userId, TestParamConstants.PASSWORD).getToken(); + + DataResult user = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(), token); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), normalToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + } + + @Test + public void testModifyUser() throws CatalogException, InterruptedException, IOException { + ObjectMap params = new ObjectMap(); + String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); + String newPassword = PasswordUtils.getStrongRandomPassword(); + String newEmail = "new@email.ac.uk"; + + params.put("name", newName); + + Thread.sleep(10); + + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + catalogManager.getUserManager().update(orgOwnerUserId, new ObjectMap("email", newEmail), null, ownerToken); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD, newPassword); + + List userList = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(QueryOptions + .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), + UserDBAdaptor.QueryParams.ATTRIBUTES.key())), ownerToken).getResults(); + User userPost = userList.get(0); + System.out.println("userPost = " + userPost); + assertEquals(userPost.getName(), newName); + assertEquals(userPost.getEmail(), newEmail); + + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("The new password has already been used")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, anotherPassword); + + try { + params = new ObjectMap(); + params.put("password", "1234321"); + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + fail("Expected exception"); + } catch (CatalogDBException e) { + System.out.println(e); + } + + try { + catalogManager.getUserManager().update(orgOwnerUserId, params, null, orgAdminToken1); + fail("Expected exception"); + } catch (CatalogException e) { + System.out.println(e); + } + } + + @Test + public void automaticPasswordExpirationTest() throws CatalogException { + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(1); + + String oneDay = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(1)); + String twoDays = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(2)); + + User user = new User().setId("tempUser"); + String password = PasswordUtils.getStrongRandomPassword(); + User storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account2 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(oneDay) <= Long.parseLong(account2.getPassword().getExpirationDate())); + Account account1 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(twoDays) > Long.parseLong(account1.getPassword().getExpirationDate())); + + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(-5); + user = new User().setId("tempUser2"); + storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account = storedUser.getInternal().getAccount(); + assertNull(account.getPassword().getExpirationDate()); + } + + @Test + public void loginUserPasswordExpiredTest() throws CatalogException { + try (CatalogManager mockCatalogManager = mockCatalogManager()) { + UserDBAdaptor userDBAdaptor = mockCatalogManager.getUserManager().getUserDBAdaptor(organizationId); + + OpenCGAResult result = mockCatalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), normalToken1); + + // Set password expired 2 days ago + Date date = TimeUtils.addDaysToCurrentDate(-2); + String beforeYesterday = TimeUtils.getTime(date); + User user = result.first(); + user.getInternal().getAccount().getPassword().setExpirationDate(beforeYesterday); + + Mockito.doReturn(result).when(userDBAdaptor).get(normalUserId1, UserManager.INCLUDE_INTERNAL); + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> mockCatalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("expired on " + beforeYesterday)); + } + } + + @Test + public void changePasswordTest() throws CatalogException { + String newPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, newPassword); + + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword)); + assertTrue(exception.getMessage().contains("verify that the current password is correct")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, anotherPassword); + } + + @Test + public void testUpdateUserConfig() throws CatalogException { + Map map = new HashMap<>(); + map.put("key1", "value1"); + map.put("key2", "value2"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + + Map config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(2, config.size()); + assertEquals("value1", config.get("key1")); + assertEquals("value2", config.get("key2")); + + map = new HashMap<>(); + map.put("key2", "value3"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(1, config.size()); + assertEquals("value3", config.get("key2")); + + catalogManager.getUserManager().deleteConfig(normalUserId1, "a", normalToken1); + + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1); + } + + private String getAdminToken() throws CatalogException, IOException { + return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + } + + @Test + public void createUserUsingMailAsId() throws CatalogException { + catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); + AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); + assertNotNull(login); + User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); + assertEquals("hello.mail@mymail.org", user.getId()); + } + + @Test + public void getUserInfoTest() throws CatalogException { + OpenCGAResult result = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, QueryOptions.empty(), ownerToken); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, QueryOptions.empty(), orgAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, QueryOptions.empty(), studyAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), normalToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(1, result.first().getProjects().size()); + } + + @Ignore + @Test + public void importLdapUsers() throws CatalogException, NamingException, IOException { + // Action only for admins + catalogManager.getUserManager().importRemoteEntities(organizationId, "ldap", Arrays.asList("pfurio", "imedina"), false, null, null, + getAdminToken()); + // TODO: Validate the users have been imported + } + + // To make this test work we will need to add a correct user and password to be able to login + @Ignore + @Test + public void loginNotRegisteredUsers() throws CatalogException { + // Action only for admins + Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); + catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", "view_only"), + ParamUtils.AclAction.SET, ownerToken); + String token = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, "password").getToken(); + + assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + + // We remove the permissions for group ldap + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", ""), + ParamUtils.AclAction.RESET, this.ownerToken); + + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + } + + @Ignore + @Test + public void syncUsers() throws CatalogException { + // Action only for admins + String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", "bio", "bio", studyFqn, true, token); + DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertEquals(0, bio.first().getUserIds().size()); + + catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, studyFqn, "ldap", token); + bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertTrue(!bio.first().getUserIds().isEmpty()); + } + + @Ignore + @Test + public void importLdapGroups() throws CatalogException, IOException { + // Action only for admins + String remoteGroup = "bio"; + String internalGroup = "test"; + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + + DataResult test = catalogManager.getStudyManager().getGroup(studyFqn, "test", ownerToken); + assertEquals(1, test.getNumResults()); + assertEquals("@test", test.first().getId()); + assertTrue(test.first().getUserIds().size() > 0); + +// internalGroup = "test1"; +// try { +// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); +// fail("Should not be possible creating another group containing the same users that belong to a different group"); +// } catch (CatalogException e) { +// System.out.println(e.getMessage()); +// } + + remoteGroup = "bioo"; + internalGroup = "test2"; + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + } + + +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java index f7b28721987..ca96f875326 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java @@ -11,7 +11,6 @@ import org.opencb.opencga.catalog.templates.TemplateManager; import org.opencb.opencga.catalog.templates.config.TemplateManifest; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -25,11 +24,14 @@ public class TemplateManagerTest extends AbstractManagerTest { public void test() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); catalogManager.getStudyManager().create("project", new Study().setId("study"), QueryOptions.empty(), ownerToken); @@ -54,13 +56,13 @@ public void test() throws Exception { public void test_yaml() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 2d7cbe0861a..d0ebcf32ba6 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -17,6 +17,25 @@ analysis: packages: # List of packages where to find analysis tools - "org.opencb.opencga" scratchDir: "/tmp/" # Scratch folder for the analysis. + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + - id: "exomiser" + version: "13.1" + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + - id: "exomiser" + version: "14.0" + defaultVersion: true + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index 6fb3c4dfffa..a45c200d12e 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -23,9 +23,9 @@ #' | loadAnnotationSets | /{apiVersion}/analysis/clinical/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | updateClinicalConfiguration | /{apiVersion}/analysis/clinical/clinical/configuration/update | study, body | #' | create | /{apiVersion}/analysis/clinical/create | include, exclude, study, skipCreateDefaultInterpretation, includeResult, body[*] | -#' | distinct | /{apiVersion}/analysis/clinical/distinct | study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, status, internalStatus, annotation, deleted, field[*] | -#' | distinctInterpretation | /{apiVersion}/analysis/clinical/interpretation/distinct | study, id, uuid, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release, field[*] | -#' | searchInterpretation | /{apiVersion}/analysis/clinical/interpretation/search | include, exclude, limit, skip, sort, study, id, uuid, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release | +#' | distinct | /{apiVersion}/analysis/clinical/distinct | study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, snapshot, status, internalStatus, annotation, deleted, field[*] | +#' | distinctInterpretation | /{apiVersion}/analysis/clinical/interpretation/distinct | study, id, uuid, name, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release, field[*] | +#' | searchInterpretation | /{apiVersion}/analysis/clinical/interpretation/search | include, exclude, limit, skip, sort, study, id, uuid, name, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release | #' | infoInterpretation | /{apiVersion}/analysis/clinical/interpretation/{interpretations}/info | include, exclude, interpretations[*], study, version, deleted | #' | runInterpreterCancerTiering | /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | runInterpreterExomiser | /{apiVersion}/analysis/clinical/interpreter/exomiser/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | @@ -41,13 +41,13 @@ #' | summaryRgaIndividual | /{apiVersion}/analysis/clinical/rga/individual/summary | limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | queryRgaVariant | /{apiVersion}/analysis/clinical/rga/variant/query | include, exclude, limit, skip, count, includeIndividual, skipIndividual, limitIndividual, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | summaryRgaVariant | /{apiVersion}/analysis/clinical/rga/variant/summary | limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | -#' | search | /{apiVersion}/analysis/clinical/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, status, internalStatus, annotation, deleted | +#' | search | /{apiVersion}/analysis/clinical/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, snapshot, status, internalStatus, annotation, deleted | #' | queryVariant | /{apiVersion}/analysis/clinical/variant/query | include, exclude, limit, skip, count, approximateCount, approximateCountSamplingSize, savedFilter, includeInterpretation, id, region, type, study, file, filter, qual, fileData, sample, sampleData, sampleAnnotation, cohort, cohortStatsRef, cohortStatsAlt, cohortStatsMaf, cohortStatsMgf, cohortStatsPass, missingAlleles, missingGenotypes, score, family, familyDisorder, familySegregation, familyMembers, familyProband, gene, ct, xref, biotype, proteinSubstitution, conservation, populationFrequencyAlt, populationFrequencyRef, populationFrequencyMaf, transcriptFlag, geneTraitId, go, expression, proteinKeyword, drug, functionalScore, clinical, clinicalSignificance, clinicalConfirmedStatus, customAnnotation, panel, panelModeOfInheritance, panelConfidence, panelRoleInCancer, panelFeatureType, panelIntersection, trait | #' | acl | /{apiVersion}/analysis/clinical/{clinicalAnalyses}/acl | clinicalAnalyses[*], study, member, silent | #' | delete | /{apiVersion}/analysis/clinical/{clinicalAnalyses}/delete | study, force, clinicalAnalyses[*] | #' | update | /{apiVersion}/analysis/clinical/{clinicalAnalyses}/update | include, exclude, clinicalAnalyses[*], study, commentsAction, flagsAction, analystsAction, filesAction, panelsAction, annotationSetsAction, includeResult, body[*] | #' | updateAnnotationSetsAnnotations | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/annotationSets/{annotationSet}/annotations/update | clinicalAnalysis[*], study, annotationSet[*], action, body | -#' | info | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/info | include, exclude, flattenAnnotations, clinicalAnalysis[*], study, deleted | +#' | info | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/info | include, exclude, flattenAnnotations, clinicalAnalysis[*], study, version, deleted | #' | createInterpretation | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/create | include, exclude, clinicalAnalysis[*], study, setAs, includeResult, body[*] | #' | clearInterpretation | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/clear | study, interpretations[*], clinicalAnalysis[*] | #' | deleteInterpretation | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/delete | study, clinicalAnalysis[*], interpretations[*], setAsPrimary | @@ -131,6 +131,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param dueDate Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param qualityControlSummary Clinical Analysis quality control summary. #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). #' @param status Filter by status. #' @param internalStatus Filter by internal status. #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. @@ -144,6 +145,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of Interpretation UUIDs up to a maximum of 100. + #' @param name Comma separated list of Interpretation names up to a maximum of 100. #' @param clinicalAnalysisId Clinical Analysis id. #' @param analystId Analyst ID. #' @param methodName Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -170,6 +172,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of Interpretation UUIDs up to a maximum of 100. + #' @param name Comma separated list of Interpretation names up to a maximum of 100. #' @param clinicalAnalysisId Clinical Analysis id. #' @param analystId Analyst ID. #' @param methodName Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -222,7 +225,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobScheduledStartTime Time when the job is scheduled to start. #' @param jobPriority Priority of the job. #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. - #' @param data Exomizer interpretation analysis params. + #' @param data Exomiser interpretation analysis params. runInterpreterExomiser=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/exomiser", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -555,6 +558,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param dueDate Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param qualityControlSummary Clinical Analysis quality control summary. #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). #' @param status Filter by status. #' @param internalStatus Filter by internal status. #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. @@ -683,6 +687,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param flattenAnnotations Flatten the annotations?. #' @param clinicalAnalysis Comma separated list of clinical analysis IDs or names up to a maximum of 100. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param version Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if multiple clinical ids are provided. #' @param deleted Boolean to retrieve deleted entries. info=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical", subcategoryId=clinicalAnalysis, action="info", params=params, httpMethod="GET", as.queryParam=NULL, diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index bd581f1ba5e..1c3fc8d7072 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -44,6 +44,7 @@ #' | configureVariantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index/configure | study, skipRebuild, body | #' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | #' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, samples | +#' | setupVariant | /{apiVersion}/operation/variant/setup | study, body | #' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | @@ -406,6 +407,13 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params subcategory="variant/secondaryIndex", subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/operation/variant/setup: + #' Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param data Variant setup params. + setupVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", + subcategoryId=NULL, action="setup", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/operation/variant/stats/delete: #' Deletes the VariantStats of a cohort/s from the database. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. diff --git a/opencga-client/src/main/R/R/Organization-methods.R b/opencga-client/src/main/R/R/Organization-methods.R index da241e16fb5..14e44864ed9 100644 --- a/opencga-client/src/main/R/R/Organization-methods.R +++ b/opencga-client/src/main/R/R/Organization-methods.R @@ -23,7 +23,7 @@ #' | createNotes | /{apiVersion}/organizations/notes/create | include, exclude, includeResult, body[*] | #' | searchNotes | /{apiVersion}/organizations/notes/search | include, exclude, creationDate, modificationDate, id, scope, visibility, uuid, userId, tags, version | #' | deleteNotes | /{apiVersion}/organizations/notes/{id}/delete | id[*], includeResult | -#' | updateNotes | /{apiVersion}/organizations/notes/{id}/update | include, exclude, id[*], includeResult, body[*] | +#' | updateNotes | /{apiVersion}/organizations/notes/{id}/update | include, exclude, id[*], tagsAction, includeResult, body[*] | #' | userUpdateStatus | /{apiVersion}/organizations/user/{user}/status/update | include, exclude, user[*], organization, includeResult, body[*] | #' | updateUser | /{apiVersion}/organizations/user/{user}/update | include, exclude, user[*], organization, includeResult, body[*] | #' | updateConfiguration | /{apiVersion}/organizations/{organization}/configuration/update | include, exclude, organization[*], includeResult, authenticationOriginsAction, body[*] | @@ -85,6 +85,7 @@ setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param id Note unique identifier. + #' @param tagsAction Action to be performed if the array of tags is being updated. Allowed values: ['ADD SET REMOVE'] #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing the Note fields to be updated. updateNotes=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="notes", diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index ddbc75ca374..ee3f4509354 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -31,7 +31,7 @@ #' | createNotes | /{apiVersion}/studies/{study}/notes/create | include, exclude, study[*], includeResult, body[*] | #' | searchNotes | /{apiVersion}/studies/{study}/notes/search | include, exclude, study[*], creationDate, modificationDate, id, uuid, userId, tags, visibility, version | #' | deleteNotes | /{apiVersion}/studies/{study}/notes/{id}/delete | study[*], id[*], includeResult | -#' | updateNotes | /{apiVersion}/studies/{study}/notes/{id}/update | include, exclude, study[*], id[*], includeResult, body[*] | +#' | updateNotes | /{apiVersion}/studies/{study}/notes/{id}/update | include, exclude, study[*], id[*], tagsAction, includeResult, body[*] | #' | permissionRules | /{apiVersion}/studies/{study}/permissionRules | study[*], entity[*] | #' | updatePermissionRules | /{apiVersion}/studies/{study}/permissionRules/update | study[*], entity[*], action, body[*] | #' | runTemplates | /{apiVersion}/studies/{study}/templates/run | study[*], jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | @@ -191,6 +191,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, group, id, members, stud #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Note unique identifier. + #' @param tagsAction Action to be performed if the array of tags is being updated. Allowed values: ['ADD SET REMOVE'] #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing the Note fields to be updated. updateNotes=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="notes", diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index 869c987e791..7e94ff82723 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -175,6 +175,7 @@ public RestResponse create(ClinicalAnalysisCreateParams data, * dueDate: Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * qualityControlSummary: Clinical Analysis quality control summary. * release: Release when it was created. + * snapshot: Snapshot value (Latest version of the entry in the specified release). * status: Filter by status. * internalStatus: Filter by internal status. * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -197,6 +198,7 @@ public RestResponse distinct(String field, ObjectMap params) throws C * id: Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + * name: Comma separated list of Interpretation names up to a maximum of 100. * clinicalAnalysisId: Clinical Analysis id. * analystId: Analyst ID. * methodName: Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' @@ -230,6 +232,7 @@ public RestResponse distinctInterpretation(String field, ObjectMap pa * id: Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + * name: Comma separated list of Interpretation names up to a maximum of 100. * clinicalAnalysisId: Clinical Analysis id. * analystId: Analyst ID. * methodName: Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' @@ -293,7 +296,7 @@ public RestResponse runInterpreterCancerTiering(CancerTieringInterpretation /** * Run exomiser interpretation analysis. - * @param data Exomizer interpretation analysis params. + * @param data Exomiser interpretation analysis params. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. @@ -717,6 +720,7 @@ public RestResponse summaryRgaVariant(ObjectMap params * dueDate: Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * qualityControlSummary: Clinical Analysis quality control summary. * release: Release when it was created. + * snapshot: Snapshot value (Latest version of the entry in the specified release). * status: Filter by status. * internalStatus: Filter by internal status. * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -917,6 +921,8 @@ public RestResponse updateAnnotationSetsAnnotations(String clinicalAnaly * exclude: Fields excluded in the response, whole JSON path must be provided. * flattenAnnotations: Flatten the annotations?. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * version: Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if multiple clinical + * ids are provided. * deleted: Boolean to retrieve deleted entries. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java index 67d55693b52..b5d5a3b7fa1 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java @@ -128,6 +128,7 @@ public RestResponse deleteNotes(String id, ObjectMap params) throws Client * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. + * tagsAction: Action to be performed if the array of tags is being updated. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index e5b054ae283..b1a8bce6959 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -288,6 +288,7 @@ public RestResponse deleteNotes(String study, String id, ObjectMap params) * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. + * tagsAction: Action to be performed if the array of tags is being updated. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index e3f43735a7d..97c999d192d 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -43,6 +43,8 @@ import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.operations.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.RestResponse; @@ -598,6 +600,20 @@ public RestResponse deleteVariantSecondaryIndex(ObjectMap params) throws Cl return execute("operation", null, "variant/secondaryIndex", null, "delete", params, DELETE, Job.class); } + /** + * Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + * @param data Variant setup params. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse setupVariant(VariantSetupParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("operation", null, "variant", null, "setup", params, POST, VariantSetupResult.class); + } + /** * Deletes the VariantStats of a cohort/s from the database. * @param data Variant stats delete params. diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index f2e4031455c..931d5fe01ce 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -118,6 +118,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.dueDate] - Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.qualityControlSummary] - Clinical Analysis quality control summary. * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). * @param {String} [params.status] - Filter by status. * @param {String} [params.internalStatus] - Filter by internal status. * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -136,6 +137,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.id] - Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of Interpretation UUIDs up to a maximum of 100. + * @param {String} [params.name] - Comma separated list of Interpretation names up to a maximum of 100. * @param {String} [params.clinicalAnalysisId] - Clinical Analysis id. * @param {String} [params.analystId] - Analyst ID. * @param {String} [params.methodName] - Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. @@ -166,6 +168,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.id] - Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of Interpretation UUIDs up to a maximum of 100. + * @param {String} [params.name] - Comma separated list of Interpretation names up to a maximum of 100. * @param {String} [params.clinicalAnalysisId] - Clinical Analysis id. * @param {String} [params.analystId] - Analyst ID. * @param {String} [params.methodName] - Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. @@ -220,7 +223,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { } /** Run exomiser interpretation analysis - * @param {Object} data - Exomizer interpretation analysis params. + * @param {Object} data - Exomiser interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not @@ -607,6 +610,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.dueDate] - Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.qualityControlSummary] - Clinical Analysis quality control summary. * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). * @param {String} [params.status] - Filter by status. * @param {String} [params.internalStatus] - Filter by internal status. * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -803,6 +807,8 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.version] - Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if + * multiple clinical ids are provided. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -834,7 +840,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ clearInterpretation(clinicalAnalysis, interpretations, params) { - return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "clear", params); + return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "clear", null, params); } /** Delete interpretation @@ -859,7 +865,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ revertInterpretation(clinicalAnalysis, interpretation, version, params) { - return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "revert", {version, ...params}); + return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "revert", null, {version, ...params}); } /** Update interpretation fields diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index 3e1abab6f85..cfedee4d933 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -266,7 +266,7 @@ export default class File extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ upload(params) { - return this._post("files", null, null, null, "upload", params); + return this._post("files", null, null, null, "upload", null, params); } /** Return the acl defined for the file or folder. If member is provided, it will only return the acl for the member. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index 02af95b1363..222eaa7c67c 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -39,7 +39,7 @@ export default class GA4GH extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ searchReads() { - return this._post("ga4gh", null, "reads", null, "search"); + return this._post("ga4gh", null, "reads", null, "search", null); } /** Fetch alignment files using HTSget protocol @@ -78,7 +78,7 @@ export default class GA4GH extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ searchVariants() { - return this._post("ga4gh", null, "variants", null, "search"); + return this._post("ga4gh", null, "variants", null, "search", null); } } \ No newline at end of file diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index f0ebb640147..bb88f18dd7a 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -213,7 +213,7 @@ export default class Job extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ kill(job, params) { - return this._post("jobs", job, null, null, "kill", params); + return this._post("jobs", job, null, null, "kill", null, params); } /** Show the first lines of a log file (up to a limit) diff --git a/opencga-client/src/main/javascript/Organization.js b/opencga-client/src/main/javascript/Organization.js index 1580a4a7828..09b79429246 100644 --- a/opencga-client/src/main/javascript/Organization.js +++ b/opencga-client/src/main/javascript/Organization.js @@ -97,6 +97,8 @@ export default class Organization extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {"ADD SET REMOVE"} [params.tagsAction = "ADD"] - Action to be performed if the array of tags is being updated. The default + * value is ADD. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index 524221df852..041fa7ed691 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -85,7 +85,7 @@ export default class Project extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ incRelease(project) { - return this._post("projects", project, null, null, "incRelease"); + return this._post("projects", project, null, null, "incRelease", null); } /** Fetch all the studies contained in the project diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 9e013fef0dc..bbe53e93df9 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -219,6 +219,8 @@ export default class Study extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {"ADD SET REMOVE"} [params.tagsAction = "ADD"] - Action to be performed if the array of tags is being updated. The default + * value is ADD. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -279,7 +281,7 @@ export default class Study extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ uploadTemplates(study, params) { - return this._post("studies", study, "templates", null, "upload", params); + return this._post("studies", study, "templates", null, "upload", null, params); } /** Delete template diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index 843416dd14e..32b28182c79 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -39,7 +39,7 @@ export default class User extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ anonymous(organization) { - return this._post("users", null, null, null, "anonymous", organization); + return this._post("users", null, null, null, "anonymous", null, organization); } /** Create a new user diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index 0d79024c496..feb3959ec70 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -488,6 +488,16 @@ export default class VariantOperation extends OpenCGAParentClass { return this._delete("operation", null, "variant/secondaryIndex", null, "delete", params); } + /** Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + * @param {Object} [data] - Variant setup params. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + setupVariant(data, params) { + return this._post("operation", null, "variant", null, "setup", data, params); + } + /** Deletes the VariantStats of a cohort/s from the database * @param {Object} data - Variant stats delete params. * @param {Object} [params] - The Object containing the following optional parameters: diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index 6ccab3a988a..0a531515217 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -141,6 +141,8 @@ def distinct(self, field, **options): :param str quality_control_summary: Clinical Analysis quality control summary. :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). :param str status: Filter by status. :param str internal_status: Filter by internal status. :param str annotation: Annotation filters. Example: @@ -167,6 +169,8 @@ def distinct_interpretation(self, field, **options): sensitive, '~/value/i' for case insensitive search. :param str uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + :param str name: Comma separated list of Interpretation names up to a + maximum of 100. :param str clinical_analysis_id: Clinical Analysis id. :param str analyst_id: Analyst ID. :param str method_name: Interpretation method name. Also admits basic @@ -208,6 +212,8 @@ def search_interpretation(self, **options): sensitive, '~/value/i' for case insensitive search. :param str uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + :param str name: Comma separated list of Interpretation names up to a + maximum of 100. :param str clinical_analysis_id: Clinical Analysis id. :param str analyst_id: Analyst ID. :param str method_name: Interpretation method name. Also admits basic @@ -280,7 +286,7 @@ def run_interpreter_exomiser(self, data=None, **options): Run exomiser interpretation analysis. PATH: /{apiVersion}/analysis/clinical/interpreter/exomiser/run - :param dict data: Exomizer interpretation analysis params. (REQUIRED) + :param dict data: Exomiser interpretation analysis params. (REQUIRED) :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the @@ -748,6 +754,8 @@ def search(self, **options): :param str quality_control_summary: Clinical Analysis quality control summary. :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). :param str status: Filter by status. :param str internal_status: Filter by internal status. :param str annotation: Annotation filters. Example: @@ -1031,6 +1039,9 @@ def info(self, clinical_analysis, **options): :param bool flatten_annotations: Flatten the annotations?. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + :param str version: Comma separated list of clinical versions. 'all' + to get all the clinical versions. Not supported if multiple + clinical ids are provided. :param bool deleted: Boolean to retrieve deleted entries. """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py index 0c695ca0a0e..0d7dc0ff159 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py @@ -104,6 +104,8 @@ def update_notes(self, id, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. + :param str tags_action: Action to be performed if the array of tags is + being updated. Allowed values: ['ADD SET REMOVE'] :param bool include_result: Flag indicating to include the created or updated document result in the response. """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 93e70823d52..b730dc06d5d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -270,6 +270,8 @@ def update_notes(self, study, id, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. + :param str tags_action: Action to be performed if the array of tags is + being updated. Allowed values: ['ADD SET REMOVE'] :param bool include_result: Flag indicating to include the created or updated document result in the response. """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index 98dfc7d4c25..f764a18198d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -628,6 +628,19 @@ def delete_variant_secondary_index(self, **options): return self._delete(category='operation', resource='delete', subcategory='variant/secondaryIndex', **options) + def setup_variant(self, data=None, **options): + """ + Execute Variant Setup to allow using the variant engine. This setup is + necessary before starting any variant operation. + PATH: /{apiVersion}/operation/variant/setup + + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param dict data: Variant setup params. + """ + + return self._post(category='operation', resource='setup', subcategory='variant', data=data, **options) + def delete_variant_stats(self, data=None, **options): """ Deletes the VariantStats of a cohort/s from the database. diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 835f89a61a6..dbde894f78f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -45,7 +45,7 @@ public class FieldConstants { public static final String ORGANIZATION_ADMINS_DESCRIPTION = "Administrative users of the organization."; public static final String ORGANIZATION_PROJECTS_DESCRIPTION = "Projects the organization holds."; public static final String ORGANIZATION_NOTES_DESCRIPTION = "Notes of organization scope."; -// public static final String ORGANIZATION_AUTHENTICATION_ORIGINS_DESCRIPTION = "Authentication origins used by the organization. This " + // public static final String ORGANIZATION_AUTHENTICATION_ORIGINS_DESCRIPTION = "Authentication origins used by the organization. This " // + "contains all the configuration necessary to be able to communicate with the external authentication origins."; public static final String ORGANIZATION_CONFIGURATION_DESCRIPTION = "Organization configuration information."; public static final String ORGANIZATION_INTERNAL_DESCRIPTION = "Organization internal information."; @@ -441,9 +441,16 @@ public class FieldConstants { public static final String USER_ACCOUNT = "User account."; //Account - public static final String ACCOUNT_TYPE = "User account type can have the values GUEST, FULL and ADMINISTRATOR."; - public static final String ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; - public static final String ACCOUNT_AUTHENTICATION = "How the account is authenticated"; + public static final String INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; + public static final String INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION = "Number of consecutive failed attempts. When the user logs" + + " in successfully, this field is automatically reset to 0."; + public static final String INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION = "Object containing the last time the password was changed and the" + + " expiration date."; + public static final String INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION = "Date the user's password was last changed. This " + + "field will be null if the user account origin is different from OpenCGA."; + public static final String INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION = "Date the user's password expires. This field will" + + " be null if the user account origin is different from OpenCGA and the passwordExpiration feature is disabled."; + public static final String INTERNAL_ACCOUNT_AUTHENTICATION = "How the account is authenticated"; public static final String USER_QUOTA = "User quota"; public static final String USER_PROJECTS = "A List with related projects."; public static final String USER_SHARED_PROJECTS = "A List with shared projects."; @@ -504,7 +511,7 @@ public class FieldConstants { public static final String HRDETECT_CNV_QUERY_DESCRIPTION = "CNV query"; public static final String HRDETECT_INDEL_QUERY_DESCRIPTION = "INDEL query"; public static final String HRDETECT_SNV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV3 input for" - + " HRDetect."; + + " HRDetect."; public static final String HRDETECT_SNV8_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV8 input for" + " HRDetect."; public static final String HRDETECT_SV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SV3 input for" @@ -541,5 +548,10 @@ public class FieldConstants { public static final String ALIGNMENT_QC_OVERWRITE_DESCRIPTION = "To overwrite the QC metrics already computed."; // Exomiser + public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID."; + public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID."; public static final String EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION = "Clinical analysis type: SINGLE or FAMILY."; + public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version in the format X.Y where X is the major version and Y the" + + " minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration" + + " file to view all installed Exomiser versions and identify the default version."; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 0c5c0d27c4b..d3739f43ce5 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -82,7 +82,7 @@ public class ParamConstants { public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase"; public static final String CELLBASE_VERSION = "v5.8"; public static final String CELLBASE_DATA_RELEASE_GRCH38 = "7"; - public static final String CELLBASE_APIKEY = ""; + public static final String CELLBASE_APIKEY = null; public static final String POP_FREQ_1000G_CB_V4 = "1kG_phase3"; public static final String POP_FREQ_1000G_CB_V5 = "1000G"; @@ -437,6 +437,9 @@ public class ParamConstants { public static final String CLINICAL_RELEASE_PARAM = RELEASE_PARAM; public static final String CLINICAL_STATUS_PARAM = STATUS_PARAM; public static final String CLINICAL_INTERNAL_STATUS_PARAM = INTERNAL_STATUS_PARAM; + public static final String CLINICAL_VERSION_PARAM = "version"; + public static final String CLINICAL_VERSION_DESCRIPTION = "Comma separated list of clinical versions. 'all' to get all the clinical" + + " versions. Not supported if multiple clinical ids are provided"; public static final String CLINICAL_TYPE_DESCRIPTION = "Clinical Analysis type"; public static final String CLINICAL_DISORDER_DESCRIPTION = "Clinical Analysis disorder" + REGEX_SUPPORT; public static final String CLINICAL_FILES_DESCRIPTION = "Clinical Analysis files"; @@ -467,6 +470,7 @@ public class ParamConstants { + "Interpretation object is passed."; public static final String INTERPRETATION_ID_PARAM = "id"; public static final String INTERPRETATION_UUID_PARAM = "uuid"; + public static final String INTERPRETATION_NAME_PARAM = "name"; public static final String INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM = "clinicalAnalysisId"; public static final String INTERPRETATION_ANALYST_ID_PARAM = "analystId"; public static final String INTERPRETATION_METHOD_NAME_PARAM = "methodName"; @@ -1519,6 +1523,7 @@ public class ParamConstants { public static final String CLINICAL_ANALYSES_DESCRIPTION = "Comma separated list of clinical analysis IDs or names" + UP_TO_100; public static final String INTERPRETATION_ID_DESCRIPTION = "Comma separated list of Interpretation IDs" + UP_TO_100 + REGEX_SUPPORT; public static final String INTERPRETATION_UUID_DESCRIPTION = "Comma separated list of Interpretation UUIDs" + UP_TO_100; + public static final String INTERPRETATION_NAME_DESCRIPTION = "Comma separated list of Interpretation names" + UP_TO_100; public static final String INTERPRETATION_DESCRIPTION = "Comma separated list of clinical interpretation IDs " + UP_TO_100; public static final String PANEL_ID_DESCRIPTION = "Comma separated list of panel IDs " + UP_TO_100 + REGEX_SUPPORT; public static final String PANEL_UUID_DESCRIPTION = "Comma separated list of panel UUIDs " + UP_TO_100; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index 4dd5543e9b8..e6b7f7353c8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.Callable; import java.util.stream.Collectors; @@ -149,7 +150,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException String inputVersion = getVersion(); CellBaseDataResponse species; try { - species = cellBaseClient.getMetaClient().species(); + species = retryMetaSpecies(); } catch (RuntimeException e) { throw new IllegalArgumentException("Unable to access cellbase url '" + getURL() + "', version '" + inputVersion + "'", e); } @@ -158,7 +159,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException // Version might be missing the starting "v" cellBaseConfiguration.setVersion("v" + cellBaseConfiguration.getVersion()); cellBaseClient = newCellBaseClient(cellBaseConfiguration, getSpecies(), getAssembly()); - species = cellBaseClient.getMetaClient().species(); + species = retryMetaSpecies(); } } if (species == null || species.firstResult() == null) { @@ -308,7 +309,7 @@ private static String majorMinor(String version) { public String getVersionFromServer() throws IOException { if (serverVersion == null) { synchronized (this) { - ObjectMap result = retryMetaAbout(3); + ObjectMap result = retryMetaAbout(); if (result == null) { throw new IOException("Unable to get version from server for cellbase " + toString()); } @@ -322,12 +323,43 @@ public String getVersionFromServer() throws IOException { return serverVersion; } - private ObjectMap retryMetaAbout(int retries) throws IOException { - ObjectMap result = cellBaseClient.getMetaClient().about().firstResult(); - if (result == null && retries > 0) { - // Retry - logger.warn("Unable to get version from server for cellbase " + toString() + ". Retrying..."); - result = retryMetaAbout(retries - 1); + private ObjectMap retryMetaAbout() throws IOException { + return retry(3, () -> cellBaseClient.getMetaClient().about().firstResult()); + } + + private CellBaseDataResponse retryMetaSpecies() throws IOException { + return retry(3, () -> cellBaseClient.getMetaClient().species()); + } + + private T retry(int retries, Callable function) throws IOException { + if (retries <= 0) { + return null; + } + T result = null; + Exception e = null; + try { + result = function.call(); + } catch (Exception e1) { + e = e1; + } + if (result == null) { + try { + // Retry + logger.warn("Unable to get reach cellbase " + toString() + ". Retrying..."); + result = retry(retries - 1, function); + } catch (Exception e1) { + if (e == null) { + e = e1; + } else { + e.addSuppressed(e1); + } + if (e instanceof IOException) { + throw (IOException) e; + } else { + throw new IOException("Error reading from cellbase " + toString(), e); + } + } + } return result; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java index a14652c67cd..e37374e76ea 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java @@ -375,6 +375,17 @@ public static String humanReadableByteCount(long bytes, boolean si) { * @return number of bytes */ public static long fromHumanReadableToByte(String value) { + return fromHumanReadableToByte(value, false); + } + + /** + * Get Bytes numbers from a human-readable string + * + * @param value Human-readable value + * @param assumeBinary Use Binary Units (power of 2) + * @return number of bytes + */ + public static long fromHumanReadableToByte(String value, boolean assumeBinary) { if (value.endsWith("B")) { value = value.substring(0, value.length() - 1); } @@ -385,8 +396,11 @@ public static long fromHumanReadableToByte(String value) { } else { si = true; } + if (assumeBinary) { + si = false; + } int unit = si ? 1000 : 1024; - int exp = (si ? "kMGTPE" : "KMGTPE").indexOf(value.charAt(value.length() - 1)) + 1; + int exp = "KMGTPE".indexOf(value.toUpperCase().charAt(value.length() - 1)) + 1; if (exp > 0) { value = value.substring(0, value.length() - 1); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java index 17255ae3255..91fb20b8455 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.common; +import org.passay.CharacterData; import org.passay.*; import java.util.ArrayList; @@ -7,6 +8,12 @@ public class PasswordUtils { + public static final int MIN_STRONG_PASSWORD_LENGTH = 8; + public static final int DEFAULT_PASSWORD_LENGTH = 10; + public static final int DEFAULT_SALT_LENGTH = 32; + public static final String PASSWORD_REQUIREMENT = "Password must contain at least " + MIN_STRONG_PASSWORD_LENGTH + + " characters, including at least one uppercase letter, one lowercase letter, one digit and one special character."; + private final static CharacterRule SPECIAL_CHARACTER_RULE = new CharacterRule(new CharacterData() { @Override public String getErrorCode() { @@ -20,7 +27,11 @@ public String getCharacters() { }); public static String getStrongRandomPassword() { - return getStrongRandomPassword(10); + return getStrongRandomPassword(DEFAULT_PASSWORD_LENGTH); + } + + public static String getStrongRandomSalt() { + return getStrongRandomPassword(DEFAULT_SALT_LENGTH); } public static String getStrongRandomPassword(int length) { @@ -33,14 +44,10 @@ public static String getStrongRandomPassword(int length) { } public static boolean isStrongPassword(String password) { - return isStrongPassword(password, 8); - } - - public static boolean isStrongPassword(String password, int minLength) { List rules = new ArrayList<>(); //Rule 1: Password length should be in between - //minLength and 100 characters - rules.add(new LengthRule(minLength, 100)); + //MIN_STRONG_PASSWORD_LENGTH and 100 characters + rules.add(new LengthRule(MIN_STRONG_PASSWORD_LENGTH, 100)); //Rule 2: No whitespace allowed rules.add(new WhitespaceRule()); //Rule 3.a: At least one Upper-case character diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java index da6aa39f1f6..c0574f0afdf 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java @@ -135,6 +135,13 @@ public static Date add1YeartoDate(Date date) { return new Date(cal.getTimeInMillis()); } + public static Date addDaysToCurrentDate(int days) { + Calendar cal = new GregorianCalendar(); + cal.setTime(new Date()); + cal.add(Calendar.DAY_OF_YEAR, days); + return new Date(cal.getTimeInMillis()); + } + public static Date toDate(String dateStr) { Date date = null; try { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java new file mode 100644 index 00000000000..f467a93d37d --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.core.config; + +public class AccountConfiguration { + + private int maxLoginAttempts; + private int passwordExpirationDays; + + public AccountConfiguration() { + } + + public AccountConfiguration(int maxLoginAttempts, int passwordExpirationDays) { + this.maxLoginAttempts = maxLoginAttempts; + this.passwordExpirationDays = passwordExpirationDays; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("AccountConfiguration{"); + sb.append("maxLoginAttempts=").append(maxLoginAttempts); + sb.append(", passwordExpirationDays=").append(passwordExpirationDays); + sb.append('}'); + return sb.toString(); + } + + public int getMaxLoginAttempts() { + return maxLoginAttempts; + } + + public AccountConfiguration setMaxLoginAttempts(int maxLoginAttempts) { + this.maxLoginAttempts = maxLoginAttempts; + return this; + } + + public int getPasswordExpirationDays() { + return passwordExpirationDays; + } + + public AccountConfiguration setPasswordExpirationDays(int passwordExpirationDays) { + this.passwordExpirationDays = passwordExpirationDays; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 6eb2beecb49..9f1c7b32683 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -17,13 +17,19 @@ package org.opencb.opencga.core.config; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class Analysis { private List packages; private String scratchDir; + private String resourceUrl; + + private String opencgaExtTools; + private List tools; private Execution execution; @@ -31,6 +37,7 @@ public class Analysis { public Analysis() { packages = new ArrayList<>(); + tools = new ArrayList<>(); execution = new Execution(); frameworks = new ArrayList<>(); } @@ -53,6 +60,33 @@ public Analysis setScratchDir(String scratchDir) { return this; } + public String getResourceUrl() { + return resourceUrl; + } + + public Analysis setResourceUrl(String resourceUrl) { + this.resourceUrl = resourceUrl; + return this; + } + + public String getOpencgaExtTools() { + return opencgaExtTools; + } + + public Analysis setOpencgaExtTools(String opencgaExtTools) { + this.opencgaExtTools = opencgaExtTools; + return this; + } + + public List getTools() { + return tools; + } + + public Analysis setTools(List tools) { + this.tools = tools; + return this; + } + public Execution getExecution() { return execution; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java new file mode 100644 index 00000000000..3744915b6b9 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java @@ -0,0 +1,110 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.config; + +import java.util.HashMap; +import java.util.Map; + +public class AnalysisTool { + + private String id; + private String version; + private boolean defaultVersion; + private String dockerId; + private String params; + private Map resources; + + public AnalysisTool() { + resources = new HashMap<>(); + } + + public AnalysisTool(String id, String version, boolean defaultVersion, String dockerId, String params, Map resources) { + this.id = id; + this.version = version; + this.defaultVersion = defaultVersion; + this.dockerId = dockerId; + this.params = params; + this.resources = resources; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("AnalysisTool{"); + sb.append("id='").append(id).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append(", defaultVersion=").append(defaultVersion); + sb.append(", dockerId='").append(dockerId).append('\''); + sb.append(", params='").append(params).append('\''); + sb.append(", resources=").append(resources); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public AnalysisTool setId(String id) { + this.id = id; + return this; + } + + public String getVersion() { + return version; + } + + public AnalysisTool setVersion(String version) { + this.version = version; + return this; + } + + public boolean isDefaultVersion() { + return defaultVersion; + } + + public AnalysisTool setDefaultVersion(boolean defaultVersion) { + this.defaultVersion = defaultVersion; + return this; + } + + public String getDockerId() { + return dockerId; + } + + public AnalysisTool setDockerId(String dockerId) { + this.dockerId = dockerId; + return this; + } + + public String getParams() { + return params; + } + + public AnalysisTool setParams(String params) { + this.params = params; + return this; + } + + public Map getResources() { + return resources; + } + + public AnalysisTool setResources(Map resources) { + this.resources = resources; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index 600def109ff..b7b869c728c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -44,7 +44,7 @@ public class Configuration { private String workspace; private String jobDir; - private int maxLoginAttempts; + private AccountConfiguration account; private Monitor monitor; private HealthCheck healthCheck; @@ -79,6 +79,7 @@ public Configuration() { analysis = new Analysis(); panel = new Panel(); server = new ServerConfiguration(); + account = new AccountConfiguration(); } public void serialize(OutputStream configurationOututStream) throws IOException { @@ -113,7 +114,6 @@ public static Configuration load(InputStream configurationInputStream, String fo } catch (IOException e) { throw new IOException("Configuration file could not be parsed: " + e.getMessage(), e); } - addDefaultValueIfMissing(configuration); // We must always overwrite configuration with environment parameters @@ -122,8 +122,15 @@ public static Configuration load(InputStream configurationInputStream, String fo } private static void addDefaultValueIfMissing(Configuration configuration) { - if (configuration.getMaxLoginAttempts() <= 0) { - configuration.setMaxLoginAttempts(5); + if (configuration.getAccount() == null) { + configuration.setAccount(new AccountConfiguration()); + } + if (configuration.getAccount().getMaxLoginAttempts() <= 0) { + configuration.getAccount().setMaxLoginAttempts(5); + } + if (configuration.getAccount().getPasswordExpirationDays() < 0) { + // Disable password expiration by default + configuration.getAccount().setPasswordExpirationDays(0); } } @@ -147,7 +154,8 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio configuration.getMonitor().setPort(Integer.parseInt(value)); break; case "OPENCGA.MAX_LOGIN_ATTEMPTS": - configuration.setMaxLoginAttempts(Integer.parseInt(value)); + case "OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS": + configuration.getAccount().setMaxLoginAttempts(Integer.parseInt(value)); break; case "OPENCGA_EXECUTION_MODE": case "OPENCGA_EXECUTION_ID": @@ -203,6 +211,16 @@ public static void reportUnusedField(String field, Object value) { } } + public static void reportMovedField(String previousField, String newField, Object value) { + // Report only if the value is not null and not an empty string + if (value != null && !(value instanceof String && ((String) value).isEmpty())) { + if (reportedFields.add(previousField)) { + // Only log the first time a field is found + logger.warn("Option '{}' with value '{}' was moved to '{}'.", previousField, value, newField); + } + } + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("Configuration{"); @@ -211,7 +229,7 @@ public String toString() { sb.append(", databasePrefix='").append(databasePrefix).append('\''); sb.append(", workspace='").append(workspace).append('\''); sb.append(", jobDir='").append(jobDir).append('\''); - sb.append(", maxLoginAttempts=").append(maxLoginAttempts); + sb.append(", account=").append(account); sb.append(", monitor=").append(monitor); sb.append(", healthCheck=").append(healthCheck); sb.append(", audit=").append(audit); @@ -292,12 +310,24 @@ public Configuration setJobDir(String jobDir) { return this; } + public AccountConfiguration getAccount() { + return account; + } + + public Configuration setAccount(AccountConfiguration account) { + this.account = account; + return this; + } + + @Deprecated public int getMaxLoginAttempts() { - return maxLoginAttempts; + return account.getMaxLoginAttempts(); } + @Deprecated public Configuration setMaxLoginAttempts(int maxLoginAttempts) { - this.maxLoginAttempts = maxLoginAttempts; + reportMovedField("configuration.yml#maxLoginAttempts", "configuration.yml#account.maxLoginAttempts", maxLoginAttempts); + account.setMaxLoginAttempts(maxLoginAttempts); return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java index 61bc01c644f..d2d48863d31 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java @@ -22,7 +22,6 @@ import org.opencb.biodata.models.clinical.ClinicalAudit; import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; @@ -100,9 +99,9 @@ public class ClinicalAnalysis extends Annotable { description = FieldConstants.CLINICAL_ANALYSIS_PANELS) private List panels; - @DataField(id = "panelLock", indexed = true, + @DataField(id = "panelLocked", indexed = true, description = FieldConstants.CLINICAL_ANALYSIS_PANEL_LOCK) - private boolean panelLock; + private boolean panelLocked; @DataField(id = "locked", indexed = true, description = FieldConstants.CLINICAL_ANALYSIS_LOCKED) @@ -179,6 +178,14 @@ public class ClinicalAnalysis extends Annotable { description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; + /** + * Generic: Autoincremental version assigned to the registered entry. + * + * @apiNote Immutable + */ + @DataField(id = "version", managed = true, indexed = true, description = FieldConstants.GENERIC_VERSION_DESCRIPTION) + private int version; + @DataField(id = "qualityControl", indexed = true, description = FieldConstants.GENERIC_QUALITY_CONTROL) private ClinicalAnalysisQualityControl qualityControl; @@ -217,19 +224,19 @@ public class ClinicalAnalysis extends Annotable { */ @DataField(id = "status", indexed = true, uncommentedClasses = {"Status"}, description = FieldConstants.GENERIC_STATUS_DESCRIPTION) - private Status status; + private ClinicalStatus status; public ClinicalAnalysis() { } public ClinicalAnalysis(String id, String description, Type type, Disorder disorder, List files, Individual proband, - Family family, List panels, boolean panelLock, boolean locked, Interpretation interpretation, + Family family, List panels, boolean panelLocked, boolean locked, Interpretation interpretation, List secondaryInterpretations, ClinicalConsentAnnotation consent, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, ClinicalPriorityAnnotation priority, List flags, String creationDate, String modificationDate, - String dueDate, int release, List comments, ClinicalAnalysisQualityControl qualityControl, - List audit, ClinicalAnalysisInternal internal, List annotationSets, - Map attributes, Status status) { + String dueDate, int release, int version, List comments, + ClinicalAnalysisQualityControl qualityControl, List audit, ClinicalAnalysisInternal internal, + List annotationSets, Map attributes, ClinicalStatus status) { this.id = id; this.description = description; this.type = type; @@ -238,7 +245,7 @@ public ClinicalAnalysis(String id, String description, Type type, Disorder disor this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.locked = locked; this.interpretation = interpretation; this.secondaryInterpretations = secondaryInterpretations; @@ -254,6 +261,7 @@ public ClinicalAnalysis(String id, String description, Type type, Disorder disor this.dueDate = dueDate; this.qualityControl = qualityControl; this.release = release; + this.version = version; this.comments = comments; this.audit = audit; this.internal = internal; @@ -274,7 +282,7 @@ public String toString() { sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", locked=").append(locked); sb.append(", interpretation=").append(interpretation); sb.append(", secondaryInterpretations=").append(secondaryInterpretations); @@ -289,6 +297,7 @@ public String toString() { sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", dueDate='").append(dueDate).append('\''); sb.append(", release=").append(release); + sb.append(", version=").append(version); sb.append(", qualityControl=").append(qualityControl); sb.append(", comments=").append(comments); sb.append(", audit=").append(audit); @@ -383,12 +392,23 @@ public ClinicalAnalysis setPanels(List panels) { return this; } + @Deprecated + @JsonIgnore public boolean isPanelLock() { - return panelLock; + return isPanelLocked(); } + @Deprecated public ClinicalAnalysis setPanelLock(boolean panelLock) { - this.panelLock = panelLock; + return setPanelLocked(panelLock); + } + + public boolean isPanelLocked() { + return panelLocked; + } + + public ClinicalAnalysis setPanelLocked(boolean panelLocked) { + this.panelLocked = panelLocked; return this; } @@ -532,6 +552,15 @@ public ClinicalAnalysis setRelease(int release) { return this; } + public int getVersion() { + return version; + } + + public ClinicalAnalysis setVersion(int version) { + this.version = version; + return this; + } + public List getComments() { return comments; } @@ -583,11 +612,11 @@ public ClinicalAnalysis setAttributes(Map attributes) { return this; } - public Status getStatus() { + public ClinicalStatus getStatus() { return status; } - public ClinicalAnalysis setStatus(Status status) { + public ClinicalAnalysis setStatus(ClinicalStatus status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java index 7ffbf4b4a9a..157b9f0b12e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java @@ -46,7 +46,7 @@ public class ClinicalAnalysisCreateParams { private FamilyParam family; private List panels; - private Boolean panelLock; + private Boolean panelLocked; @Deprecated private ClinicalAnalystParam analyst; @@ -75,7 +75,7 @@ public ClinicalAnalysisCreateParams() { public ClinicalAnalysisCreateParams(String id, String description, ClinicalAnalysis.Type type, DisorderReferenceParam disorder, List files, ProbandParam proband, FamilyParam family, - List panels, Boolean panelLock, List analysts, + List panels, Boolean panelLocked, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, InterpretationCreateParams interpretation, ClinicalConsentAnnotationParam consent, String creationDate, String modificationDate, String dueDate, List comments, @@ -90,7 +90,7 @@ public ClinicalAnalysisCreateParams(String id, String description, ClinicalAnaly this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.report = report; this.request = request; this.responsible = responsible; @@ -120,7 +120,7 @@ public static ClinicalAnalysisCreateParams of(ClinicalAnalysis clinicalAnalysis) clinicalAnalysis.getPanels() != null ? clinicalAnalysis.getPanels().stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList()) : null, - clinicalAnalysis.isPanelLock(), + clinicalAnalysis.isPanelLocked(), clinicalAnalysis.getAnalysts() != null ? clinicalAnalysis.getAnalysts().stream().map(ClinicalAnalystParam::of).collect(Collectors.toList()) : null, @@ -155,7 +155,7 @@ public String toString() { sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", analysts=").append(analysts); sb.append(", report=").append(report); sb.append(", request=").append(request); @@ -231,14 +231,14 @@ public ClinicalAnalysis toClinicalAnalysis() { } return new ClinicalAnalysis(id, description, type, disorder != null ? disorder.toDisorder() : null, caFiles, individual, f, - diseasePanelList, panelLock != null ? panelLock : false, false, primaryInterpretation, new LinkedList<>(), + diseasePanelList, panelLocked != null ? panelLocked : false, false, primaryInterpretation, new LinkedList<>(), consent != null ? consent.toClinicalConsentAnnotation() : null, clinicalAnalystList, report, request, responsible, priority != null ? priority.toClinicalPriorityAnnotation() : null, flags != null ? flags.stream().map(FlagValueParam::toFlagAnnotation).collect(Collectors.toList()) : null, creationDate, modificationDate, dueDate, - 1, + 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, new LinkedList<>(), null, - annotationSets, attributes, status != null ? status.toStatus() : null); + annotationSets, attributes, status != null ? status.toClinicalStatus() : null); } public String getId() { @@ -313,12 +313,12 @@ public ClinicalAnalysisCreateParams setPanels(List panels) return this; } - public Boolean getPanelLock() { - return panelLock; + public Boolean getPanelLocked() { + return panelLocked; } - public ClinicalAnalysisCreateParams setPanelLock(Boolean panelLock) { - this.panelLock = panelLock; + public ClinicalAnalysisCreateParams setPanelLocked(Boolean panelLocked) { + this.panelLocked = panelLocked; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java index a529d6d7d27..425654bcc55 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java @@ -18,21 +18,19 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; +import org.opencb.opencga.core.models.common.InternalStatus; public class ClinicalAnalysisInternal extends Internal { - private ClinicalAnalysisStatus status; - public ClinicalAnalysisInternal() { } - public ClinicalAnalysisInternal(String registrationDate, String modificationDate, ClinicalAnalysisStatus status) { - super(null, registrationDate, modificationDate); - this.status = status; + public ClinicalAnalysisInternal(String registrationDate, String modificationDate, InternalStatus status) { + super(status, registrationDate, modificationDate); } public static ClinicalAnalysisInternal init() { - return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new ClinicalAnalysisStatus()); + return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new InternalStatus()); } @Override @@ -45,11 +43,11 @@ public String toString() { return sb.toString(); } - public ClinicalAnalysisStatus getStatus() { + public InternalStatus getStatus() { return status; } - public ClinicalAnalysisInternal setStatus(ClinicalAnalysisStatus status) { + public ClinicalAnalysisInternal setStatus(InternalStatus status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java index e0cdfadd0c0..724f4ae04a5 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java @@ -9,7 +9,8 @@ public enum ClinicalAnalysisPermissions { DELETE(Arrays.asList(VIEW, WRITE)), VIEW_ANNOTATIONS(Collections.singletonList(VIEW)), WRITE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, VIEW)), - DELETE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, VIEW)); + DELETE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, VIEW)), + ADMIN(Arrays.asList(VIEW, WRITE, DELETE, VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, DELETE_ANNOTATIONS)); private final List implicitPermissions; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java deleted file mode 100644 index b898a7c21d5..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.core.models.clinical; - -import org.opencb.opencga.core.models.common.InternalStatus; - -import java.util.Arrays; -import java.util.List; - -public class ClinicalAnalysisStatus extends InternalStatus { - - public static final String INCOMPLETE = "INCOMPLETE"; - public static final String READY_FOR_VALIDATION = "READY_FOR_VALIDATION"; - public static final String READY_FOR_INTERPRETATION = "READY_FOR_INTERPRETATION"; - public static final String INTERPRETATION_IN_PROGRESS = "INTERPRETATION_IN_PROGRESS"; - // public static final String INTERPRETED = "INTERPRETED"; - public static final String READY_FOR_INTEPRETATION_REVIEW = "READY_FOR_INTEPRETATION_REVIEW"; - public static final String INTERPRETATION_REVIEW_IN_PROGRESS = "INTERPRETATION_REVIEW_IN_PROGRESS"; - public static final String READY_FOR_REPORT = "READY_FOR_REPORT"; - public static final String REPORT_IN_PROGRESS = "REPORT_IN_PROGRESS"; - public static final String DONE = "DONE"; - public static final String REVIEW_IN_PROGRESS = "REVIEW_IN_PROGRESS"; - public static final String CLOSED = "CLOSED"; - public static final String REJECTED = "REJECTED"; - - public static final List STATUS_LIST = Arrays.asList(INCOMPLETE, READY, DELETED, READY_FOR_VALIDATION, - READY_FOR_INTERPRETATION, INTERPRETATION_IN_PROGRESS, READY_FOR_INTEPRETATION_REVIEW, INTERPRETATION_REVIEW_IN_PROGRESS, - READY_FOR_REPORT, REPORT_IN_PROGRESS, DONE, REVIEW_IN_PROGRESS, CLOSED, REJECTED); - - public ClinicalAnalysisStatus(String status, String message) { - if (isValid(status)) { - init(status, message); - } else { - throw new IllegalArgumentException("Unknown status " + status); - } - } - - public ClinicalAnalysisStatus(String status) { - this(status, ""); - } - - public ClinicalAnalysisStatus() { - this(READY_FOR_INTERPRETATION, ""); - } - - public static boolean isValid(String status) { - if (InternalStatus.isValid(status)) { - return true; - } - - if (STATUS_LIST.contains(status)) { - return true; - } - return false; - } -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java index 020a44ead6a..e22feef43ae 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java @@ -44,7 +44,7 @@ public class ClinicalAnalysisUpdateParams { private List files; private List panels; - private Boolean panelLock; + private Boolean panelLocked; private ProbandParam proband; private FamilyParam family; @@ -77,7 +77,7 @@ public ClinicalAnalysisUpdateParams() { public ClinicalAnalysisUpdateParams(String id, String description, ClinicalAnalysis.Type type, DisorderReferenceParam disorder, List files, ProbandParam proband, FamilyParam family, - List panels, Boolean panelLock, Boolean locked, + List panels, Boolean panelLocked, Boolean locked, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, ClinicalAnalysisQualityControlUpdateParam qualityControl, ClinicalConsentAnnotationParam consent, String creationDate, String modificationDate, @@ -92,7 +92,7 @@ public ClinicalAnalysisUpdateParams(String id, String description, ClinicalAnaly this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.locked = locked; this.analysts = analysts; this.report = report; @@ -123,7 +123,7 @@ public ClinicalAnalysis toClinicalAnalysis() { proband != null ? proband.toIndividual() : null, family != null ? family.toFamily() : null, panels != null ? panels.stream().map(p -> new Panel().setId(p.getId())).collect(Collectors.toList()) : null, - panelLock != null ? panelLock : false, + panelLocked != null ? panelLocked : false, locked != null && locked, null, null, consent != null ? consent.toClinicalConsentAnnotation() : null, @@ -133,10 +133,10 @@ public ClinicalAnalysis toClinicalAnalysis() { report, request, responsible, priority != null ? priority.toClinicalPriorityAnnotation() : null, flags != null ? flags.stream().map(FlagValueParam::toFlagAnnotation).collect(Collectors.toList()) : null, creationDate, modificationDate, dueDate, - 1, + 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, null, null, annotationSets, attributes, - status != null ? status.toStatus() : null); + status != null ? status.toClinicalStatus() : null); } @Override @@ -148,7 +148,7 @@ public String toString() { sb.append(", disorder=").append(disorder); sb.append(", files=").append(files); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", locked=").append(locked); @@ -243,12 +243,22 @@ public ClinicalAnalysisUpdateParams setFamily(FamilyParam family) { return this; } + @Deprecated public Boolean getPanelLock() { - return panelLock; + return getPanelLocked(); } + @Deprecated public ClinicalAnalysisUpdateParams setPanelLock(Boolean panelLock) { - this.panelLock = panelLock; + return setPanelLocked(panelLock); + } + + public Boolean getPanelLocked() { + return panelLocked; + } + + public ClinicalAnalysisUpdateParams setPanelLocked(Boolean panelLocked) { + this.panelLocked = panelLocked; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java new file mode 100644 index 00000000000..baa92b6f22a --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java @@ -0,0 +1,77 @@ +package org.opencb.opencga.core.models.clinical; + +public class ClinicalStatus extends ClinicalStatusValue { + + private String author; + private String version; + private String commit; + private String date; + + public ClinicalStatus() { + } + + public ClinicalStatus(String id, String description, ClinicalStatusType type, String author, String version, String commit, + String date) { + super(id, description, type); + this.author = author; + this.version = version; + this.commit = commit; + this.date = date; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ClinicalStatus{"); + sb.append("id='").append(id).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", type=").append(type); + sb.append(", author='").append(author).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append(", commit='").append(commit).append('\''); + sb.append(", date='").append(date).append('\''); + sb.append('}'); + return sb.toString(); + } + + @Override + public ClinicalStatus setId(String id) { + super.setId(id); + return this; + } + + public String getVersion() { + return version; + } + + public ClinicalStatus setVersion(String version) { + this.version = version; + return this; + } + + public String getCommit() { + return commit; + } + + public ClinicalStatus setCommit(String commit) { + this.commit = commit; + return this; + } + + public String getAuthor() { + return author; + } + + public ClinicalStatus setAuthor(String author) { + this.author = author; + return this; + } + + public String getDate() { + return date; + } + + public ClinicalStatus setDate(String date) { + this.date = date; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java index 9efa33d4a33..68816055dde 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java @@ -2,15 +2,15 @@ public class ClinicalStatusValue { - private String id; - private String description; - private ClinicalStatusType type; + protected String id; + protected String description; + protected ClinicalStatusType type; public enum ClinicalStatusType { NOT_STARTED, - IN_PROGRESS, - CLOSED, - UNKNOWN + ACTIVE, + DONE, + CLOSED } public ClinicalStatusValue() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java index 0ebc7f92d22..deb5f28ba00 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java @@ -1,26 +1,32 @@ package org.opencb.opencga.core.models.clinical; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; - public class ExomiserInterpretationAnalysisParams extends ToolParams { - public static final String DESCRIPTION = "Exomizer interpretation analysis params"; + public static final String DESCRIPTION = "Exomiser interpretation analysis params"; + @DataField(id = "clinicalAnalysis", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, required = true) private String clinicalAnalysis; + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) + private String exomiserVersion; + public ExomiserInterpretationAnalysisParams() { } - public ExomiserInterpretationAnalysisParams(String clinicalAnalysis) { + public ExomiserInterpretationAnalysisParams(String clinicalAnalysis, String exomiserVersion) { this.clinicalAnalysis = clinicalAnalysis; - + this.exomiserVersion = exomiserVersion; } @Override public String toString() { - final StringBuilder sb = new StringBuilder("TieringInterpretationAnalysisParams{"); + final StringBuilder sb = new StringBuilder("ExomiserInterpretationAnalysisParams{"); sb.append("clinicalAnalysis='").append(clinicalAnalysis).append('\''); + sb.append(", exomiserVersion='").append(exomiserVersion).append('\''); sb.append('}'); return sb.toString(); } @@ -34,5 +40,12 @@ public ExomiserInterpretationAnalysisParams setClinicalAnalysis(String clinicalA return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + public ExomiserInterpretationAnalysisParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java index 3b836063439..33a5415422a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java @@ -4,14 +4,15 @@ import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; -import java.util.Map; - public class ExomiserWrapperParams extends ToolParams { public static final String DESCRIPTION = "Exomiser parameters"; - @DataField(id = "sample", description = FieldConstants.SAMPLE_ID_DESCRIPTION) + @DataField(id = "sample", description = FieldConstants.EXOMISER_SAMPLE_DESCRIPTION, required = true) private String sample; + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) + private String exomiserVersion; + @DataField(id = "clinicalAnalysisType", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION, defaultValue = "SINGLE") private String clinicalAnalysisType; @@ -22,9 +23,10 @@ public class ExomiserWrapperParams extends ToolParams { public ExomiserWrapperParams() { } - public ExomiserWrapperParams(String sample, String clinicalAnalysisType, String outdir) { + public ExomiserWrapperParams(String sample, String clinicalAnalysisType, String exomiserVersion, String outdir) { this.sample = sample; this.clinicalAnalysisType = clinicalAnalysisType; + this.exomiserVersion = exomiserVersion; this.outdir = outdir; } @@ -33,6 +35,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("ExomiserWrapperParams{"); sb.append("sample='").append(sample).append('\''); sb.append(", clinicalAnalysisType=").append(clinicalAnalysisType); + sb.append("exomiserVersion='").append(exomiserVersion).append('\''); sb.append(", outdir='").append(outdir).append('\''); sb.append('}'); return sb.toString(); @@ -56,6 +59,15 @@ public ExomiserWrapperParams setClinicalAnalysisType(String clinicalAnalysisType return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } + public String getOutdir() { return outdir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java index 273ce53a4c9..a6774d4f751 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java @@ -20,7 +20,6 @@ import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.models.IPrivateStudyUid; @@ -54,35 +53,56 @@ public class Interpretation extends org.opencb.biodata.models.clinical.interpret description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; + @DataField(id = "status", indexed = true, + description = FieldConstants.GENERIC_STATUS_DESCRIPTION) + private ClinicalStatus status; + public Interpretation() { super(); } - public Interpretation(String id, String description, String clinicalAnalysisId, ClinicalAnalyst analyst, + public Interpretation(String id, String name, String description, String clinicalAnalysisId, ClinicalAnalyst analyst, InterpretationMethod method, String creationDate, String modificationDate, boolean locked, List primaryFindings, List secondaryFindings, List panels, - List comments, Status status, Map attributes) { - super(id, "", description, clinicalAnalysisId, analyst, method, primaryFindings, secondaryFindings, comments, null, status, - creationDate, modificationDate, locked, 0, attributes); - + List comments, ClinicalStatus status, Map attributes) { + super(id, "", name, description, clinicalAnalysisId, analyst, method, primaryFindings, secondaryFindings, comments, null, locked, + creationDate, modificationDate, 0, attributes); + this.status = status; this.panels = panels; } public Interpretation(org.opencb.biodata.models.clinical.interpretation.Interpretation interpretation) { - this(interpretation.getId(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), interpretation.getAnalyst(), - interpretation.getMethod(), interpretation.getCreationDate(), interpretation.getModificationDate(), - interpretation.isLocked(), interpretation.getPrimaryFindings(), interpretation.getSecondaryFindings(), - Collections.emptyList(), interpretation.getComments(), interpretation.getStatus(), interpretation.getAttributes()); + this(interpretation.getId(), interpretation.getName(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), + interpretation.getAnalyst(), interpretation.getMethod(), interpretation.getCreationDate(), + interpretation.getModificationDate(), interpretation.isLocked(), interpretation.getPrimaryFindings(), + interpretation.getSecondaryFindings(), Collections.emptyList(), interpretation.getComments(), null, + interpretation.getAttributes()); } @Override public String toString() { final StringBuilder sb = new StringBuilder("Interpretation{"); - sb.append("studyUid=").append(studyUid); + sb.append("id='").append(id).append('\''); + sb.append(", uuid='").append(uuid).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", studyUid=").append(studyUid); sb.append(", uid=").append(uid); sb.append(", panels=").append(panels); sb.append(", internal=").append(internal); sb.append(", release=").append(release); + sb.append(", status=").append(status); + sb.append(", clinicalAnalysisId='").append(clinicalAnalysisId).append('\''); + sb.append(", analyst=").append(analyst); + sb.append(", method=").append(method); + sb.append(", primaryFindings=").append(primaryFindings); + sb.append(", secondaryFindings=").append(secondaryFindings); + sb.append(", comments=").append(comments); + sb.append(", stats=").append(stats); + sb.append(", locked=").append(locked); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", version=").append(version); sb.append('}'); return sb.toString(); } @@ -185,14 +205,17 @@ public Interpretation setComments(List comments) { } @Override - public Interpretation setStatus(Status status) { - super.setStatus(status); + public Interpretation setLocked(boolean locked) { + super.setLocked(locked); return this; } - @Override - public Interpretation setLocked(boolean locked) { - super.setLocked(locked); + public ClinicalStatus getStatus() { + return status; + } + + public Interpretation setStatus(ClinicalStatus status) { + this.status = status; return this; } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java index e35ccec46f8..9c710fa0493 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java @@ -26,13 +26,13 @@ import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; public class InterpretationCreateParams { + private String name; private String description; private String clinicalAnalysisId; private String creationDate; @@ -50,11 +50,12 @@ public class InterpretationCreateParams { public InterpretationCreateParams() { } - public InterpretationCreateParams(String description, String clinicalAnalysisId, String creationDate, String modificationDate, + public InterpretationCreateParams(String name, String description, String clinicalAnalysisId, String creationDate, String modificationDate, ClinicalAnalystParam analyst, InterpretationMethod method, List primaryFindings, List secondaryFindings, List panels, List comments, StatusParam status, Boolean locked, Map attributes) { + this.name = name; this.description = description; this.clinicalAnalysisId = clinicalAnalysisId; this.creationDate = creationDate; @@ -71,7 +72,7 @@ public InterpretationCreateParams(String description, String clinicalAnalysisId, } public static InterpretationCreateParams of(Interpretation interpretation) { - return new InterpretationCreateParams(interpretation.getDescription(), + return new InterpretationCreateParams(interpretation.getName(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), interpretation.getCreationDate(), interpretation.getModificationDate(), ClinicalAnalystParam.of(interpretation.getAnalyst()), interpretation.getMethod(), interpretation.getPrimaryFindings(), interpretation.getSecondaryFindings(), @@ -89,7 +90,8 @@ public static InterpretationCreateParams of(Interpretation interpretation) { @Override public String toString() { final StringBuilder sb = new StringBuilder("InterpretationCreateParams{"); - sb.append("description='").append(description).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); sb.append(", clinicalAnalysisId='").append(clinicalAnalysisId).append('\''); sb.append(", creationDate='").append(creationDate).append('\''); sb.append(", modificationDate='").append(modificationDate).append('\''); @@ -99,25 +101,34 @@ public String toString() { sb.append(", secondaryFindings=").append(secondaryFindings); sb.append(", panels=").append(panels); sb.append(", comments=").append(comments); - sb.append(", status=").append(status); sb.append(", locked=").append(locked); + sb.append(", status=").append(status); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); } public Interpretation toClinicalInterpretation() { - return new Interpretation(null, description, clinicalAnalysisId, analyst != null ? analyst.toClinicalAnalyst() : null, method, + return new Interpretation("", name, description, clinicalAnalysisId, analyst != null ? analyst.toClinicalAnalyst() : null, method, creationDate, modificationDate, locked != null ? locked : false, primaryFindings, secondaryFindings, panels != null ? panels.stream().map(p -> new Panel().setId(p.getId())).collect(Collectors.toList()) : null, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, - status != null ? status.toStatus() : null, attributes); + status != null ? status.toClinicalStatus() : null, attributes); } public ObjectMap toInterpretationObjectMap() throws JsonProcessingException { return new ObjectMap(getUpdateObjectMapper().writeValueAsString(this.toClinicalInterpretation())); } + public String getName() { + return name; + } + + public InterpretationCreateParams setName(String name) { + this.name = name; + return this; + } + public String getDescription() { return description; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java index c0bee8b89ce..3da416d1b28 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java @@ -31,6 +31,7 @@ public class InterpretationUpdateParams { + private String name; private String description; private ClinicalAnalystParam analyst; private InterpretationMethod method; @@ -47,11 +48,12 @@ public class InterpretationUpdateParams { public InterpretationUpdateParams() { } - public InterpretationUpdateParams(String description, ClinicalAnalystParam analyst, InterpretationMethod method, + public InterpretationUpdateParams(String name, String description, ClinicalAnalystParam analyst, InterpretationMethod method, String creationDate, String modificationDate, List primaryFindings, List secondaryFindings, List panels, List comments, StatusParam status, Boolean locked, Map attributes) { + this.name = name; this.description = description; this.analyst = analyst; this.method = method; @@ -74,7 +76,8 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { @Override public String toString() { final StringBuilder sb = new StringBuilder("InterpretationUpdateParams{"); - sb.append("description='").append(description).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); sb.append(", analyst=").append(analyst); sb.append(", method=").append(method); sb.append(", creationDate='").append(creationDate).append('\''); @@ -90,6 +93,15 @@ public String toString() { return sb.toString(); } + public String getName() { + return name; + } + + public InterpretationUpdateParams setName(String name) { + this.name = name; + return this; + } + public String getDescription() { return description; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java index 83836d0f7d4..99c35d2b029 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java @@ -19,7 +19,7 @@ public class IndexStatus extends InternalStatus { public IndexStatus(String status, String message) { if (isValid(status)) { - init(status, status, message); + init(status, message); } else { throw new IllegalArgumentException("Unknown status " + status); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java index a934efdc3ee..a1d28c6914b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java @@ -57,8 +57,8 @@ public InternalStatus(String id, String description) { } } - public InternalStatus(String id, String name, String description, String date, String version, String commit) { - super(id, name, description, date); + public InternalStatus(String id, String description, String date, String version, String commit) { + super(id, description, date); if (!isValid(id)) { throw new IllegalArgumentException("Unknown status id '" + id + "'"); } @@ -112,12 +112,7 @@ private static void fillPositiveNegativeList(String[] statusList, List p } protected void init(String statusId, String description) { - init(statusId, statusId, description); - } - - protected void init(String statusId, String statusName, String description) { super.id = statusId; - super.name = statusName; super.description = description; super.date = TimeUtils.getTime(); this.version = GitRepositoryState.getInstance().getBuildVersion(); @@ -150,7 +145,6 @@ public String toString() { sb.append("version='").append(version).append('\''); sb.append(", commit='").append(commit).append('\''); sb.append(", id='").append(id).append('\''); - sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); sb.append(", date='").append(date).append('\''); sb.append('}'); @@ -184,15 +178,6 @@ public InternalStatus setId(String id) { return this; } - public String getName() { - return StringUtils.isNotEmpty(name) ? name : id; - } - - public InternalStatus setName(String name) { - this.name = name; - return this; - } - public String getDate() { return date; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java index 806ef09be4d..d826354369e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java @@ -2,6 +2,7 @@ import org.opencb.biodata.models.common.Status; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.clinical.ClinicalStatus; public class StatusParam { @@ -18,10 +19,18 @@ public static StatusParam of(Status status) { return status != null ? new StatusParam(status.getId()) : null; } + public static StatusParam of(ClinicalStatus status) { + return status != null ? new StatusParam(status.getId()) : null; + } + public Status toStatus() { return new Status(id, "", "", TimeUtils.getTime()); } + public ClinicalStatus toClinicalStatus() { + return new ClinicalStatus(id, "", null, "", "", "", ""); + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("StatusParam{"); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java index a6592c369bf..1e44390761c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java @@ -24,7 +24,7 @@ public class VariantIndexStatus extends IndexStatus { public VariantIndexStatus(String status, String message) { if (isValid(status)) { - init(status, status, message); + init(status, message); } else { throw new IllegalArgumentException("Unknown status " + status); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java index 85ba64ce78c..2234cb489e7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java @@ -16,6 +16,7 @@ package org.opencb.opencga.core.models.operations.variant; +import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.tools.ToolParams; import java.util.List; @@ -32,8 +33,13 @@ public VariantFileDeleteParams(List file, boolean resume) { this.resume = resume; } + @DataField(description = "List of file ids to delete. Use 'all' to remove the whole study", required = true) private List file; + + @DataField(description = "Resume failed delete operation.", defaultValue = "false") private boolean resume; + + @DataField(description = "Force delete operation. This would allow deleting partially loaded files.", defaultValue = "false") private boolean force; public List getFile() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java index f45e545a292..10b42230abe 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java @@ -107,8 +107,7 @@ public class Sample extends Annotable { private int release; /** - * Generic: Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable - * versioning, users must set the `incVersion` flag from the /update web service when updating the document. + * Generic: Autoincremental version assigned to the registered entry. * * @apiNote Immutable */ diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java index b7dba9c36de..75f627af7f3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java @@ -109,7 +109,10 @@ public enum Permissions { WRITE_CLINICAL_ANNOTATIONS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, VIEW_CLINICAL_ANNOTATIONS), ClinicalAnalysisPermissions.WRITE_ANNOTATIONS.name(), CLINICAL_ANALYSIS), DELETE_CLINICAL_ANNOTATIONS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, VIEW_CLINICAL_ANNOTATIONS, WRITE_CLINICAL_ANNOTATIONS), - ClinicalAnalysisPermissions.DELETE_ANNOTATIONS.name(), CLINICAL_ANALYSIS); + ClinicalAnalysisPermissions.DELETE_ANNOTATIONS.name(), CLINICAL_ANALYSIS), + ADMIN_CLINICAL_ANALYSIS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, WRITE_CLINICAL_ANALYSIS, DELETE_CLINICAL_ANALYSIS, + VIEW_CLINICAL_ANNOTATIONS, WRITE_CLINICAL_ANNOTATIONS, DELETE_CLINICAL_ANNOTATIONS), + ClinicalAnalysisPermissions.ADMIN.name(), CLINICAL_ANALYSIS); private final static Map map; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java index 3bd986b2fb4..a0a47596cd1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.models.study; +import org.opencb.commons.annotations.DataField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; @@ -8,6 +9,9 @@ public class StudyVariantEngineConfiguration { private ObjectMap options; private SampleIndexConfiguration sampleIndex; + @DataField(description = "Variant setup run", since = "3.2.0") + private VariantSetupResult setup; + public StudyVariantEngineConfiguration() { } @@ -34,11 +38,21 @@ public StudyVariantEngineConfiguration setSampleIndex(SampleIndexConfiguration s return this; } + public VariantSetupResult getSetup() { + return setup; + } + + public StudyVariantEngineConfiguration setSetup(VariantSetupResult setup) { + this.setup = setup; + return this; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("StudyVariantEngineConfiguration{"); - // sb.append("options=").append(options != null ? options.toJson() : ""); + sb.append("options=").append(options != null ? options.toJson() : ""); sb.append(", sampleIndex=").append(sampleIndex); + sb.append(", setup=").append(setup); sb.append('}'); return sb.toString(); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java new file mode 100644 index 00000000000..a1127f873f8 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java @@ -0,0 +1,74 @@ +package org.opencb.opencga.core.models.study; + +import org.opencb.commons.annotations.DataField; +import org.opencb.commons.datastore.core.ObjectMap; + +public class VariantSetupResult { + + @DataField(description = "User ID that started the setup run") + private String userId; + @DataField(description = "Date when the variant setup was executed") + private String date; + @DataField(description = "Variant setup status") + private Status status; + @DataField(description = "Input params for the variant setup") + private ObjectMap params; + + @DataField(description = "Generated variant storage configuration options given the input params.") + private ObjectMap options; + + public enum Status { + READY, + NOT_READY + } + + public VariantSetupResult() { + } + + public String getUserId() { + return userId; + } + + public VariantSetupResult setUserId(String userId) { + this.userId = userId; + return this; + } + + public String getDate() { + return date; + } + + public VariantSetupResult setDate(String date) { + this.date = date; + return this; + } + + public Status getStatus() { + return status; + } + + public VariantSetupResult setStatus(Status status) { + this.status = status; + return this; + } + + public ObjectMap getParams() { + return params; + } + + public VariantSetupResult setParams(ObjectMap params) { + this.params = params; + return this; + } + + public ObjectMap getOptions() { + return options; + } + + public VariantSetupResult setOptions(ObjectMap options) { + this.options = options; + return this; + } + + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java index 97d09661407..5f71fe0c6b9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java @@ -1,26 +1,27 @@ package org.opencb.opencga.core.models.study.configuration; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.common.FlagValue; import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; +import org.opencb.opencga.core.models.common.FlagValue; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; public class ClinicalAnalysisStudyConfiguration { - private Map> status; + private List status; private InterpretationStudyConfiguration interpretation; private List priorities; - private Map> flags; + private List flags; private ClinicalConsentConfiguration consent; public ClinicalAnalysisStudyConfiguration() { } - public ClinicalAnalysisStudyConfiguration(Map> status, - InterpretationStudyConfiguration interpretation, List priorities, - Map> flags, ClinicalConsentConfiguration consent) { + public ClinicalAnalysisStudyConfiguration(List status, InterpretationStudyConfiguration interpretation, + List priorities, List flags, + ClinicalConsentConfiguration consent) { this.status = status; this.interpretation = interpretation; this.priorities = priorities; @@ -29,20 +30,21 @@ public ClinicalAnalysisStudyConfiguration(Map> status = new HashMap<>(); - Map> interpretationStatus = new HashMap<>(); + List clinicalStatusValueList = new ArrayList<>(4); + List interpretationStatusList = new ArrayList<>(3); List priorities = new ArrayList<>(5); - Map> flags = new HashMap<>(); List clinicalConsentList = new ArrayList<>(); - List clinicalStatusValueList = new ArrayList<>(4); clinicalStatusValueList.add( new ClinicalStatusValue("READY_FOR_INTERPRETATION", "The Clinical Analysis is ready for interpretations", ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) ); clinicalStatusValueList.add( new ClinicalStatusValue("READY_FOR_REPORT", "The Interpretation is finished and it is to create the report", - ClinicalStatusValue.ClinicalStatusType.IN_PROGRESS) + ClinicalStatusValue.ClinicalStatusType.ACTIVE) + ); + clinicalStatusValueList.add( + new ClinicalStatusValue("DONE", "The Clinical Analysis is done", ClinicalStatusValue.ClinicalStatusType.DONE) ); clinicalStatusValueList.add( new ClinicalStatusValue("CLOSED", "The Clinical Analysis is closed", ClinicalStatusValue.ClinicalStatusType.CLOSED) @@ -50,21 +52,12 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { clinicalStatusValueList.add( new ClinicalStatusValue("REJECTED", "The Clinical Analysis is rejected", ClinicalStatusValue.ClinicalStatusType.CLOSED) ); - status.put(ClinicalAnalysis.Type.FAMILY, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.CANCER, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.COHORT, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.SINGLE, clinicalStatusValueList); - List interpretationStatusList = new ArrayList<>(3); - interpretationStatusList.add(new ClinicalStatusValue("IN_PROGRESS", "Interpretation in progress", ClinicalStatusValue.ClinicalStatusType.IN_PROGRESS)); + interpretationStatusList.add(new ClinicalStatusValue("NOT_STARTED", "Interpretation not started", ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)); + interpretationStatusList.add(new ClinicalStatusValue("IN_PROGRESS", "Interpretation in progress", ClinicalStatusValue.ClinicalStatusType.ACTIVE)); + interpretationStatusList.add(new ClinicalStatusValue("DONE", "Interpretation done", ClinicalStatusValue.ClinicalStatusType.DONE)); interpretationStatusList.add(new ClinicalStatusValue("READY", "Interpretation ready", ClinicalStatusValue.ClinicalStatusType.CLOSED)); interpretationStatusList.add(new ClinicalStatusValue("REJECTED", "Interpretation rejected", ClinicalStatusValue.ClinicalStatusType.CLOSED)); - interpretationStatus.put(ClinicalAnalysis.Type.FAMILY, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.CANCER, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.COHORT, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.SINGLE, interpretationStatusList); priorities.add(new ClinicalPriorityValue("URGENT", "Highest priority of all", 1, false)); priorities.add(new ClinicalPriorityValue("HIGH", "Second highest priority of all", 2, false)); @@ -80,20 +73,15 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { flagValueList.add(new FlagValue("UNUSUAL_KARYOTYPE", "")); flagValueList.add(new FlagValue("SUSPECTED_MOSAICISM", "")); flagValueList.add(new FlagValue("LOW_QUALITY_SAMPLE", "")); - flags.put(ClinicalAnalysis.Type.FAMILY, flagValueList); - flags.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, flagValueList); - flags.put(ClinicalAnalysis.Type.CANCER, flagValueList); - flags.put(ClinicalAnalysis.Type.COHORT, flagValueList); - flags.put(ClinicalAnalysis.Type.SINGLE, flagValueList); clinicalConsentList.add(new ClinicalConsent("PRIMARY_FINDINGS", "Primary findings", "")); clinicalConsentList.add(new ClinicalConsent("SECONDARY_FINDINGS", "Secondary findings", "")); clinicalConsentList.add(new ClinicalConsent("CARRIER_FINDINGS", "Carrier findings", "")); clinicalConsentList.add(new ClinicalConsent("RESEARCH_FINDINGS", "Research findings", "")); - return new ClinicalAnalysisStudyConfiguration(status, - new InterpretationStudyConfiguration(interpretationStatus, Collections.emptyList(), Collections.emptyMap(), - Collections.emptyList()), priorities, flags, new ClinicalConsentConfiguration(clinicalConsentList)); + return new ClinicalAnalysisStudyConfiguration(clinicalStatusValueList, + new InterpretationStudyConfiguration(interpretationStatusList, Collections.emptyList(), Collections.emptyMap(), + Collections.emptyList()), priorities, flagValueList, new ClinicalConsentConfiguration(clinicalConsentList)); } @Override @@ -108,11 +96,11 @@ public String toString() { return sb.toString(); } - public Map> getStatus() { + public List getStatus() { return status; } - public ClinicalAnalysisStudyConfiguration setStatus(Map> status) { + public ClinicalAnalysisStudyConfiguration setStatus(List status) { this.status = status; return this; } @@ -135,11 +123,11 @@ public ClinicalAnalysisStudyConfiguration setPriorities(List> getFlags() { + public List getFlags() { return flags; } - public ClinicalAnalysisStudyConfiguration setFlags(Map> flags) { + public ClinicalAnalysisStudyConfiguration setFlags(List flags) { this.flags = flags; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java index b038fd5674d..0b9c98a780a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java @@ -1,6 +1,5 @@ package org.opencb.opencga.core.models.study.configuration; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; import java.util.List; @@ -8,16 +7,16 @@ public class InterpretationStudyConfiguration { - private Map> status; + private List status; private List variantCallers; + @Deprecated private Map defaultFilter; private List inclusion; public InterpretationStudyConfiguration() { } - public InterpretationStudyConfiguration(Map> status, - List variantCallers, + public InterpretationStudyConfiguration(List status, List variantCallers, Map defaultFilter, List inclusion) { this.status = status; this.variantCallers = variantCallers; @@ -36,11 +35,11 @@ public String toString() { return sb.toString(); } - public Map> getStatus() { + public List getStatus() { return status; } - public InterpretationStudyConfiguration setStatus(Map> status) { + public InterpretationStudyConfiguration setStatus(List status) { this.status = status; return this; } @@ -54,10 +53,12 @@ public InterpretationStudyConfiguration setVariantCallers(List getDefaultFilter() { return defaultFilter; } + @Deprecated public InterpretationStudyConfiguration setDefaultFilter(Map defaultFilter) { this.defaultFilter = defaultFilter; return this; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java index f31d4d477b4..293ade331be 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java @@ -27,44 +27,49 @@ */ public class Account { - @DataField(id = "creationDate", indexed = true, - description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) - private String creationDate; - @DataField(id = "expirationDate", indexed = true, - description = FieldConstants.ACCOUNT_EXPIRATION_DATE_DESCRIPTION) + description = FieldConstants.INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION) private String expirationDate; + @DataField(id = "password", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION) + private Password password; + + @DataField(id = "failedAttempts", description = FieldConstants.INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION) + private int failedAttempts; @DataField(id = "authentication", indexed = true, uncommentedClasses = {"AccountType"}, - description = FieldConstants.ACCOUNT_AUTHENTICATION) + description = FieldConstants.INTERNAL_ACCOUNT_AUTHENTICATION) private AuthenticationOrigin authentication; public Account() { String creationDate = TimeUtils.getTime(); + // Default 1 year Calendar cal = Calendar.getInstance(); cal.setTime(TimeUtils.toDate(creationDate)); cal.add(Calendar.YEAR, +1); String expirationDate = TimeUtils.getTime(cal.getTime()); - this.creationDate = creationDate; this.expirationDate = expirationDate; + this.password = new Password(); + this.failedAttempts = 0; this.authentication = null; } - public Account(String creationDate, String expirationDate, AuthenticationOrigin authentication) { + public Account(String expirationDate, Password password, int failedAttempts, AuthenticationOrigin authentication) { this.expirationDate = expirationDate; - this.creationDate = creationDate; + this.password = password; + this.failedAttempts = failedAttempts; this.authentication = authentication; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Account{"); - sb.append("creationDate='").append(creationDate).append('\''); - sb.append(", expirationDate='").append(expirationDate).append('\''); - sb.append(", authentication='").append(authentication).append('\''); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", password=").append(password); + sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", authentication=").append(authentication); sb.append('}'); return sb.toString(); } @@ -78,12 +83,21 @@ public Account setExpirationDate(String expirationDate) { return this; } - public String getCreationDate() { - return creationDate; + public Password getPassword() { + return password; + } + + public Account setPassword(Password password) { + this.password = password; + return this; + } + + public int getFailedAttempts() { + return failedAttempts; } - public Account setCreationDate(String creationDate) { - this.creationDate = creationDate; + public Account setFailedAttempts(int failedAttempts) { + this.failedAttempts = failedAttempts; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java index ce30fd63c41..938ebd6dc7f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java @@ -11,16 +11,16 @@ public class OrganizationUserUpdateParams extends UserUpdateParams { private UserQuota quota; - private Account account; + private Internal internal; private Map attributes; public OrganizationUserUpdateParams() { } - public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Account account, Map attributes) { + public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Internal internal, Map attributes) { super(name, email); this.quota = quota; - this.account = account; + this.internal = internal; this.attributes = attributes; } @@ -33,7 +33,7 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { public String toString() { final StringBuilder sb = new StringBuilder("OrganizationUserUpdateParams{"); sb.append("quota=").append(quota); - sb.append(", account=").append(account); + sb.append(", internal=").append(internal); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); @@ -48,12 +48,27 @@ public OrganizationUserUpdateParams setQuota(UserQuota quota) { return this; } + @Deprecated + @JsonIgnore public Account getAccount() { - return account; + return getInternal().getAccount(); } + @Deprecated public OrganizationUserUpdateParams setAccount(Account account) { - this.account = account; + if (internal == null) { + internal = new Internal(); + } + internal.setAccount(account); + return this; + } + + public Internal getInternal() { + return internal; + } + + public OrganizationUserUpdateParams setInternal(Internal internal) { + this.internal = internal; return this; } @@ -78,6 +93,30 @@ public OrganizationUserUpdateParams setEmail(String email) { return this; } + public static class Internal { + private Account account; + + public Internal() { + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Internal{"); + sb.append("account=").append(account); + sb.append('}'); + return sb.toString(); + } + + public Account getAccount() { + return account; + } + + public Internal setAccount(Account account) { + this.account = account; + return this; + } + } + public static class Account { private String expirationDate; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java new file mode 100644 index 00000000000..5b375936bd3 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java @@ -0,0 +1,48 @@ +package org.opencb.opencga.core.models.user; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +public class Password { + + @DataField(id = "expirationDate", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION) + private String expirationDate; + + @DataField(id = "lastModified", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION) + private String lastModified; + + public Password() { + } + + public Password(String expirationDate, String lastModified) { + this.expirationDate = expirationDate; + this.lastModified = lastModified; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Password{"); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getExpirationDate() { + return expirationDate; + } + + public Password setExpirationDate(String expirationDate) { + this.expirationDate = expirationDate; + return this; + } + + public String getLastModified() { + return lastModified; + } + + public Password setLastModified(String lastModified) { + this.lastModified = lastModified; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java index b4d3ef26c02..98273a80528 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java @@ -49,14 +49,15 @@ public class User { @DataField(id = "organization", indexed = true, description = FieldConstants.USER_ORGANIZATION) private String organization; - @DataField(id = "account", indexed = true, description = FieldConstants.USER_ACCOUNT) - private Account account; + @DataField(id = "creationDate", since = "3.2.1", description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; - /** - * An object describing the internal information of the User. This is managed by OpenCGA. - * - * @apiNote Internal - */ + @DataField(id = "modificationDate", since = "3.2.1", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "account", description = FieldConstants.USER_ACCOUNT) + @Deprecated + private Account account; @DataField(id = "internal", indexed = true, description = FieldConstants.GENERIC_INTERNAL) private UserInternal internal; @@ -89,22 +90,21 @@ public class User { public User() { } - public User(String id, Account account) { - this(id, id, null, null, account, new UserInternal(new UserStatus()), null, Collections.emptyList(), Collections.emptyMap(), + public User(String id) { + this(id, id, null, null, null, null, new UserInternal(new UserStatus()), null, Collections.emptyMap(), new LinkedList<>(), Collections.emptyMap()); } public User(String id, String name, String email, String organization, UserInternal internal) { - this(id, name, email, organization, null, internal, null, new ArrayList<>(), new HashMap<>(), new LinkedList<>(), new HashMap<>()); + this(id, name, email, organization, null, null, internal, null, new HashMap<>(), new LinkedList<>(), new HashMap<>()); } - public User(String id, String name, String email, String organization, Account account, UserInternal internal, UserQuota quota, - List projects, Map configs, List filters, Map attributes) { + public User(String id, String name, String email, String organization, UserInternal internal, UserQuota quota, List projects, + Map configs, List filters, Map attributes) { this.id = id; this.name = name; this.email = email; this.organization = organization; - this.account = account != null ? account : new Account(); this.internal = internal; this.quota = quota; this.projects = projects; @@ -113,6 +113,22 @@ public User(String id, String name, String email, String organization, Account a this.attributes = attributes; } + public User(String id, String name, String email, String organization, String creationDate, String modificationDate, + UserInternal internal, UserQuota quota, Map configs, List filters, + Map attributes) { + this.id = id; + this.name = name; + this.email = email; + this.organization = organization; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.internal = internal; + this.quota = quota; + this.configs = configs; + this.filters = filters; + this.attributes = attributes; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("User{"); @@ -120,7 +136,8 @@ public String toString() { sb.append(", name='").append(name).append('\''); sb.append(", email='").append(email).append('\''); sb.append(", organization='").append(organization).append('\''); - sb.append(", account=").append(account); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", internal=").append(internal); sb.append(", quota=").append(quota); sb.append(", projects=").append(projects); @@ -167,10 +184,30 @@ public User setOrganization(String organization) { return this; } + public String getCreationDate() { + return creationDate; + } + + public User setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public User setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + @Deprecated public Account getAccount() { return account; } + @Deprecated public User setAccount(Account account) { this.account = account; return this; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java index 329db613b5b..55a75709bd1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java @@ -16,32 +16,41 @@ package org.opencb.opencga.core.models.user; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; public class UserInternal extends Internal { + @DataField(id = "status", description = FieldConstants.INTERNAL_STATUS_DESCRIPTION) private UserStatus status; - private int failedAttempts; + + @DataField(id = "account", since = "3.2.1", description = FieldConstants.USER_ACCOUNT) + private Account account; public UserInternal() { } public UserInternal(UserStatus status) { - this(TimeUtils.getTime(), TimeUtils.getTime(), status); + this(TimeUtils.getTime(), TimeUtils.getTime(), status, new Account()); + } + + public UserInternal(UserStatus status, Account account) { + this(TimeUtils.getTime(), TimeUtils.getTime(), status, account); } - public UserInternal(String registrationDate, String lastModified, UserStatus status) { + public UserInternal(String registrationDate, String lastModified, UserStatus status1, Account account) { super(null, registrationDate, lastModified); - this.status = status; - this.failedAttempts = 0; + this.status = status1; + this.account = account; } @Override public String toString() { final StringBuilder sb = new StringBuilder("UserInternal{"); sb.append("status=").append(status); - sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", account=").append(account); sb.append(", registrationDate='").append(registrationDate).append('\''); sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); @@ -57,12 +66,13 @@ public UserInternal setStatus(UserStatus status) { return this; } - public int getFailedAttempts() { - return failedAttempts; + public Account getAccount() { + return account; } - public UserInternal setFailedAttempts(int failedAttempts) { - this.failedAttempts = failedAttempts; + public UserInternal setAccount(Account account) { + this.account = account; return this; } + } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java new file mode 100644 index 00000000000..1a97c0de539 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -0,0 +1,173 @@ +package org.opencb.opencga.core.models.variant; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.tools.ToolParams; + +import java.util.List; + +public class VariantSetupParams extends ToolParams { + + @DataField(description = "Expected number of samples that will be loaded. Used to infer some parameters. " + + "This number is only used as a hint. " + + "If the real number of samples is different, if it grows beyond expectation, or if , the loader should be able to handle it.", + required = true) + private Integer expectedSamples; + + @DataField(description = "Expected number of files that will be loaded. Used to infer some parameters. " + + "This number is only used as a hint. " + + "If the real number of files is different, the loader should be able to handle it.", required = true) + private Integer expectedFiles; + + @DataField(description = "Main type of the files that will be loaded. If the dataset contains multiple types of files," + + " provide the one that matches most of the files.") + private FileType fileType; + + @DataField(description = "Average size of the files that will be loaded. This number is only used as a hint. " + + "If the real size of the files is different, the loader should be able to handle it. Accepts units. e.g. 435MB, 2GB, 86KB. " + + "If not provided, the value will be inferred from the file type.") + private String averageFileSize; + + @DataField(description = "Number of variants per sample (non hom_ref variants). This number is only used as a hint. " + + "If the real number of variants per sample is different, the loader should be able to handle it. " + + "If not provided, the value will be inferred from the file type.") + private Integer variantsPerSample; + + @DataField(description = "Average number of samples per file. This number is only used as a hint. " + + "If the real number of samples per file is different, the loader should be able to handle it. " + + "If not provided, the value will be inferred from the expectedSamples and expectedFiles and dataDistribution.") + private Float averageSamplesPerFile; + + @DataField(description = "Data distribution of the files. This parameter is used to infer the number of samples per file.") + private DataDistribution dataDistribution; + + @DataField(description = "List of normalization extensions that will be used to normalize the files.") + private List normalizeExtensions; + + public VariantSetupParams(VariantSetupParams params) { + this.expectedSamples = params.expectedSamples; + this.expectedFiles = params.expectedFiles; + this.fileType = params.fileType; + this.averageFileSize = params.averageFileSize; + this.variantsPerSample = params.variantsPerSample; + this.averageSamplesPerFile = params.averageSamplesPerFile; + this.dataDistribution = params.dataDistribution; + this.normalizeExtensions = params.normalizeExtensions; + } + + public VariantSetupParams() { + } + + public enum DataDistribution { + // Single sample VCF files. One file per sample. + // e.g. + // - Platinum gVCF + // - Cancer germline + // - RD germline without family calling + @DataField(description = "Single sample VCF files. One file per sample. e.g. Platinum gVCF, Cancer germline, RD germline without family calling") + SINGLE_SAMPLE_PER_FILE, + + // Multi samples VCF files. One file with multiple samples. + // e.g. + // - Corpasome + // - RD germline with family calling + @DataField(description = "Multi samples VCF files. One file with multiple samples. e.g. Corpasome, RD germline with family calling") + MULTIPLE_SAMPLES_PER_FILE, + + // Multiple files per sample. Each file might have multiple samples. + // e.g. + // - Somatic study with multiple callers + @DataField(description = "Multiple files per sample. Each file might have multiple samples. e.g. Somatic study with multiple callers") + MULTIPLE_FILES_PER_SAMPLE, + + // Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. + // e.g. + // - 1000 genomes + @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. e.g. 1000 genomes") + FILES_SPLIT_BY_CHROMOSOME, + + // Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region. + @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region.") + FILES_SPLIT_BY_REGION, + } + + public enum FileType { + @DataField(description = "Whole genome VCF file.") + GENOME_VCF, + @DataField(description = "Whole genome gVCF file.") + GENOME_gVCF, + @DataField(description = "Exome VCF file.") + EXOME + } + + public Integer getExpectedSamples() { + return expectedSamples; + } + + public VariantSetupParams setExpectedSamples(Integer expectedSamples) { + this.expectedSamples = expectedSamples; + return this; + } + + public Integer getExpectedFiles() { + return expectedFiles; + } + + public VariantSetupParams setExpectedFiles(Integer expectedFiles) { + this.expectedFiles = expectedFiles; + return this; + } + + public FileType getFileType() { + return fileType; + } + + public VariantSetupParams setFileType(FileType fileType) { + this.fileType = fileType; + return this; + } + + public String getAverageFileSize() { + return averageFileSize; + } + + public VariantSetupParams setAverageFileSize(String averageFileSize) { + this.averageFileSize = averageFileSize; + return this; + } + + public Integer getVariantsPerSample() { + return variantsPerSample; + } + + public VariantSetupParams setVariantsPerSample(Integer variantsPerSample) { + this.variantsPerSample = variantsPerSample; + return this; + } + + public Float getAverageSamplesPerFile() { + return averageSamplesPerFile; + } + + public VariantSetupParams setAverageSamplesPerFile(Float averageSamplesPerFile) { + this.averageSamplesPerFile = averageSamplesPerFile; + return this; + } + + public DataDistribution getDataDistribution() { + return dataDistribution; + } + + public VariantSetupParams setDataDistribution(DataDistribution dataDistribution) { + this.dataDistribution = dataDistribution; + return this; + } + + public List getNormalizeExtensions() { + return normalizeExtensions; + } + + public VariantSetupParams setNormalizeExtensions(List normalizeExtensions) { + this.normalizeExtensions = normalizeExtensions; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java index 15cf7a51215..0b16875bebb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java @@ -17,9 +17,10 @@ package org.opencb.opencga.core.tools; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.core.tools.annotations.ToolExecutor; +import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import java.nio.file.Path; @@ -31,6 +32,7 @@ public abstract class OpenCgaToolExecutor { private ObjectMap executorParams; private Path outDir; private ExecutionResultManager arm; + private Configuration configuration; protected OpenCgaToolExecutor() { } @@ -51,10 +53,11 @@ public final ToolExecutor.Source getSource() { return this.getClass().getAnnotation(ToolExecutor.class).source(); } - public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir) { + public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir, Configuration configuration) { this.arm = arm; this.executorParams = executorParams; this.outDir = outDir; + this.configuration = configuration; } public final void execute() throws ToolException { @@ -77,6 +80,10 @@ public final Path getOutDir() { return outDir; } + public final Configuration getConfiguration() { + return configuration; + } + protected final String getToken() { return getExecutorParams().getString("token"); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java index 2af4d769513..980a0a8d494 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java @@ -33,7 +33,7 @@ public abstract class GwasAnalysisExecutor extends OpenCgaToolExecutor { private String cohort1; private String cohort2; private Path outputFile; - private GwasConfiguration configuration; + private GwasConfiguration gwasConfiguration; public GwasAnalysisExecutor() { } @@ -49,7 +49,7 @@ protected List getHeaderColumns() { columns.add("gene"); columns.add("biotype"); columns.add("consequence-types"); - if (configuration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { + if (gwasConfiguration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { columns.add("chi-square"); } columns.add("p-value"); @@ -70,7 +70,7 @@ public String toString() { sb.append(", phenotype2='").append(phenotype2).append('\''); sb.append(", cohort1='").append(cohort1).append('\''); sb.append(", cohort2='").append(cohort2).append('\''); - sb.append(", configuration=").append(configuration); + sb.append(", configuration=").append(gwasConfiguration); sb.append(", executorParams=").append(getExecutorParams()); sb.append(", outDir=").append(getOutDir()); sb.append('}'); @@ -149,12 +149,12 @@ public GwasAnalysisExecutor setOutputFile(Path outputFile) { return this; } - public GwasConfiguration getConfiguration() { - return configuration; + public GwasConfiguration getGwasConfiguration() { + return gwasConfiguration; } - public GwasAnalysisExecutor setConfiguration(GwasConfiguration configuration) { - this.configuration = configuration; + public GwasAnalysisExecutor setGwasConfiguration(GwasConfiguration gwasConfiguration) { + this.gwasConfiguration = gwasConfiguration; return this; } } diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 62adc8b6728..bb513b9df58 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -7,7 +7,9 @@ workspace: ${OPENCGA.USER.WORKSPACE} jobDir: ${OPENCGA.USER.WORKSPACE}/jobs # Maximum number of login attempts before banning a user account -maxLoginAttempts: ${OPENCGA.MAX_LOGIN_ATTEMPTS} +account: + maxLoginAttempts: ${OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS} + passwordExpirationDays: 0 panel: host: "http://resources.opencb.org/opencb/opencga/disease-panels" @@ -65,9 +67,32 @@ healthCheck: analysis: - packages: # List of packages where to find analysis tools + # List of packages where to find analysis tools + packages: - "org.opencb.opencga" - scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" # Scratch folder for the analysis. + # Scratch folder for the analysis. + scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + - id: "exomiser" + version: "13.1" + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG19: "exomiser/2109_hg19.zip" + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + - id: "exomiser" + version: "14.0" + defaultVersion: true + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG19: "exomiser/2402_hg19.zip" + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index 5956eb68e65..01c02bde4ad 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -205,7 +205,6 @@ public class ExecutionDaemon extends MonitorParentDaemon implements Closeable { put(RvtestsWrapperAnalysis.ID, "variant " + RvtestsWrapperAnalysis.ID + "-run"); put(GatkWrapperAnalysis.ID, "variant " + GatkWrapperAnalysis.ID + "-run"); put(ExomiserWrapperAnalysis.ID, "variant " + ExomiserWrapperAnalysis.ID + "-run"); - put(VariantFileDeleteOperationTool.ID, "variant file-delete"); put(VariantSecondaryAnnotationIndexOperationTool.ID, "variant secondary-index"); put(VariantSecondaryIndexSamplesDeleteOperationTool.ID, "variant secondary-index-delete"); put(VariantScoreDeleteOperationTool.ID, "variant score-delete"); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java index 8befc2876c5..faa3dd8626e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java @@ -401,9 +401,11 @@ private String getBodyParams(RestParameter body) { String javaCommandOptionsField = "commandOptions." + getJavaFieldName(restParameter); String label = StringUtils.isEmpty(restParameter.getParentName()) ? restParameter.getName() : restParameter.getParentName() + "." + restParameter.getName(); if (restParameter.getTypeClass().equals("java.lang.String;")) { - sb.append(" putNestedIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\"," + javaCommandOptionsField + ", true);\n"); + sb.append(" putNestedIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); + } else if (isValidMap(restParameter)) { + sb.append(" putNestedMapIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); } else { - sb.append(" putNestedIfNotNull(beanParams, \"" + label.replaceAll("body_", "") + "\"," + javaCommandOptionsField + ", true);\n"); + sb.append(" putNestedIfNotNull(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); } } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java index 15085581bca..73079cf2704 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java @@ -17,6 +17,7 @@ package org.opencb.opencga.server.rest; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -204,9 +205,17 @@ public Response createNote( }) public Response updateNote( @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = "Action to be performed if the array of tags is being updated.", allowableValues = "ADD,REMOVE,SET", defaultValue = "ADD") + @QueryParam("tagsAction") ParamUtils.BasicUpdateAction tagsAction, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(value = "JSON containing the Note fields to be updated.", required = true) NoteUpdateParams parameters) { try { + if (tagsAction == null) { + tagsAction = ParamUtils.BasicUpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), tagsAction); + queryOptions.put(Constants.ACTIONS, actionMap); OpenCGAResult result = catalogManager.getNotesManager().updateOrganizationNote(noteId, parameters, queryOptions, token); return createOkResponse(result); } catch (Exception e) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java index f6cc3fe8882..8492d30c490 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java @@ -23,6 +23,7 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.templates.TemplateRunner; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.Constants; @@ -46,7 +47,9 @@ import javax.ws.rs.core.*; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; @@ -613,9 +616,17 @@ public Response createNote( public Response updateNote( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @PathParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = "Action to be performed if the array of tags is being updated.", allowableValues = "ADD,REMOVE,SET", defaultValue = "ADD") + @QueryParam("tagsAction") ParamUtils.BasicUpdateAction tagsAction, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(value = "JSON containing the Note fields to be updated.", required = true) NoteUpdateParams parameters) { try { + if (tagsAction == null) { + tagsAction = ParamUtils.BasicUpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), tagsAction); + queryOptions.put(Constants.ACTIONS, actionMap); OpenCGAResult result = catalogManager.getNotesManager().updateStudyNote(studyStr, noteId, parameters, queryOptions, token); return createOkResponse(result); } catch (Exception e) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java index f893c1f5986..3dd6fa53aa4 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java @@ -401,13 +401,14 @@ public Response delete( public Response info( @ApiParam(value = ParamConstants.CLINICAL_ANALYSES_DESCRIPTION) @PathParam(value = "clinicalAnalysis") String clinicalAnalysisStr, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.CLINICAL_VERSION_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_VERSION_PARAM) String version, @ApiParam(value = ParamConstants.DELETED_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.DELETED_PARAM) boolean deleted) { try { query.remove(ParamConstants.STUDY_PARAM); query.remove("clinicalAnalysis"); List analysisList = getIdList(clinicalAnalysisStr); - DataResult analysisResult = clinicalManager.get(studyStr, analysisList, queryOptions, true, token); + DataResult analysisResult = clinicalManager.get(studyStr, analysisList, query, queryOptions, true, token); return createOkResponse(analysisResult); } catch (Exception e) { return createErrorResponse(e); @@ -450,6 +451,7 @@ public Response search( @ApiParam(value = ParamConstants.CLINICAL_DUE_DATE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_DUE_DATE_PARAM) String dueDate, @ApiParam(value = ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_PARAM) String qualityControl, @ApiParam(value = ParamConstants.CLINICAL_RELEASE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, @ApiParam(value = ParamConstants.CLINICAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_STATUS_PARAM) String status, @ApiParam(value = ParamConstants.CLINICAL_INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam(Constants.ANNOTATION) String annotation, @@ -490,6 +492,7 @@ public Response distinct( @ApiParam(value = ParamConstants.CLINICAL_DUE_DATE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_DUE_DATE_PARAM) String dueDate, @ApiParam(value = ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_PARAM) String qualityControl, @ApiParam(value = ParamConstants.CLINICAL_RELEASE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, @ApiParam(value = ParamConstants.CLINICAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_STATUS_PARAM) String status, @ApiParam(value = ParamConstants.CLINICAL_INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam(Constants.ANNOTATION) String annotation, @@ -767,6 +770,7 @@ public Response interpretationSearch( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INTERPRETATION_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ID_PARAM) String id, @ApiParam(value = ParamConstants.INTERPRETATION_UUID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.INTERPRETATION_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_NAME_PARAM) String name, @ApiParam(value = ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM) String clinicalAnalysisId, @ApiParam(value = ParamConstants.INTERPRETATION_ANALYST_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ANALYST_ID_PARAM) String analystId, @ApiParam(value = ParamConstants.INTERPRETATION_METHOD_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_METHOD_NAME_PARAM) String method, @@ -800,6 +804,7 @@ public Response interpretationDistinct( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INTERPRETATION_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ID_PARAM) String id, @ApiParam(value = ParamConstants.INTERPRETATION_UUID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.INTERPRETATION_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_NAME_PARAM) String name, @ApiParam(value = ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM) String clinicalAnalysisId, @ApiParam(value = ParamConstants.INTERPRETATION_ANALYST_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ANALYST_ID_PARAM) String analystId, @ApiParam(value = ParamConstants.INTERPRETATION_METHOD_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_METHOD_NAME_PARAM) String methodsName, diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java index be6ec91c91f..70748fffb0c 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java @@ -1332,7 +1332,7 @@ public Response circos( ObjectMap executorParams = new ObjectMap(); executorParams.put("opencgaHome", opencgaHome); executorParams.put("token", token); - executor.setUp(null, executorParams, outDir.toPath()); + executor.setUp(null, executorParams, outDir.toPath(), null); // Run Circos executor StopWatch watch = StopWatch.createStarted(); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java index a2143f45090..14fb051c85c 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java @@ -29,6 +29,7 @@ import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.operations.variant.*; +import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.*; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.core.tools.annotations.Api; @@ -101,6 +102,23 @@ public Response variantConfigure( }); } + @POST + @Path("/variant/setup") + @ApiOperation(value = "Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation.", + response = VariantSetupResult.class) + public Response variantConfigure( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = "Variant setup params") VariantSetupParams params) { + return run(() -> { + StopWatch stopWatch = StopWatch.createStarted(); + VariantSetupResult result = variantManager.variantSetup(study, params, token); + return new DataResult<>() + .setResults(Collections.singletonList(result)) + .setNumResults(1) + .setTime(((int) stopWatch.getTime(TimeUnit.MILLISECONDS))); + }); + } + @POST @Path("/variant/index") @ApiOperation(value = VariantIndexOperationTool.DESCRIPTION, response = Job.class) diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java index 398b7f775cd..acd37799e0e 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java @@ -218,7 +218,10 @@ public static class GenericVariantDeleteOptions { splitter = CommaParameterSplitter.class, required = true) public List file = null; - @Parameter(names = {"--resume"}, description = "Resume a previously failed indexation") + @Parameter(names = {"--force"}, description = "Force delete operation. This would allow deleting partially loaded files.") + public boolean force; + + @Parameter(names = {"--resume"}, description = "Resume failed delete operation.") public boolean resume; } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index 680d30a20db..1e8dcdc94dc 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -56,7 +56,6 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.BiPredicate; @@ -191,14 +190,7 @@ public ObjectMap getConfiguration() { public Lock lockGlobal(long lockDuration, long timeout, String lockName) throws StorageEngineException { - try { - return projectDBAdaptor.lockProject(lockDuration, timeout, lockName); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new StorageEngineException("Unable to lock the Project", e); - } catch (TimeoutException e) { - throw new StorageEngineException("Unable to lock the Project", e); - } + return projectDBAdaptor.lockProject(lockDuration, timeout, lockName); } public Lock lockStudy(int studyId) throws StorageEngineException { @@ -282,8 +274,7 @@ public StudyMetadata updateStudyMetadata(Object study, Upd throws StorageEngineException, E { int studyId = getStudyId(study); - Lock lock = lockStudy(studyId); - try { + try (Lock lock = lockStudy(studyId)) { StudyMetadata sm = getStudyMetadata(studyId); sm = updater.update(sm); @@ -291,8 +282,6 @@ public StudyMetadata updateStudyMetadata(Object study, Upd lock.checkLocked(); unsecureUpdateStudyMetadata(sm); return sm; - } finally { - lock.unlock(); } } @@ -557,16 +546,8 @@ public ProjectMetadata updateProjectMetadata(UpdateConsume public ProjectMetadata updateProjectMetadata(UpdateFunction function) throws StorageEngineException, E { Objects.requireNonNull(function); - Lock lock; - try { - lock = projectDBAdaptor.lockProject(lockDuration, lockTimeout); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new StorageEngineException("Unable to lock the Project", e); - } catch (TimeoutException e) { - throw new StorageEngineException("Unable to lock the Project", e); - } - try { + + try (Lock lock = projectDBAdaptor.lockProject(lockDuration, lockTimeout)) { ProjectMetadata projectMetadata = getProjectMetadata(); int countersHash = (projectMetadata == null ? Collections.emptyMap() : projectMetadata.getCounters()).hashCode(); @@ -579,8 +560,6 @@ public ProjectMetadata updateProjectMetadata(UpdateFunctio lock.checkLocked(); projectDBAdaptor.updateProjectMetadata(projectMetadata, updateCounters); return projectMetadata; - } finally { - lock.unlock(); } } @@ -594,11 +573,15 @@ public ProjectMetadata getProjectMetadata() { public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws StorageEngineException { ProjectMetadata projectMetadata = getProjectMetadata(); + + checkSameSpeciesAndAssembly(options, projectMetadata); if (options != null && (projectMetadata == null || StringUtils.isEmpty(projectMetadata.getSpecies()) && options.containsKey(SPECIES.key()) || StringUtils.isEmpty(projectMetadata.getAssembly()) && options.containsKey(ASSEMBLY.key()))) { projectMetadata = updateProjectMetadata(pm -> { + // Check again, in case it was updated by another thread + checkSameSpeciesAndAssembly(options, pm); if (pm == null) { pm = new ProjectMetadata(); } @@ -619,6 +602,25 @@ public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws Sto return projectMetadata; } + private static void checkSameSpeciesAndAssembly(ObjectMap options, ProjectMetadata projectMetadata) throws StorageEngineException { + if (options != null && projectMetadata != null) { + if (options.containsKey(ASSEMBLY.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getAssembly()) && !projectMetadata.getAssembly() + .equalsIgnoreCase(options.getString(ASSEMBLY.key()))) { + throw new StorageEngineException("Incompatible assembly change from '" + projectMetadata.getAssembly() + "' to '" + + options.getString(ASSEMBLY.key()) + "'"); + } + } + if (options.containsKey(SPECIES.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getSpecies()) && !projectMetadata.getSpecies() + .equalsIgnoreCase(toCellBaseSpeciesName(options.getString(SPECIES.key())))) { + throw new StorageEngineException("Incompatible species change from '" + projectMetadata.getSpecies() + "' to '" + + options.getString(SPECIES.key()) + "'"); + } + } + } + } + public DataResult getVariantFileMetadata(int studyId, int fileId, QueryOptions options) throws StorageEngineException { return fileDBAdaptor.getVariantFileMetadata(studyId, fileId, options); @@ -673,16 +675,14 @@ public void unsecureUpdateFileMetadata(int studyId, FileMetadata file) { public FileMetadata updateFileMetadata(int studyId, int fileId, UpdateConsumer update) throws E, StorageEngineException { getFileName(studyId, fileId); // Check file exists - Lock lock = fileDBAdaptor.lock(studyId, fileId, lockDuration, lockTimeout); - try { + + try (Lock lock = fileDBAdaptor.lock(studyId, fileId, lockDuration, lockTimeout)) { FileMetadata fileMetadata = getFileMetadata(studyId, fileId); update.update(fileMetadata); lock.checkLocked(); unsecureUpdateFileMetadata(studyId, fileMetadata); fileIdIndexedCache.put(studyId, fileId, fileMetadata.isIndexed()); return fileMetadata; - } finally { - lock.unlock(); } } @@ -863,6 +863,19 @@ public Iterator fileMetadataIterator(int studyId) { return fileDBAdaptor.fileIterator(studyId); } + public SampleMetadata getSampleMetadata(Integer studyId, Integer sampleId) { + return getSampleMetadata(studyId.intValue(), sampleId.intValue()); + } + + public SampleMetadata getSampleMetadata(int studyId, Integer sampleId) { + return getSampleMetadata(studyId, sampleId.intValue()); + } + + public SampleMetadata getSampleMetadata(int studyId, Object sample) { + int sampleId = getSampleIdOrFail(studyId, sample); + return getSampleMetadata(studyId, sampleId); + } + public SampleMetadata getSampleMetadata(int studyId, int sampleId) { return sampleDBAdaptor.getSampleMetadata(studyId, sampleId, null); } @@ -875,15 +888,13 @@ public void unsecureUpdateSampleMetadata(int studyId, SampleMetadata sample) { public SampleMetadata updateSampleMetadata(int studyId, int sampleId, UpdateConsumer consumer) throws E, StorageEngineException { getSampleName(studyId, sampleId); // Check sample exists - Lock lock = sampleDBAdaptor.lock(studyId, sampleId, lockDuration, lockTimeout); - try { + + try (Lock lock = sampleDBAdaptor.lock(studyId, sampleId, lockDuration, lockTimeout)) { SampleMetadata sample = getSampleMetadata(studyId, sampleId); sample = consumer.toFunction().update(sample); lock.checkLocked(); unsecureUpdateSampleMetadata(studyId, sample); return sample; - } finally { - lock.unlock(); } } @@ -1054,15 +1065,12 @@ public void unsecureUpdateCohortMetadata(int studyId, CohortMetadata cohort) { public CohortMetadata updateCohortMetadata(int studyId, int cohortId, UpdateConsumer update) throws E, StorageEngineException { getCohortName(studyId, cohortId); // Check cohort exists - Lock lock = cohortDBAdaptor.lock(studyId, cohortId, lockDuration, lockTimeout); - try { + try (Lock lock = cohortDBAdaptor.lock(studyId, cohortId, lockDuration, lockTimeout)) { CohortMetadata cohortMetadata = getCohortMetadata(studyId, cohortId); update.update(cohortMetadata); lock.checkLocked(); unsecureUpdateCohortMetadata(studyId, cohortMetadata); return cohortMetadata; - } finally { - lock.unlock(); } } @@ -1190,13 +1198,19 @@ private CohortMetadata updateCohortSamples(int studyId, String cohortName, Colle for (Integer sampleId : sampleIds) { Integer finalCohortId = cohortId; if (secondaryIndexCohort) { - updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.addSecondaryIndexCohort(finalCohortId); - }); + if (!getSampleMetadata(studyId, sampleId).getSecondaryIndexCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.addSecondaryIndexCohort(finalCohortId); + }); + } } else { - updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.addCohort(finalCohortId); - }); + if (!getSampleMetadata(studyId, sampleId).getCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.addCohort(finalCohortId); + }); + } } } @@ -1209,13 +1223,19 @@ private CohortMetadata updateCohortSamples(int studyId, String cohortName, Colle Integer finalCohortId = cohortId; if (!sampleIds.contains(sampleFromCohort)) { if (secondaryIndexCohort) { - updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { - sampleMetadata.getSecondaryIndexCohorts().remove(finalCohortId); - }); + if (getSampleMetadata(studyId, sampleFromCohort).getSecondaryIndexCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { + sampleMetadata.getSecondaryIndexCohorts().remove(finalCohortId); + }); + } } else { - updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { - sampleMetadata.getCohorts().remove(finalCohortId); - }); + if (getSampleMetadata(studyId, sampleFromCohort).getCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { + sampleMetadata.getCohorts().remove(finalCohortId); + }); + } } } } @@ -1326,15 +1346,12 @@ public void unsecureUpdateTask(int studyId, TaskMetadata task) throws StorageEng public TaskMetadata updateTask(int studyId, int taskId, UpdateConsumer consumer) throws E, StorageEngineException { getTask(studyId, taskId); // Check task exists - Lock lock = taskDBAdaptor.lock(studyId, taskId, lockDuration, lockTimeout); - try { + try (Lock lock = taskDBAdaptor.lock(studyId, taskId, lockDuration, lockTimeout)) { TaskMetadata task = getTask(studyId, taskId); consumer.update(task); lock.checkLocked(); unsecureUpdateTask(studyId, task); return task; - } finally { - lock.unlock(); } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java index dc88a85d338..3045ee8f3cd 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java @@ -7,7 +7,6 @@ import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import java.io.IOException; -import java.util.concurrent.TimeoutException; /** * Created on 02/05/18. @@ -17,14 +16,12 @@ public interface ProjectMetadataAdaptor extends AutoCloseable { default Lock lockProject(long lockDuration, long timeout) - throws InterruptedException, TimeoutException, StorageEngineException { + throws StorageEngineException { return lockProject(lockDuration, timeout, null); } Lock lockProject(long lockDuration, long timeout, String lockName) - throws InterruptedException, TimeoutException, StorageEngineException; - - void unLockProject(long lockId) throws StorageEngineException; + throws StorageEngineException; DataResult getProjectMetadata(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index dd24ee1334d..77327d9d76a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -24,13 +24,16 @@ import org.opencb.biodata.models.variant.avro.VariantAnnotation; import org.opencb.biodata.models.variant.metadata.SampleVariantStats; import org.opencb.biodata.models.variant.metadata.VariantMetadata; +import org.opencb.biodata.tools.variant.normalizer.extensions.VariantNormalizerExtensionFactory; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.commons.datastore.core.*; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngine; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -153,7 +156,12 @@ public static SplitData from(ObjectMap options) { String loadSplitDataStr = options.getString(LOAD_SPLIT_DATA.key()); boolean multiFile = options.getBoolean(LOAD_MULTI_FILE_DATA.key()); if (StringUtils.isNotEmpty(loadSplitDataStr) && multiFile) { - throw new IllegalArgumentException("Unable to mix loadSplitFile and loadMultiFile"); + if (loadSplitDataStr.equalsIgnoreCase("multi")) { + return MULTI; + } else { + throw new IllegalArgumentException("Unable to mix " + LOAD_MULTI_FILE_DATA.key() + "=true and " + + LOAD_SPLIT_DATA.key() + "='" + loadSplitDataStr + "'"); + } } if (StringUtils.isEmpty(loadSplitDataStr) && !multiFile) { return null; @@ -1348,7 +1356,7 @@ public VariantQueryExecutor getVariantQueryExecutor(Query query, QueryOptions op public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQuery) { try { for (VariantQueryExecutor executor : getVariantQueryExecutors()) { - if (executor.canUseThisExecutor(variantQuery.getQuery(), variantQuery.getInputOptions())) { + if (executor.canUseThisExecutor(variantQuery, variantQuery.getInputOptions())) { logger.info("Using VariantQueryExecutor : " + executor.getClass().getName()); logger.info(" Query : " + VariantQueryUtils.printQuery(variantQuery.getInputQuery())); logger.info(" Options : " + variantQuery.getInputOptions().toJson()); @@ -1362,6 +1370,19 @@ public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQu throw new VariantQueryException("No VariantQueryExecutor found to run the query!"); } + public final VariantQueryExecutor getVariantQueryExecutor(Class clazz) + throws StorageEngineException { + Optional first = getVariantQueryExecutors() + .stream() + .filter(e -> e instanceof SearchIndexVariantQueryExecutor) + .findFirst(); + if (first.isPresent()) { + return first.get(); + } else { + throw new StorageEngineException("VariantQueryExecutor " + clazz + " not found"); + } + } + public Query preProcessQuery(Query originalQuery, QueryOptions options) { try { return getVariantQueryParser().preProcessQuery(originalQuery, options); @@ -1485,6 +1506,45 @@ public VariantAggregationExecutor getVariantAggregationExecutor(Query query, Que throw new VariantQueryException("No VariantAggregationExecutor found to run the query. " + messages).setQuery(query); } + public ObjectMap inferConfigurationParams(VariantSetupParams params) { + ObjectMap options = new ObjectMap(); + + List normalizeExtensions = params.getNormalizeExtensions(); + if (normalizeExtensions != null && !normalizeExtensions.isEmpty()) { + if (!normalizeExtensions.equals(Collections.singletonList(ParamConstants.ALL))) { + List unsupportedExtensions = new ArrayList<>(); + for (String normalizeExtension : normalizeExtensions) { + if (!VariantNormalizerExtensionFactory.ALL_EXTENSIONS.contains(normalizeExtension)) { + unsupportedExtensions.add(normalizeExtension); + } + } + if (!unsupportedExtensions.isEmpty()) { + throw new IllegalArgumentException("Unsupported normalize extensions: " + unsupportedExtensions + ". Supported " + + "extensions are: " + VariantNormalizerExtensionFactory.ALL_EXTENSIONS); + } + } + options.put(NORMALIZATION_EXTENSIONS.key(), normalizeExtensions); + } + if (params.getDataDistribution() != null) { + switch (params.getDataDistribution()) { + case FILES_SPLIT_BY_CHROMOSOME: + options.put(LOAD_SPLIT_DATA.key(), SplitData.CHROMOSOME); + break; + case FILES_SPLIT_BY_REGION: + options.put(LOAD_SPLIT_DATA.key(), SplitData.REGION); + break; + case MULTIPLE_FILES_PER_SAMPLE: + options.put(LOAD_MULTI_FILE_DATA.key(), true); + options.put(LOAD_SPLIT_DATA.key(), SplitData.MULTI); + break; + default: + break; + } + } + + return options; + } + @Override public void close() throws IOException { cellBaseUtils = null; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java index caefbb5260e..847ae860d70 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java @@ -1,10 +1,11 @@ package org.opencb.opencga.storage.core.variant; import org.opencb.biodata.models.variant.metadata.Aggregation; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.config.ConfigurationOption; +import java.util.Arrays; + public enum VariantStorageOptions implements ConfigurationOption { STUDY("study"), @@ -26,7 +27,7 @@ public enum VariantStorageOptions implements ConfigurationOption { TRANSFORM_ISOLATE("transform.isolate", false), // Do not store file in metadata NORMALIZATION_SKIP("normalization.skip", false), // Do not run normalization NORMALIZATION_REFERENCE_GENOME("normalization.referenceGenome"), - NORMALIZATION_EXTENSIONS("normalization.extensions", ParamConstants.NONE), + NORMALIZATION_EXTENSIONS("normalization.extensions", Arrays.asList("VAF", "SV", "CUSTOM")), DEDUPLICATION_POLICY("deduplication.policy", "maxQual"), DEDUPLICATION_BUFFER_SIZE("deduplication.bufferSize", 100), @@ -83,8 +84,8 @@ public enum VariantStorageOptions implements ConfigurationOption { INDEX_SEARCH("indexSearch", false), // Build secondary indexes using search engine. - METADATA_LOCK_DURATION("metadata.lock.duration", 5000), - METADATA_LOCK_TIMEOUT("metadata.lock.timeout", 60000), + METADATA_LOCK_DURATION("metadata.lock.duration", 60000), + METADATA_LOCK_TIMEOUT("metadata.lock.timeout", 600000), METADATA_LOAD_BATCH_SIZE("metadata.load.batchSize", 10), METADATA_LOAD_THREADS("metadata.load.numThreads", 4), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 5b37d2512b1..34bbd5cfe5a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -60,10 +60,7 @@ import org.opencb.opencga.storage.core.io.plain.StringDataReader; import org.opencb.opencga.storage.core.io.plain.StringDataWriter; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils; @@ -78,6 +75,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; @@ -424,15 +422,22 @@ protected Task initNormalizer(VariantFileMetadata metadata) th .then(new VariantSorterTask(100)) // Sort before generating reference blocks .then(new VariantReferenceBlockCreatorTask(metadata.getHeader())); } - if (CollectionUtils.isNotEmpty(enabledExtensions)) { + if (CollectionUtils.isEmpty(enabledExtensions)) { + enabledExtensions = NORMALIZATION_EXTENSIONS.defaultValue(); + } + if ((enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.NONE))) { + logger.info("Skip normalization extensions"); + } else { + logger.info("Enable normalization extensions: {}", enabledExtensions); VariantNormalizerExtensionFactory extensionFactory; if (enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.ALL)) { - extensionFactory = new VariantNormalizerExtensionFactory(); - } else { - extensionFactory = new VariantNormalizerExtensionFactory(new HashSet<>(enabledExtensions)); + enabledExtensions = NORMALIZATION_EXTENSIONS.defaultValue(); } + extensionFactory = new VariantNormalizerExtensionFactory(new HashSet<>(enabledExtensions)); Task extension = extensionFactory.buildExtensions(metadata); - if (extension != null) { + if (extension == null) { + logger.info("No normalization extensions can be used."); + } else { normalizer = normalizer.then(extension); } } @@ -561,17 +566,19 @@ public URI preLoad(URI input, URI output) throws StorageEngineException { return input; } - protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata fileMetadata) throws StorageEngineException { + protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata variantFileMetadata) throws StorageEngineException { final int fileId; String virtualFile = options.getString(LOAD_VIRTUAL_FILE.key()); + boolean loadSampleIndex = YesNoAuto.parse(options, LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); + VariantStorageEngine.SplitData splitData = VariantStorageEngine.SplitData.from(options); - if (VariantStorageEngine.SplitData.isPartialSplit(options)) { + if (VariantStorageEngine.SplitData.isPartialSplit(splitData)) { if (StringUtils.isEmpty(virtualFile)) { - fileId = getMetadataManager().registerFile(studyId, fileMetadata); + fileId = getMetadataManager().registerFile(studyId, variantFileMetadata); // throw new StorageEngineException("Unable to load file with 'split-data'. Missing virtual file belonging! " // + "Please, define " + LOAD_VIRTUAL_FILE.key()); } else { - fileId = getMetadataManager().registerPartialFile(studyId, virtualFile, fileMetadata); + fileId = getMetadataManager().registerPartialFile(studyId, virtualFile, variantFileMetadata); } } else { if (StringUtils.isNotEmpty(virtualFile)) { @@ -580,10 +587,85 @@ protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata f + " to " + VariantStorageEngine.SplitData.REGION + " or " + VariantStorageEngine.SplitData.CHROMOSOME); } else { - fileId = getMetadataManager().registerFile(studyId, fileMetadata); + fileId = getMetadataManager().registerFile(studyId, variantFileMetadata); } } setFileId(fileId); + FileMetadata fileMetadata = getMetadataManager().getFileMetadata(studyId, getFileId()); + + int version = getMetadataManager().getStudyMetadata(studyId).getSampleIndexConfigurationLatest().getVersion(); + Set alreadyIndexedSamples = new LinkedHashSet<>(); + Set processedSamples = new LinkedHashSet<>(); + Set samplesWithoutSplitData = new LinkedHashSet<>(); + for (String sample : variantFileMetadata.getSampleIds()) { + Integer sampleId = getMetadataManager().getSampleId(studyId, sample); + SampleMetadata sampleMetadata = getMetadataManager().getSampleMetadata(studyId, sampleId); + if (splitData != null && sampleMetadata.getSplitData() != null) { + if (splitData != sampleMetadata.getSplitData()) { + throw new StorageEngineException("Incompatible split data methods. " + + "Unable to mix requested " + splitData + + " with existing " + sampleMetadata.getSplitData()); + } + } + if (sampleMetadata.isIndexed()) { + if (sampleMetadata.getFiles().size() == 1 && sampleMetadata.getFiles().contains(fileMetadata.getId())) { + // It might happen that the sample is marked as INDEXED, but not the file. + // If the sample only belongs to this file (i.e. it's only file is this file), then ignore + // the overwrite the current sample metadata index status + sampleMetadata = getMetadataManager().updateSampleMetadata(studyId, sampleId, + sm -> sm.setIndexStatus(fileMetadata.getIndexStatus())); + } + } + if (sampleMetadata.isIndexed()) { + alreadyIndexedSamples.add(sample); + if (sampleMetadata.isAnnotated() + || !loadSampleIndex && sampleMetadata.getSampleIndexStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.getSampleIndexAnnotationStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.getFamilyIndexStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.isFamilyIndexDefined()) { + processedSamples.add(sampleMetadata.getId()); + } + } + + if (splitData != null && splitData != sampleMetadata.getSplitData()) { + samplesWithoutSplitData.add(sampleId); + } + } + + if (!alreadyIndexedSamples.isEmpty()) { + if (splitData != null) { + logger.info("Loading split data"); + } else { + String fileName = Paths.get(variantFileMetadata.getPath()).getFileName().toString(); + throw StorageEngineException.alreadyLoadedSamples(fileName, new ArrayList<>(alreadyIndexedSamples)); + } + for (Integer sampleId : processedSamples) { + getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + if (!loadSampleIndex) { + for (Integer v : sampleMetadata.getSampleIndexVersions()) { + sampleMetadata.setSampleIndexStatus(TaskMetadata.Status.NONE, v); + } + } + for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { + sampleMetadata.setSampleIndexAnnotationStatus(TaskMetadata.Status.NONE, v); + } + for (Integer v : sampleMetadata.getFamilyIndexVersions()) { + sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.NONE, v); + } + sampleMetadata.setAnnotationStatus(TaskMetadata.Status.NONE); + sampleMetadata.setMendelianErrorStatus(TaskMetadata.Status.NONE); + }); + } + } + + if (splitData != null) { + // Register loadSplitData + for (Integer sampleId : samplesWithoutSplitData) { + getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.setSplitData(splitData); + }); + } + } } /** diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java index 8468ab34317..300f2e6b48a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java @@ -267,14 +267,6 @@ public VariantStudyQuery setStudies(ParsedQuery studies) { return this; } - public String getStudyOrFail() { - if (studies == null || studies.size() != 1) { - throw new VariantQueryException("Require exactly one study"); - } else { - return studies.get(0); - } - } - public ParsedQuery>> getGenotypes() { return genotypes; } @@ -311,6 +303,19 @@ public void setDefaultStudy(StudyMetadata defaultStudy) { public StudyMetadata getDefaultStudy() { return defaultStudy; } + + public StudyMetadata getDefaultStudyOrFail() { + if (defaultStudy == null) { + if (studies.size() != 1) { + throw new VariantQueryException("Only one study is allowed. Found " + studies.size() + " studies"); + } else { + throw new VariantQueryException("One study required. None provided"); + } + } else { + return defaultStudy; + } + } + } public static class VariantQueryXref { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java index 641e365a51d..bd02a6b4303 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java @@ -17,10 +17,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.core.models.variant.VariantAnnotationConstants; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; -import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -170,15 +167,14 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.defaultValue())); - variantQuery.setProjection(projectionParser.parseVariantQueryProjection(inputQuery, options)); - VariantQuery query; if (!skipPreProcess) { - query = new VariantQuery(preProcessQuery(inputQuery, options, variantQuery.getProjection())); + query = new VariantQuery(preProcessQuery(inputQuery, options)); } else { query = new VariantQuery(inputQuery); } variantQuery.setQuery(query); + variantQuery.setProjection(projectionParser.parseVariantQueryProjection(query, options)); List geneRegions = Region.parseRegions(query.getString(ANNOT_GENE_REGIONS.key())); variantQuery.setGeneRegions(geneRegions == null ? Collections.emptyList() : geneRegions); @@ -221,10 +217,7 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo = new ParsedQuery<>(sampleDataQuery.getKey(), sampleDataQuery.getOperation(), new ArrayList<>(sampleDataQuery.size())); for (KeyValues> keyValues : sampleDataQuery) { sampleDataQueryWithMetadata.getValues().add( - keyValues.mapKey(sample -> { - int sampleId = metadataManager.getSampleIdOrFail(defaultStudy.getId(), sample); - return metadataManager.getSampleMetadata(defaultStudy.getId(), sampleId); - })); + keyValues.mapKey(sample -> metadataManager.getSampleMetadata(defaultStudy.getId(), sample))); } studyQuery.setSampleDataQuery(sampleDataQueryWithMetadata); } @@ -232,17 +225,13 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo return variantQuery; } - public final Query preProcessQuery(Query originalQuery, QueryOptions options) { - return preProcessQuery(originalQuery, options, null); - } - - protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { + public Query preProcessQuery(Query originalQuery, QueryOptions options) { // Copy input query! Do not modify original query! Query query = VariantQueryUtils.copy(originalQuery); preProcessAnnotationParams(query); - preProcessStudyParams(query, options, projection); + preProcessStudyParams(query, options); if (options != null && options.getLong(QueryOptions.LIMIT) < 0) { throw VariantQueryException.malformedParam(QueryOptions.LIMIT, options.getString(QueryOptions.LIMIT), @@ -388,7 +377,7 @@ private VariantType parseVariantType(String type) { } } - protected void preProcessStudyParams(Query query, QueryOptions options, VariantQueryProjection projection) { + protected void preProcessStudyParams(Query query, QueryOptions options) { StudyMetadata defaultStudy = getDefaultStudy(query); QueryOperation formatOperator = null; if (isValidParam(query, SAMPLE_DATA)) { @@ -613,10 +602,13 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ if (isValidParam(query, SAMPLE_MENDELIAN_ERROR) || isValidParam(query, SAMPLE_DE_NOVO) - || isValidParam(query, SAMPLE_DE_NOVO_STRICT)) { + || isValidParam(query, SAMPLE_DE_NOVO_STRICT) + || isValidParam(query, SAMPLE_COMPOUND_HETEROZYGOUS)) { + boolean requireMendelianReady = false; QueryParam param = null; if (isValidParam(query, SAMPLE_MENDELIAN_ERROR)) { param = SAMPLE_MENDELIAN_ERROR; + requireMendelianReady = true; } if (isValidParam(query, SAMPLE_DE_NOVO)) { if (param != null) { @@ -624,6 +616,7 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ param, query.getString(param.key()), SAMPLE_DE_NOVO, query.getString(SAMPLE_DE_NOVO.key())); } + requireMendelianReady = true; param = SAMPLE_DE_NOVO; } if (isValidParam(query, SAMPLE_DE_NOVO_STRICT)) { @@ -632,8 +625,21 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ param, query.getString(param.key()), SAMPLE_DE_NOVO_STRICT, query.getString(SAMPLE_DE_NOVO_STRICT.key())); } + requireMendelianReady = true; param = SAMPLE_DE_NOVO_STRICT; } + if (isValidParam(query, SAMPLE_COMPOUND_HETEROZYGOUS)) { + if (param != null) { + throw VariantQueryException.unsupportedParamsCombination( + param, query.getString(param.key()), + SAMPLE_COMPOUND_HETEROZYGOUS, query.getString(SAMPLE_COMPOUND_HETEROZYGOUS.key())); + } + requireMendelianReady = false; + param = SAMPLE_COMPOUND_HETEROZYGOUS; + } + if (param == null) { + throw new IllegalStateException("Unknown param"); + } if (defaultStudy == null) { throw VariantQueryException.missingStudyForSamples(query.getAsStringList(param.key()), metadataManager.getStudyNames()); @@ -645,15 +651,18 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ genotypeParam, query.getString(genotypeParam.key()) ); } - List samples = query.getAsStringList(param.key()); + Object value = query.get(param.key()); + List samples; + if (value instanceof Trio) { + samples = Collections.singletonList(((Trio) value).getChild()); + } else { + samples = query.getAsStringList(param.key()); + } Set samplesAndParents = new LinkedHashSet<>(samples); for (String sample : samples) { - Integer sampleId = metadataManager.getSampleId(defaultStudy.getId(), sample); - if (sampleId == null) { - throw VariantQueryException.sampleNotFound(sample, defaultStudy.getName()); - } - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(defaultStudy.getId(), sampleId); - if (TaskMetadata.Status.READY != sampleMetadata.getMendelianErrorStatus()) { + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(defaultStudy.getId(), sample); + if (requireMendelianReady + && TaskMetadata.Status.READY != sampleMetadata.getMendelianErrorStatus()) { throw VariantQueryException.malformedParam(param, "Sample \"" + sampleMetadata.getName() + "\" does not have the Mendelian Errors precomputed yet"); } @@ -674,6 +683,21 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ } else { query.put(INCLUDE_SAMPLE.key(), new ArrayList<>(samplesAndParents)); } + if (param == SAMPLE_COMPOUND_HETEROZYGOUS) { + int studyId = defaultStudy.getId(); + if (!(value instanceof Trio)) { + if (samples.size() > 1) { + throw VariantQueryException.malformedParam(SAMPLE, value.toString(), + "More than one sample provided for compound heterozygous filter."); + } + SampleMetadata sm = metadataManager.getSampleMetadata(studyId, samples.get(0)); + Trio trio = new Trio(null, + metadataManager.getSampleName(studyId, sm.getFather()), + metadataManager.getSampleName(studyId, sm.getMother()), + sm.getName()); + query.put(SAMPLE_COMPOUND_HETEROZYGOUS.key(), trio); + } + } } if (isValidParam(query, SCORE)) { @@ -704,9 +728,7 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ || isValidParam(query, SAMPLE_SKIP) || isValidParam(query, SAMPLE_LIMIT) ) { - if (projection == null) { - projection = projectionParser.parseVariantQueryProjection(query, options); - } + VariantQueryProjection projection = projectionParser.parseVariantQueryProjection(query, options); // Apply the sample pagination. // Remove the sampleLimit and sampleSkip to avoid applying the pagination twice query.remove(SAMPLE_SKIP.key()); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java index 6eb237ea4b3..bc40c5b6418 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java @@ -14,6 +14,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; @@ -40,7 +41,8 @@ public BreakendVariantQueryExecutor(String storageEngineId, ObjectMap options, } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); return query.getString(VariantQueryParam.TYPE.key()).equals(VariantType.BREAKEND.name()) && VariantQueryUtils.isValidParam(query, VariantQueryParam.GENOTYPE); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java index c6f4b87a5f0..bcecac57b5a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java @@ -64,14 +64,14 @@ public CompoundHeterozygousQueryExecutor(VariantStorageMetadataManager metadataM } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { - return isValidParam(query, VariantQueryUtils.SAMPLE_COMPOUND_HETEROZYGOUS); + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + return isValidParam(variantQuery.getQuery(), VariantQueryUtils.SAMPLE_COMPOUND_HETEROZYGOUS); } @Override protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { Trio trio = getCompHetTrio(variantQuery.getQuery()); - String study = variantQuery.getStudyQuery().getStudyOrFail(); + String study = variantQuery.getStudyQuery().getDefaultStudyOrFail().getName(); return getOrIterator(study, trio.getChild(), trio.getFather(), trio.getMother(), variantQuery, iterator); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java index 474cbc3fa9f..e667e2a1485 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java @@ -52,9 +52,9 @@ protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { for (QueryParam unsupportedParam : UNSUPPORTED_PARAMS) { - if (VariantQueryUtils.isValidParam(query, unsupportedParam)) { + if (VariantQueryUtils.isValidParam(variantQuery.getQuery(), unsupportedParam)) { logger.warn("Unsupported variant query param {} in {}", unsupportedParam.key(), DBAdaptorVariantQueryExecutor.class.getSimpleName()); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java index e286b4a07ce..5f2da2c2056 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java @@ -8,6 +8,7 @@ import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.query.*; @@ -34,7 +35,8 @@ public NoOpVariantQueryExecutor(VariantStorageMetadataManager metadataManager, S } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); boolean sampleQuery = false; String sample = null; if (VariantQueryUtils.isValidParam(query, VariantQueryParam.GENOTYPE)) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java index 26d53e89e5c..49ddc339658 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java @@ -2,7 +2,6 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -65,12 +64,12 @@ public static void setDefaultTimeout(QueryOptions queryOptions, ObjectMap config /** * Determine if this VariantQueryExecutor can run the given query. - * @param query Query to execute + * @param variantQuery Query to execute * @param options Options for the query * @return True if this variant query executor is valid for the query * @throws StorageEngineException if there is an error */ - public abstract boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException; + public abstract boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException; protected abstract Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) throws StorageEngineException; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java index bdcd501ad88..95add22ec8e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java @@ -71,7 +71,14 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti } Map> sampleIdsMap = getIncludeSampleIds(query, options, includeStudies, metadataManager); - int numTotalSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); + int numTotalSamples; + if (isValidParam(query, NUM_TOTAL_SAMPLES)) { + // NUM_TOTAL_SAMPLES might have been defined in the PreProcess step. + // This implies that the current query has the samples already paginated. + numTotalSamples = query.getInt(NUM_TOTAL_SAMPLES.key()); + } else { + numTotalSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); + } skipAndLimitSamples(query, sampleIdsMap); int numSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java index 200b2eb463d..cfa794cbe10 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java @@ -7,6 +7,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; @@ -28,7 +29,8 @@ public SamplesSearchIndexVariantQueryExecutor(VariantDBAdaptor dbAdaptor, Varian } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); String samplesCollection = inferSpecificSearchIndexSamplesCollection(query, options, getMetadataManager(), dbName); return samplesCollection != null && searchActiveAndAlive(samplesCollection); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index 12c86cc4e2b..ccaae7dbe39 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -11,10 +11,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; -import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; -import org.opencb.opencga.storage.core.variant.adaptors.VariantField; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.adaptors.*; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; @@ -76,7 +73,8 @@ public SearchIndexVariantQueryExecutor setIntersectParamsThreshold(int intersect } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); return doQuerySearchManager(query, options) || doIntersectWithSearch(query, options); } @@ -185,7 +183,9 @@ public VariantQueryResult approximateCount(ParsedVariantQuery variantQuery DataResult nativeResult = searchManager .nativeQuery(dbName, searchEngineQuery, queryOptions); - List variantIds = nativeResult.getResults().stream().map(VariantSearchModel::getId).collect(Collectors.toList()); + List variantIds = nativeResult.getResults().stream() + .map(VariantSearchModel::toVariantSimple) + .collect(Collectors.toList()); // Adjust numSamples if the results from SearchManager is smaller than numSamples // If this happens, the count is not approximated if (variantIds.size() < sampling) { @@ -264,6 +264,10 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) { intersect = true; } else { if (options.getBoolean(QueryOptions.COUNT)) { + // The SearchIndex is better than anyone in terms of counting + intersect = true; + } else if (options.getInt(QueryOptions.SKIP, 0) > 500) { + // Large "skip" queries should use SearchIndex when possible intersect = true; } else { // TODO: Improve this heuristic @@ -285,12 +289,12 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) { return intersect; } - protected Iterator variantIdIteratorFromSearch(Query query) { + protected Iterator variantIdIteratorFromSearch(Query query) { return variantIdIteratorFromSearch(query, Integer.MAX_VALUE, 0, null); } - protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { - Iterator variantsIterator; + protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { + Iterator variantsIterator; QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, limit) .append(QueryOptions.SKIP, skip) @@ -304,14 +308,14 @@ protected Iterator variantIdIteratorFromSearch(Query query, int limit, i } variantsIterator = nativeResult.getResults() .stream() - .map(VariantSearchModel::getId) + .map(VariantSearchModel::toVariantSimple) .iterator(); } else { SolrNativeIterator nativeIterator = searchManager.nativeIterator(dbName, query, queryOptions); if (numTotalResults != null) { numTotalResults.set(nativeIterator.getNumFound()); } - variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::getId); + variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::toVariantSimple); } } catch (VariantSearchException | IOException e) { throw new VariantQueryException("Error querying " + VariantSearchManager.SEARCH_ENGINE_ID, e); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java index 9b0bb69792c..835af18a0a0 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java @@ -17,12 +17,15 @@ package org.opencb.opencga.storage.core.variant.search; import org.apache.solr.client.solrj.beans.Field; +import org.opencb.biodata.models.variant.Variant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter.HASH_PREFIX; + /** * Created by wasim on 09/11/16. */ @@ -140,6 +143,9 @@ public class VariantSearchModel { @Field("fileInfo_*") private Map fileInfo; + @Field("attr_*") + private Map attr; + public static final double MISSING_VALUE = -100.0; @@ -171,6 +177,7 @@ public VariantSearchModel() { this.qual = new HashMap<>(); this.filter = new HashMap<>(); this.fileInfo = new HashMap<>(); + this.attr = new HashMap<>(); } public VariantSearchModel(VariantSearchModel init) { @@ -210,6 +217,7 @@ public VariantSearchModel(VariantSearchModel init) { this.qual = init.getQual(); this.filter = init.getFilter(); this.fileInfo = init.getFileInfo(); + this.attr = init.getAttr(); } @Override @@ -251,6 +259,7 @@ public String toString() { sb.append(", qual=").append(qual); sb.append(", filter=").append(filter); sb.append(", fileInfo=").append(fileInfo); + sb.append(", attr=").append(attr); sb.append('}'); return sb.toString(); } @@ -259,6 +268,17 @@ public String getId() { return id; } + public Variant toVariantSimple() { + String variantId = getId(); + if (variantId.startsWith(HASH_PREFIX)) { + Object o = getAttr().get("attr_id"); + variantId = o instanceof String ? (String) o : ((List) o).get(0); + } + Variant variant = new Variant(variantId); + variant.setId(variantId); + return variant; + } + public VariantSearchModel setId(String id) { this.id = id; return this; @@ -579,4 +599,12 @@ public VariantSearchModel setFileInfo(Map fileInfo) { return this; } + public Map getAttr() { + return attr; + } + + public VariantSearchModel setAttr(Map attr) { + this.attr = attr; + return this; + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java index 6f55a892f7c..d9c119c2340 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java @@ -56,6 +56,7 @@ public class VariantSearchToVariantConverter implements ComplexTypeConverter includeFields; @@ -79,10 +80,9 @@ public VariantSearchToVariantConverter(Set includeFields) { @Override public Variant convertToDataModelType(VariantSearchModel variantSearchModel) { // set chromosome, start, end, ref, alt from ID - Variant variant = new Variant(variantSearchModel.getId()); + Variant variant = variantSearchModel.toVariantSimple(); - // set ID, chromosome, start, end, ref, alt, type - variant.setId(variantSearchModel.getVariantId()); + // set chromosome, start, end, ref, alt, type // set variant type if (StringUtils.isNotEmpty(variantSearchModel.getType())) { @@ -662,8 +662,10 @@ public VariantSearchModel convertToStorageType(Variant variant) { List other = new ArrayList<>(); // Set general Variant attributes: id, dbSNP, chromosome, start, end, type - variantSearchModel.setId(variant.toString()); // Internal unique ID e.g. 3:1000:AT:- - variantSearchModel.setVariantId(variant.getId()); + String variantId = getVariantId(variant); + variantSearchModel.setId(variantId); // Internal unique ID e.g. 3:1000:AT:- + variantSearchModel.setVariantId(variantId); + variantSearchModel.getAttr().put("attr_id", variant.toString()); variantSearchModel.setChromosome(variant.getChromosome()); variantSearchModel.setStart(variant.getStart()); variantSearchModel.setEnd(variant.getEnd()); @@ -1024,8 +1026,7 @@ public VariantSearchModel convertToStorageType(Variant variant) { // This field contains all possible IDs: id, dbSNP, names, genes, transcripts, protein, clinvar, hpo, ... // This will help when searching by variant id. This is added at the end of the method after collecting all IDs Set xrefs = variantAnnotationModelUtils.extractXRefs(variant.getAnnotation()); - xrefs.add(variantSearchModel.getId()); - xrefs.add(variantSearchModel.getVariantId()); + xrefs.add(variantId); if (variant.getNames() != null && !variant.getNames().isEmpty()) { variant.getNames().forEach(name -> { if (name != null) { @@ -1037,6 +1038,20 @@ public VariantSearchModel convertToStorageType(Variant variant) { return variantSearchModel; } + public static String getVariantId(Variant variant) { + String variantString = variant.toString(); + if (variantString.length() > 32766) { + // variantString.length() >= Short.MAX_VALUE + return hashVariantId(variant, variantString); + } else { + return variantString; + } + } + + public static String hashVariantId(Variant variant, String variantString) { + return HASH_PREFIX + variant.getChromosome() + ":" + variant.getStart() + ":" + Integer.toString(variantString.hashCode()); + } + private void convertStudies(Variant variant, VariantSearchModel variantSearchModel, List other) { // Sanity check if (CollectionUtils.isEmpty(variant.getStudies())) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java index e46c6379dc6..5365643b1fa 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java @@ -280,7 +280,9 @@ public static String inferSpecificSearchIndexSamplesCollection( Set samples = new HashSet<>(); if (isValidParam(query, VariantQueryParam.SAMPLE)) { String value = query.getString(VariantQueryParam.SAMPLE.key()); - samples.addAll(splitValue(value).getValue()); + for (String sample : splitValue(value).getValue()) { + samples.add(sample.split(IS)[0]); + } } if (isValidParam(query, VariantQueryParam.GENOTYPE)) { HashMap> map = new HashMap<>(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java index 0cf045ada4c..f12bb85a8b1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java @@ -25,7 +25,6 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.common.SolrException; import org.opencb.biodata.models.core.Region; -import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.solr.FacetQueryParser; @@ -35,7 +34,10 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.*; +import org.opencb.opencga.storage.core.variant.query.KeyOpValue; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.Values; +import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; import org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter; import org.slf4j.Logger; @@ -79,7 +81,7 @@ public class SolrQueryParser { static { includeMap = new HashMap<>(); - includeMap.put("id", "id,variantId"); + includeMap.put("id", "id,variantId,attr_id"); includeMap.put("chromosome", "chromosome"); includeMap.put("start", "start"); includeMap.put("end", "end"); @@ -477,7 +479,9 @@ private String parseGenomicFilter(Query query) { genes.addAll(variantQueryXref.getGenes()); xrefs.addAll(variantQueryXref.getIds()); xrefs.addAll(variantQueryXref.getOtherXrefs()); - xrefs.addAll(variantQueryXref.getVariants().stream().map(Variant::toString).collect(Collectors.toList())); + xrefs.addAll(variantQueryXref.getVariants().stream() + .map(VariantSearchToVariantConverter::getVariantId) + .collect(Collectors.toList())); // Regions if (StringUtils.isNotEmpty(query.getString(REGION.key()))) { @@ -492,7 +496,7 @@ private String parseGenomicFilter(Query query) { // Consequence types (cts) String ctLogicalOperator = " OR "; if (StringUtils.isNotEmpty(query.getString(ANNOT_CONSEQUENCE_TYPE.key(), ""))) { - consequenceTypes = Arrays.asList(query.getString(ANNOT_CONSEQUENCE_TYPE.key()).split("[,;]")); + consequenceTypes = parseConsequenceTypes(Arrays.asList(query.getString(ANNOT_CONSEQUENCE_TYPE.key()).split("[,;]"))); if (query.getString(ANNOT_CONSEQUENCE_TYPE.key()).contains(";")) { ctLogicalOperator = " AND "; // TODO This must be removed as soon as we have the Query procesing in use @@ -1616,15 +1620,12 @@ private String[] includeFieldsWithMandatory(String[] includes) { return new String[0]; } - String[] mandatoryIncludeFields = new String[]{"id", "chromosome", "start", "end", "type"}; - String[] includeWithMandatory = new String[includes.length + mandatoryIncludeFields.length]; - for (int i = 0; i < includes.length; i++) { - includeWithMandatory[i] = includes[i]; - } - for (int i = 0; i < mandatoryIncludeFields.length; i++) { - includeWithMandatory[includes.length + i] = mandatoryIncludeFields[i]; - } - return includeWithMandatory; + Set mandatoryIncludeFields = new HashSet<>(Arrays.asList("id", "attr_id", "chromosome", "start", "end", "type")); + Set includeWithMandatory = new LinkedHashSet<>(includes.length + mandatoryIncludeFields.size()); + + includeWithMandatory.addAll(Arrays.asList(includes)); + includeWithMandatory.addAll(mandatoryIncludeFields); + return includeWithMandatory.toArray(new String[0]); } /** diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java index 2aacde2e3ce..30e4bba2e5f 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java @@ -9,6 +9,7 @@ import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.metadata.VariantMetadata; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.io.managers.LocalIOConnector; @@ -45,6 +46,7 @@ public class VariantMetadataConverterTest { @Before public void setUp() throws Exception { metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + projectMetadata = metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); URI uri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); variantReaderUtils = new VariantReaderUtils(new IOConnectorProvider(LocalIOConnector.class)); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java index 1b3311958f4..71ea72de3c0 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java @@ -4,12 +4,14 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageTest; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -103,4 +105,36 @@ public List getTasks(StudyMetadata study, List stat .map(TaskMetadata::getName) .collect(Collectors.toList()); } + + @Test + public void testAddSampleToCohort() throws Exception { + StudyMetadata study = metadataManager.createStudy("study"); + + metadataManager.registerCohort(study.getName(), "cohort1", Collections.emptyList()); + + int numSamples = 100; + List sampleIds = new ArrayList<>(numSamples); + for (int i = 0; i < numSamples; i++) { + sampleIds.add(metadataManager.registerSample(study.getId(), null, "sample_" + i)); + } + + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 10)); + VariantStorageMetadataManager metadataManager = Mockito.spy(this.metadataManager); + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.times(1)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + Mockito.reset(metadataManager); + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.never()).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.never()).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 12)); + Mockito.verify(metadataManager, Mockito.times(1)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + Mockito.reset(metadataManager); + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 6)); + Mockito.verify(metadataManager, Mockito.times(6)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + } + } \ No newline at end of file diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java index fe5313bdc80..23ece394c79 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java @@ -374,6 +374,11 @@ public static StoragePipelineResult runDefaultETL(URI inputUri, VariantStorageEn return storagePipelineResult; } + public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, + ObjectMap params) throws StorageEngineException, FileFormatException, IOException { + return runETL(variantStorageManager, inputUri, outputUri, params, true, true, true); + } + public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, ObjectMap params, boolean doExtract, diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index 32e1657fcb8..19c23958e47 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -48,6 +48,8 @@ public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + if (!loaded) { clearDB(DB_NAME); loadFiles(); @@ -59,6 +61,9 @@ protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.NORMALIZATION_EXTENSIONS.key(), ParamConstants.NONE); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + studyMetadata = new StudyMetadata(1, "s1"); // variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); input1 = getResourceUri("variant-test-bnd.vcf"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 64fc14c4c6a..404411d09ce 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -1,5 +1,6 @@ package org.opencb.opencga.storage.core.variant; +import org.junit.Assume; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -7,7 +8,6 @@ import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.SampleEntry; -import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.commons.datastore.core.Query; @@ -20,6 +20,10 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutor; +import org.opencb.opencga.storage.core.variant.search.SearchIndexVariantQueryExecutor; import java.net.URI; import java.nio.file.Paths; @@ -53,21 +57,24 @@ public abstract class VariantStorageEngineSVTest extends VariantStorageBaseTest public void before() throws Exception { if (!loaded) { clearDB(DB_NAME); + } + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.getOptions().put(VariantStorageOptions.NORMALIZATION_EXTENSIONS.key(), ParamConstants.NONE); + variantStorageEngine.reloadCellbaseConfiguration(); + + if (!loaded) { loadFiles(); loaded = true; } } protected void loadFiles() throws Exception { - variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); - variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); - variantStorageEngine.reloadCellbaseConfiguration(); input1 = getResourceUri("variant-test-sv.vcf"); studyMetadata = new StudyMetadata(1, "s1"); - variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); pipelineResult1 = runDefaultETL(input1, variantStorageEngine, studyMetadata, new QueryOptions() .append(VariantStorageOptions.ANNOTATE.key(), true) .append(VariantStorageOptions.STATS_CALCULATE.key(), true) @@ -106,6 +113,19 @@ public void checkCount() throws Exception { assertEquals(expected, count); } + @Test + public void checkSecondaryAnnotationIndex() throws Exception { + Assume.assumeTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + VariantQueryExecutor variantQueryExecutor = variantStorageEngine.getVariantQueryExecutor(SearchIndexVariantQueryExecutor.class); + for (Variant variant : variantStorageEngine) { + ParsedVariantQuery query = variantStorageEngine + .parseQuery(new Query(VariantQueryParam.ID.key(), variant.toString()), new QueryOptions()); + VariantQueryResult result = variantQueryExecutor.get(query); + assertEquals(1, result.getNumResults()); + assertEquals(variant.toString(), result.first().toString()); + } + } + @Test public void checkCorrectnessFile1() throws Exception { checkCorrectness(VariantStorageEngineSVTest.input1); @@ -149,12 +169,7 @@ protected void checkCorrectness(URI file) throws StorageEngineException, NonStan actualStudyEntry.getFiles().get(0).setFileId(""); assertEquals(expectedStudyEntry.getFiles().get(0), actualStudyEntry.getFiles().get(0)); - - if (actual.getAlternate().equals("") || actual.getType().equals(VariantType.BREAKEND)) { - System.err.println("WARN: Variant " + actual + (actual.getAnnotation() == null ? " without annotation" : " with annotation")); - } else { - assertNotNull(actual.toString(), actual.getAnnotation()); - } + assertNotNull(actual.toString(), actual.getAnnotation()); } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java index 905f64a5373..ee382b95557 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java @@ -10,6 +10,7 @@ import org.opencb.biodata.models.variant.avro.SampleEntry; import org.opencb.biodata.models.variant.stats.VariantStats; import org.opencb.commons.datastore.core.*; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; @@ -78,6 +79,7 @@ protected void load() throws Exception { VariantStorageEngine storageEngine = getVariantStorageEngine(); ObjectMap options = getOptions(); options.put(VariantStorageOptions.STATS_CALCULATE.key(), true); + options.put(VariantStorageOptions.NORMALIZATION_EXTENSIONS.key(), ParamConstants.NONE); int maxStudies = 2; int studyId = 1; @@ -377,18 +379,22 @@ public void testSampleLimitSkip() throws Exception { VariantQueryResult result = query(new Query(SAMPLE_METADATA.key(), true).append(VariantQueryParam.INCLUDE_SAMPLE.key(), ALL), options); System.out.println("samples(ALL) = " + result.getSamples()); - for (int i : new int[]{1, 3, 6, 8, 10}) { + int numSamples = metadataManager.getStudyIds().stream().mapToInt(id -> metadataManager.getIndexedSamples(id).size()).sum(); + assertEquals(8, numSamples); + for (int i : new int[]{1, 3, 6, numSamples, 10}) { result = query(new VariantQuery().sampleSkip(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(SKIP=" + i + ") = " + result.getSamples()); - assertEquals(Math.max(0, 8 - i), result.getSamples().values().stream().mapToInt(List::size).sum()); - assertEquals(Math.max(0, 8 - i), result.getNumSamples().intValue()); - assertEquals(8, result.getNumTotalSamples().intValue()); + int expected = Math.max(0, numSamples - i); + assertEquals("Skip = " + i + " , expected " + expected + " out of 8 samples", expected, result.getSamples().values().stream().mapToInt(List::size).sum()); + assertEquals("Skip = " + i + " , expected " + expected + " out of 8 samples", expected, result.getNumSamples().intValue()); + assertEquals(numSamples, result.getNumTotalSamples().intValue()); result = query(new VariantQuery().sampleLimit(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(LIMIT=" + i + ") = " + result.getSamples()); - assertEquals(Math.min(8, i), result.getSamples().values().stream().mapToInt(List::size).sum()); - assertEquals(Math.min(8, i), result.getNumSamples().intValue()); - assertEquals(8, result.getNumTotalSamples().intValue()); + expected = Math.min(numSamples, i); + assertEquals("Limit = " + i + " , expected " + expected + " out of 8 samples", expected, result.getSamples().values().stream().mapToInt(List::size).sum()); + assertEquals("Limit = " + i + " , expected " + expected + " out of 8 samples", expected, result.getNumSamples().intValue()); + assertEquals(numSamples, result.getNumTotalSamples().intValue()); } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java index 6194d25c6d8..cdc96e15377 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java @@ -2,16 +2,22 @@ import com.google.common.base.Throwables; import org.junit.ClassRule; +import org.junit.Test; import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager.SEARCH_ENGINE_ID; + /** * Created on 22/12/17. * @@ -87,4 +93,32 @@ public DataResult rank(int limit, Query query, String field, boolean asc) { throw Throwables.propagate(e); } } + + @Test + public void testQueryExecutor() { + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 0)).getSource(), + not(containsString(SEARCH_ENGINE_ID))); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, true) + .append(QueryOptions.SKIP, 0)).getSource(), + containsString(SEARCH_ENGINE_ID)); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 100)).getSource(), + not(containsString(SEARCH_ENGINE_ID))); + + assertThat(variantStorageEngine.get(new VariantQuery(), new QueryOptions() + .append(QueryOptions.LIMIT, 10) + .append(QueryOptions.COUNT, false) + .append(QueryOptions.SKIP, 1000)).getSource(), + containsString(SEARCH_ENGINE_ID)); + + } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java index bed8d419666..3ba92ed7f1c 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java @@ -16,7 +16,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; /** @@ -29,7 +28,7 @@ public class DummyProjectMetadataAdaptor implements ProjectMetadataAdaptor { private static Map counters = new HashMap<>(); @Override - public Lock lockProject(long lockDuration, long timeout, String lockName) throws InterruptedException, TimeoutException { + public Lock lockProject(long lockDuration, long timeout, String lockName) { return new Lock(0) { @Override public void unlock0() { @@ -43,17 +42,12 @@ public void refresh() { }; } - @Override - public void unLockProject(long lockId) { - } - @Override public synchronized DataResult getProjectMetadata() { final DataResult result = new DataResult<>(); - if (projectMetadata == null) { - projectMetadata = new ProjectMetadata("hsapiens", "grch37", 1); + if (projectMetadata != null) { + result.setResults(Collections.singletonList(projectMetadata.copy())); } - result.setResults(Collections.singletonList(projectMetadata.copy())); return result; } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java index 55866e24160..55903d221f2 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java @@ -84,6 +84,10 @@ public static void configure(StorageEngineFactory factory, boolean clear) { } } + public static VariantStorageMetadataManager getVariantMetadataManager() { + return new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + } + @Override public String getStorageEngineId() { return STORAGE_ENGINE_ID; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java index 221a126cd92..38f995c823d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java @@ -23,11 +23,13 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.metadata.VariantFileHeader; import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.io.DataWriter; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantDBAdaptor; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageMetadataDBAdaptorFactory; @@ -85,8 +87,10 @@ public void testContigLengthNull() throws IOException, StorageEngineException { new VariantFileHeaderComplexLine("contig", "chr3", null, null, null, Collections.singletonMap("length", ".")), new VariantFileHeaderComplexLine("contig", "chr4", null, null, null, Collections.singletonMap("length", "1234")) )); - StudyMetadata study = dbAdaptor.getMetadataManager().createStudy("study"); - dbAdaptor.getMetadataManager().unsecureUpdateStudyMetadata(study.setVariantHeader(header)); + VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager(); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); + StudyMetadata study = metadataManager.createStudy("study"); + metadataManager.unsecureUpdateStudyMetadata(study.setVariantHeader(header)); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(10000); DataWriter writer = new VariantWriterFactory(dbAdaptor).newDataWriter( VariantWriterFactory.VariantOutputFormat.VCF, diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java index 1fc7617422c..f5e4c2526b9 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java @@ -12,6 +12,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -88,6 +89,9 @@ public void setUp() throws Exception { fileIndexed = true; Integer indexedFileId = metadataManager.getIndexedFiles(studyMetadata.getId()).iterator().next(); + Trio trio = new Trio("NA19660", "NA19661", "NA19685"); + variantStorageEngine.familyIndex(studyMetadata.getName(), Collections.singletonList(trio), new ObjectMap()); + //Calculate stats QueryOptions options = new QueryOptions(VariantStorageOptions.STUDY.key(), STUDY_NAME) .append(VariantStorageOptions.LOAD_BATCH_SIZE.key(), 100) @@ -206,23 +210,53 @@ public void testXRefRs() throws StorageEngineException { } } + @Test + public void testCompHetQuery() throws StorageEngineException { +// Matcher matcher = allOf( +// anyOf( +// samePosition(new Variant("1:2441358:T:C")), +// samePosition(new Variant("1:2458010:G:C")), +// samePosition(new Variant("19:501725:G:A")), +// samePosition(new Variant("19:501900:C:A"))), +// withStudy(STUDY_NAME, withSampleGt("NA19685"))); + Matcher matcher = null; + testQuery(new VariantQuery().sample("NA19685:compoundheterozygous") + .study(STUDY_NAME) + .biotype("protein_coding"), + new QueryOptions(), + matcher, + false); + } + public VariantQueryResult testQuery(Query query, QueryOptions options, Matcher matcher) throws StorageEngineException { + return testQuery(query, options, matcher, true); + } + + public VariantQueryResult testQuery(Query query, QueryOptions options, Matcher matcher, boolean expectDBAdaptorExecutor) + throws StorageEngineException { logger.info(""); logger.info(""); logger.info("####################################################"); logger.info("########## TEST QUERY :" + query.toJson()); logger.info("####################################################"); logger.info("## Allowed VariantQueryExecutors:"); + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { - if (variantQueryExecutor.canUseThisExecutor(query, options)) { + if (variantQueryExecutor.canUseThisExecutor(variantQuery, options)) { logger.info("## - " + variantQueryExecutor.getClass().getSimpleName()); } } - logger.info("## Using DBAdaptorVariantQueryExecutor for expected results"); - Assert.assertTrue(dbQueryExecutor.canUseThisExecutor(query, options)); + VariantQueryResult expected; + if (expectDBAdaptorExecutor) { + logger.info("## Using DBAdaptorVariantQueryExecutor for expected results"); + Assert.assertTrue(dbQueryExecutor.canUseThisExecutor(variantQuery, options)); - ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); - VariantQueryResult expected = dbQueryExecutor.get(variantQuery); + expected = dbQueryExecutor.get(variantQuery); + } else { + logger.info("## DBAdaptorVariantQueryExecutor can not be used for expected results"); + Assert.assertFalse(dbQueryExecutor.canUseThisExecutor(variantQuery, options)); + expected = null; + } VariantQueryResult unfilteredResult = null; VariantQueryResult result = null; @@ -261,21 +295,24 @@ public VariantQueryResult testQuery(Query query, QueryOptions options, QueryOptions emptyOptions = new QueryOptions(); emptyOptions.putIfNotEmpty(QueryOptions.INCLUDE, options.getString(QueryOptions.INCLUDE)); emptyOptions.putIfNotEmpty(QueryOptions.EXCLUDE, options.getString(QueryOptions.EXCLUDE)); + logger.info("## unfiltered query " + VariantQueryUtils.printQuery(emptyQuery)); + logger.info("## unfiltered options " + emptyOptions.toJson()); unfilteredResult = dbQueryExecutor.get(variantStorageEngine.parseQuery(emptyQuery, emptyOptions)); } for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { - if (variantQueryExecutor.canUseThisExecutor(query, options)) { + if (variantQueryExecutor.canUseThisExecutor(variantQuery, options)) { logger.info(""); logger.info("###################"); logger.info("### Testing " + variantQueryExecutor.getClass().getSimpleName()); result = variantQueryExecutor.get(variantQuery); logger.info("### Num results : " + result.getNumResults()); logger.info("###################"); - expected.getResults().sort(Comparator.comparing(Variant::toString)); - result.getResults().sort(Comparator.comparing(Variant::toString)); - Assert.assertEquals(expected.getResults(), result.getResults()); - + if (expected != null) { + expected.getResults().sort(Comparator.comparing(Variant::toString)); + result.getResults().sort(Comparator.comparing(Variant::toString)); + Assert.assertEquals(expected.getResults(), result.getResults()); + } assertThat(result, numResults(gt(0))); if (matcher != null) { diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java index 5839db1745b..539ea8c22d6 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java @@ -49,7 +49,7 @@ public void test() throws Exception { expectedVariant.addStudyEntry(aux.getStudy("2")); VariantSearchModel variantSearchModel = converter.convertToStorageType(expectedVariant); - assertNull(variantSearchModel.getVariantId()); + assertNotNull(variantSearchModel.getVariantId()); assertEquals(variantId, variantSearchModel.getId()); Variant actualVariant = converter.convertToDataModelType(variantSearchModel); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java index d46cafe43df..2b1c3c3ba08 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java @@ -43,8 +43,8 @@ public class SolrQueryParserTest { private String studyName = "platinum"; private String flBase = "fl=other,geneToSoAcc,traits,type,soAcc,score_*,sift,passStats_*,caddRaw,biotypes,polyphenDesc,studies,end,id,variantId," - + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,siftDesc," - + "phastCons,phylop,altStats_*,id,chromosome,start,end,type"; + + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,attr_id,siftDesc," + + "phastCons,phylop,altStats_*"; private String flDefault1 = flBase + ",fileInfo__*,qual__*,filter__*,sampleFormat__*"; private String flDefaultStudy = flBase + ",fileInfo__" + studyName + "__*,qual__" + studyName + "__*," + "filter__" + studyName + "__*,sampleFormat__" + studyName + "__*"; diff --git a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf index 9fc4fe9c325..6c1d32cf79a 100644 --- a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf +++ b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf @@ -44,7 +44,7 @@ 1 600000 . T . PASS SVTYPE=DEL;END=600250;SVLEN=-205;CIPOS=-56,20;CIEND=-10,62 GT 0/1 0/1 1 650000 . C . PASS SVTYPE=DEL;END=650100;CIEND=0,48;CIPOS=-49,0 GT 0/1 0/1 1 700000 . C . PASS SVTYPE=DEL;END=700297;SVLEN=-297;CIPOS=-22,18;CIEND=-12,32 GT 0/1 0/1 -1 800000 . A . PASS SVTYPE=INS;END=800000;SVLEN=6027;CIPOS=-16,22;RIGHT_SVINSSEQ=ACCACACCCACACAACACACA;LEFT_SVINSSEQ=TGTGGTGTGTGTGGTGTG GT 0/1 0/1 +1 800000 . A . PASS SVTYPE=INS;END=800000;SVLEN=6027;CIPOS=-16,22;CIEND=-16,22;RIGHT_SVINSSEQ=ACCACACCCACACAACACACA;LEFT_SVINSSEQ=TGTGGTGTGTGTGGTGTG GT 0/1 0/1 1 850000 . A . PASS SVTYPE=INS;END=850000;SVINSSEQ=ACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACA GT 0/1 0/1 1 860000 . A . PASS SVTYPE=INVERSION;END=870000 GT 0/1 0/1 1 900000 . G . PASS SVTYPE=INS;END=900000;SVLEN=6027;CIPOS=-16,22 GT 0/1 0/1 diff --git a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf index 32ad6f4c4f0..0cfa097d85f 100644 --- a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf +++ b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf @@ -56,4 +56,4 @@ 1 1500000 . C ,, . PASS SVTYPE=CNV;CIEND=-150,150;CIPOS=-150,150;END=1504000 GT 0/1 2/3 1 1600000 . T G . PASS . GT 0/1 0/1 2 100001 BND_V T ]chr13:300000]T . PASS SVTYPE=BND GT 0/1 0/1 -13 300000 BND_U C C[chr2:100001[ . PASS SVTYPE=BND;CIPOS=-150,0 GT 0/1 0/1 \ No newline at end of file +13 300000 BND_U C C[chr2:100001[ . PASS SVTYPE=BND;CIPOS=-150,0;CIEND=-150,0 GT 0/1 0/1 \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml index 8cdb1ff4d43..3b408a1127e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -43,6 +43,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java index 4bfb6cd45fd..48ab8846f65 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java @@ -1,9 +1,13 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.List; public abstract class HBaseCompatApi { @@ -28,4 +32,8 @@ public static HBaseCompatApi getInstance() { public abstract void available(Configuration configuration) throws IOException; public abstract boolean isSolrTestingAvailable(); + + public abstract List getServerList(Admin admin) throws IOException; + + public abstract byte[][] getTableStartKeys(Admin admin, Table table) throws IOException; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml index 16b0a3dc68a..ce30f130cf2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -49,6 +49,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index f3f9170f6f7..bb74dabf4c6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -1,11 +1,18 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class HBaseCompat extends HBaseCompatApi { @@ -24,4 +31,18 @@ public PhoenixCompatApi getPhoenixCompat() { return new PhoenixCompat(); } + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterStatus().getServers()); + } + + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + List regions = admin.getRegions(table.getName()); + regions.sort((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())); + byte[][] startKeys = new byte[regions.size()][]; + for (int i = 0; i < regions.size(); i++) { + startKeys[i] = regions.get(i).getStartKey(); + } + return startKeys; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml index 199c4b18590..145210cc55b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -49,6 +49,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index 451238f6da5..f7cf534508a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -1,11 +1,18 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class HBaseCompat extends HBaseCompatApi { @@ -22,4 +29,20 @@ public boolean isSolrTestingAvailable() { public PhoenixCompatApi getPhoenixCompat() { return new PhoenixCompat(); } + + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterMetrics().getServersName()); + } + + @Override + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + List regions = admin.getRegions(table.getName()); + regions.sort((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())); + byte[][] startKeys = new byte[regions.size()][]; + for (int i = 0; i < regions.size(); i++) { + startKeys[i] = regions.get(i).getStartKey(); + } + return startKeys; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml index 52e5cd4c764..e78bfbe0823 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -49,6 +49,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.hadoop hadoop-common diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java index 93f61922b9f..194b47b7794 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -1,11 +1,16 @@ package org.opencb.opencga.storage.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Table; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class HBaseCompat extends HBaseCompatApi { @@ -24,4 +29,13 @@ public PhoenixCompatApi getPhoenixCompat() { return new PhoenixCompat(); } + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterMetrics().getServersName()); + } + + @Override + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + return table.getRegionLocator().getStartKeys(); + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml index 99ab1f37bc5..05bdba53885 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -45,6 +45,12 @@ ${hbase.version} provided + + org.apache.hbase + hbase-common + ${hbase.version} + provided + org.apache.phoenix phoenix-core diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java index 509a758f268..7e80e787dea 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java @@ -3,18 +3,19 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.tools.ant.types.Commandline; import org.opencb.commons.ProgressLogger; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; @@ -78,9 +79,15 @@ public void run(String[] args) throws Exception { case MOVE_TABLE_REGIONS: moveTableRegions(args); break; - case BALANCE_TABLE_REGIONS: - balanceTableRegions(getArg(args, 1), getArgsMap(args, 2, "maxMoves")); + case BALANCE_TABLE_REGIONS: { + ObjectMap argsMap = getArgsMap(args, 2, "maxMoves", "dryRun", "ignoreExceptions", "maxRetries"); + balanceTableRegions(getArg(args, 1), + argsMap.getInt("maxMoves", 50000), + argsMap.getBoolean("dryRun", false), + argsMap.getBoolean("ignoreExceptions", false), + argsMap.getInt("maxRetries", 1)); break; + } case "tables": case LIST_TABLES: print(listTables(getArg(args, 1, "")).stream().map(TableName::getNameWithNamespaceInclAsString).iterator()); @@ -197,8 +204,10 @@ public void run(String[] args) throws Exception { System.out.println(" (see " + CHECK_TABLES_WITH_REGIONS_ON_DEAD_SERVERS + ") by creating a temporary snapshot"); System.out.println(" " + MOVE_TABLE_REGIONS + " "); System.out.println(" Move all regions from selected tables to new random nodes."); -// System.out.println(" " + BALANCE_TABLE_REGIONS + " [--maxMoves N]"); // FIXME -// System.out.println(" " + REGIONS_PER_TABLE + " "); // FIXME + System.out.println(" " + BALANCE_TABLE_REGIONS + " [--maxMoves N]" + + " [--maxRetries 1] [--ignoreExceptions]" + + " [--dryRun]"); + System.out.println(" " + REGIONS_PER_TABLE + " "); // FIXME System.out.println(" " + CLONE_TABLES + " " + "[--keepSnapshots] [--dryRun] [--snapshotSuffix ]"); System.out.println(" Clone all selected tables by creating an intermediate snapshot."); @@ -207,8 +216,8 @@ public void run(String[] args) throws Exception { + "[--skipTablesWithSnapshot]"); System.out.println(" " + SNAPSHOT_TABLE + " [--dryRun] [--snapshotSuffix ] " + "[--skipTablesWithSnapshot]"); - System.out.println(" " + DELETE_SNAPSHOTS + " [--dryRun] [--skipMissing]"); System.out.println(" Create a snapshot for all selected tables."); + System.out.println(" " + DELETE_SNAPSHOTS + " [--dryRun] [--skipMissing]"); System.out.println(" " + CLONE_SNAPSHOTS + " [--dryRun] " + "[--tablePrefixChange :] " + "[--onExistingTables [fail|skip|drop] ]"); @@ -326,47 +335,38 @@ private void exportSnapshot(String storageConfigurationPath, String snapshot, St } private void regionsPerTable(String tableNameStr) throws Exception { -// TableName tableName = getTable(tableNameStr); -// hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { -// List servers = new ArrayList<>(admin.getClusterStatus().getServers()); -// Map regionsPerServer = new HashMap<>(); -// -// List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); -// -// System.out.println("#REGION\tSERVER\tSTART_KEY\tEND_KEY"); -// for (Pair pair : tableRegionsAndLocations) { -// RegionInfo region = pair.getFirst(); -// ServerName server = pair.getSecond(); -// regionsPerServer.merge(server.getServerName(), 1, Integer::sum); -// -// System.out.println(region.getEncodedName() -// + "\t" + server.getServerName() -// + "\t" + Bytes.toStringBinary(region.getStartKey()) -// + "\t" + Bytes.toStringBinary(region.getEndKey())); -// } -// -// System.out.println(""); -// System.out.println("#SERVER\tREGIONS"); -// for (ServerName server : servers) { -// System.out.println(server.getServerName() + "\t" + regionsPerServer.getOrDefault(server.getServerName(), 0)); -// } -// -// -// -// return null; -// }); - } - -// private List> getTableRegionsAndLocations(TableName tableName, Admin admin) throws IOException { -// List> tableRegionsAndLocations; -//// try (ZooKeeperWatcher zkw = new ZooKeeperWatcher(admin.getConfiguration(), "hbase-main", null)) { -//// tableRegionsAndLocations = MetaTableAccessor -//// .getTableRegionsAndLocations(zkw, admin.getConnection(), tableName); -//// } -// tableRegionsAndLocations = MetaTableAccessor -// .getTableRegionsAndLocations(admin.getConnection(), tableName); -// return tableRegionsAndLocations; -// } + TableName tableName = getTable(tableNameStr); + hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { + List servers = new ArrayList<>(admin.getClusterStatus().getServers()); + Map regionsPerServer = new HashMap<>(); + + List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); + + System.out.println("#REGION\tSERVER\tSTART_KEY\tEND_KEY"); + for (Pair pair : tableRegionsAndLocations) { + RegionInfo region = pair.getFirst(); + ServerName server = pair.getSecond(); + regionsPerServer.merge(server.getServerName(), 1, Integer::sum); + + System.out.println(region.getEncodedName() + + "\t" + server.getServerName() + + "\t" + Bytes.toStringBinary(region.getStartKey()) + + "\t" + Bytes.toStringBinary(region.getEndKey())); + } + + System.out.println(""); + System.out.println("#SERVER\tREGIONS"); + for (ServerName server : servers) { + System.out.println(server.getServerName() + "\t" + regionsPerServer.getOrDefault(server.getServerName(), 0)); + } + + return null; + }); + } + + private List> getTableRegionsAndLocations(TableName tableName, Admin admin) throws IOException { + return MetaTableAccessor.getTableRegionsAndLocations(admin.getConnection(), tableName); + } private void reassignTablesWithRegionsOnDeadServers(String[] args) throws Exception { String tableNameFilter = getArg(args, 1); @@ -415,47 +415,81 @@ private void moveTableRegions(String[] args) throws Exception { }); } - private void balanceTableRegions(String tableNameStr, ObjectMap options) throws Exception { -// TableName tableName = getTable(tableNameStr); -// -// int regionCount = hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { -// int maxMoves = options.getInt("maxMoves", 50000); -// List servers = new ArrayList<>(admin.getClusterStatus().getServers()); -// List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); -// int expectedRegionsPerServer = (tableRegionsAndLocations.size() / servers.size()) + 1; -// Map regionsPerServer = new HashMap<>(); -// servers.forEach(s -> regionsPerServer.put(s.getServerName(), 0)); -// for (Pair pair : tableRegionsAndLocations) { -// regionsPerServer.merge(pair.getSecond().getServerName(), 1, Integer::sum); -// } -// -// for (Pair pair : tableRegionsAndLocations) { -// if (maxMoves < 0) { -// System.out.println("Reached max moves!"); -// break; -// } -// -// String sourceHost = pair.getSecond().getServerName(); -// if (regionsPerServer.get(sourceHost) > expectedRegionsPerServer) { -// Collections.shuffle(servers); -// Optional targetOptional = servers.stream() -// .filter(s -> regionsPerServer.get(s.getServerName()) < expectedRegionsPerServer).findAny(); -// if (!targetOptional.isPresent()) { -// break; -// } -// String testHost = targetOptional.get().getServerName(); -// regionsPerServer.merge(sourceHost, -1, Integer::sum); -// regionsPerServer.merge(testHost, 1, Integer::sum); -// System.out.println("Move region '" + pair.getFirst().getEncodedName() + "' from " + sourceHost + " to " + testHost); -// StopWatch stopWatch = StopWatch.createStarted(); -// admin.move(pair.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(testHost)); -// System.out.println("Moved in "+TimeUtils.durationToString(stopWatch)); -// -// maxMoves--; -// } -// } -// return tableRegionsAndLocations.size(); -// }); + private void balanceTableRegions(String tableNameStr, int maxMoves, boolean dryRun, boolean ignoreExceptions, int maxRetries) + throws Exception { + TableName tableName = getTable(tableNameStr); + + LOGGER.info("Balancing table " + tableName.getNameAsString() + " with maxMoves=" + maxMoves + ", dryRun=" + dryRun + + ", ignoreExceptions=" + ignoreExceptions + ", maxRetries=" + maxRetries); + int regionCount = hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { + List servers = HBaseCompat.getInstance().getServerList(admin); + List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); + int expectedRegionsPerServer = (tableRegionsAndLocations.size() / servers.size()) + 1; + + Map regionsPerServer = new HashMap<>(); + servers.forEach(s -> regionsPerServer.put(s.getServerName(), 0)); + for (Pair pair : tableRegionsAndLocations) { + regionsPerServer.merge(pair.getSecond().getServerName(), 1, Integer::sum); + } + + // Shuffle the regions to avoid hotspots + Collections.shuffle(tableRegionsAndLocations); + + int moves = 0; + for (Pair pair : tableRegionsAndLocations) { + if (moves > maxMoves) { + LOGGER.info("Reached max moves!"); + break; + } + + ServerName sourceHost = pair.getSecond(); + // If the source host has more regions than expected, move one to another server + if (regionsPerServer.get(sourceHost.getServerName()) > expectedRegionsPerServer) { + // Iterate over the servers in a random order + Collections.shuffle(servers); + Optional targetOptional = servers.stream() + .filter(s -> !s.equals(sourceHost)) + .filter(s -> regionsPerServer.get(s.getServerName()) < expectedRegionsPerServer) + .findAny(); + if (!targetOptional.isPresent()) { + break; + } + ServerName targetHost = targetOptional.get(); + LOGGER.info("Move region '" + pair.getFirst().getEncodedName() + "' from " + sourceHost + " to " + targetHost); + StopWatch stopWatch = StopWatch.createStarted(); + int attempts = 0; + while (attempts < maxRetries) { + try { + if (dryRun) { + LOGGER.info("[DRY-RUN]: admin.move('" + pair.getFirst().getEncodedName() + "," + targetHost + ")"); + } else { + admin.move(pair.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(targetHost.getServerName())); + } + break; + } catch (Exception e) { + LOGGER.info("Error moving region: " + e.getMessage()); + attempts++; + if (attempts < maxRetries) { + LOGGER.info("Retrying... " + attempts + "/" + maxRetries); + } else if (!ignoreExceptions) { + throw e; + } else { + LOGGER.info("Ignoring exception. Unable to move region after " + attempts + " attempts."); + break; + } + } + } + LOGGER.info("Moved in " + TimeUtils.durationToString(stopWatch)); + + regionsPerServer.merge(sourceHost.getServerName(), -1, Integer::sum); + regionsPerServer.merge(targetHost.getServerName(), 1, Integer::sum); + + moves++; + } + } + return tableRegionsAndLocations.size(); + }); + System.out.println("#Balanced regions for table '" + tableNameStr + "' . Total regions: " + regionCount); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java index 52879f01eed..53473a977b2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java @@ -81,10 +81,15 @@ public void setupJob(Job job, String table) throws IOException { List scans; if (!regions.isEmpty()) { - scans = new ArrayList<>(regions.size() / 2); + LOGGER.info("Delete rows from {} table ranges (start - end)", regions.size()); + scans = new ArrayList<>(regions.size()); for (Pair region : regions) { Scan scan = new Scan(templateScan); scans.add(scan); + LOGGER.info(" - [ '" + + Bytes.toStringBinary(region.getFirst()) + "' , '" + + Bytes.toStringBinary(region.getSecond()) + + "' )"); if (region.getFirst() != null && region.getFirst().length != 0) { scan.setStartRow(region.getFirst()); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java index 6fc7609b998..b1f827b2baf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.solr.common.StringUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.metadata.models.Lock; import org.slf4j.Logger; @@ -134,18 +133,17 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) // Minimum lock duration of 100ms lockDuration = Math.max(lockDuration, 100); - byte[] lockValue; - String readToken = ""; + LockToken lockToken; StopWatch stopWatch = new StopWatch(); stopWatch.start(); do { - lockValue = readLockValue(row, column); + lockToken = readLockToken(row, column); // If the lock is taken, wait - while (isLockTaken(lockValue)) { + while (lockToken.isTaken()) { Thread.sleep(100); - lockValue = readLockValue(row, column); + lockToken = readLockToken(row, column); //Check if the lock is still valid if (stopWatch.getTime() > timeout) { throw new TimeoutException("Unable to get the lock"); @@ -157,19 +155,19 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) } // Try to lock cell - if (tryToPutToken(token, lockDuration, row, column, lockValue, CURRENT)) { - readToken = parseValidLockToken(readLockValue(row, column)); + if (tryToPutToken(token, lockDuration, row, column, lockToken, CURRENT)) { + lockToken = readLockToken(row, column); } - // You win the lock if the first available lock is yours. - } while (!token.equals(readToken)); + // You win the lock if you manage to write your lock. + } while (!lockToken.equals(token)); - boolean prevTokenExpired = lockValue != null && lockValue.length > 0; + boolean prevTokenExpired = !lockToken.isEmpty() && lockToken.isExpired(); boolean slowQuery = stopWatch.getTime() > 60000; if (prevTokenExpired || slowQuery) { StringBuilder msg = new StringBuilder("Lock column '").append(Bytes.toStringBinary(column)).append("'"); if (prevTokenExpired) { - long expireDate = parseExpireDate(lockValue); + long expireDate = lockToken.getExpireDate(); msg.append(". Previous token expired ") .append(TimeUtils.durationToString(System.currentTimeMillis() - expireDate)) .append(" ago"); @@ -181,105 +179,65 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) logger.warn(msg.toString()); } - long tokenHash = token.hashCode(); - logger.debug("Won the lock with token " + token + " (" + tokenHash + ")"); - - long finalLockDuration = lockDuration; - return new Lock(threadPool, (int) (finalLockDuration / 4), tokenHash) { - @Override - public void unlock0() { - try { - HBaseLockManager.this.unlock(row, column, tokenHash); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + logger.debug("Won the lock with token " + token + " (" + token.hashCode() + ")"); - @Override - public synchronized void refresh() throws IOException { - HBaseLockManager.this.refresh(row, column, tokenHash, finalLockDuration); - } - }; + return new HBaseLock(lockDuration, token, row, column); } - /** - * Refreshes the lock. - * - * @param column Column to find the lock cell - * @param lockToken Lock token - * @param lockDuration Duration un milliseconds of the token. After this time the token is expired. - * @throws IOException if there is an error writing or reading from HBase. - */ - public void refresh(byte[] column, long lockToken, int lockDuration) throws IOException { - refresh(defaultRow, column, lockToken, lockDuration); - } - - /** * Refreshes the lock. * * @param row Row to find the lock cell * @param column Column to find the lock cell - * @param lockToken Lock token + * @param lockTokenHash Lock token * @param lockDuration Duration un milliseconds of the token. After this time the token is expired. * @throws IOException if there is an error writing or reading from HBase. */ - public void refresh(byte[] row, byte[] column, long lockToken, long lockDuration) throws IOException { + private void refresh(byte[] row, byte[] column, long lockTokenHash, long lockDuration) throws IOException { // Check token is valid - byte[] lockValue = readLockValue(row, column); - String currentLockToken = parseValidLockToken(lockValue); - if (currentLockToken == null || currentLockToken.hashCode() != lockToken) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentLockToken, lockValue); + LockToken currentLockToken = readLockToken(row, column); + if (currentLockToken.isEmpty() || currentLockToken.isExpired() || !currentLockToken.equals(lockTokenHash)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentLockToken); + } + if (currentLockToken.getRemainingTime() < lockDuration / 2) { + logger.warn("Refreshing lock with less than half of the duration remaining. Expected duration: {} Remaining time: {}ms", + lockDuration, + currentLockToken.getRemainingTime()); } - if (!tryToPutToken(currentLockToken, lockDuration, row, column, lockValue, REFRESH)) { + if (!tryToPutToken(currentLockToken.token, lockDuration, row, column, currentLockToken, REFRESH)) { // Error refreshing! - lockValue = readLockValue(row, column); - String newLockToken = parseValidLockToken(lockValue); + LockToken newLockToken = readLockToken(row, column); - logger.error("Current lock token:" + currentLockToken); - logger.error("New lock token: " + newLockToken); - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentLockToken, lockValue); + logger.error("Current lock token:" + currentLockToken.token); + logger.error("New lock token: " + newLockToken.token); + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentLockToken); } } - /** - * Releases the lock. - * - * @param column Column to find the lock cell - * @param lockToken Lock token - * @throws IOException if there is an error writing or reading from HBase. - * @throws IllegalLockStatusException if the lockToken does not match with the current lockToken - */ - public void unlock(byte[] column, long lockToken) throws IOException, IllegalLockStatusException { - unlock(defaultRow, column, lockToken); - } - /** * Releases the lock. * * @param row Row to find the lock cell * @param column Column to find the lock cell - * @param lockToken Lock token + * @param lockTokenHash Lock token * @throws IOException if there is an error writing or reading from HBase. * @throws IllegalLockStatusException if the lockToken does not match with the current lockToken */ - public void unlock(byte[] row, byte[] column, long lockToken) throws IOException, IllegalLockStatusException { - byte[] lockValue = readLockValue(row, column); + private void unlock(byte[] row, byte[] column, long lockTokenHash) throws IOException, IllegalLockStatusException { + LockToken currentToken = readLockToken(row, column); - String currentToken = parseValidLockToken(lockValue); - - if (currentToken == null || currentToken.hashCode() != lockToken) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentToken, lockValue); + if (currentToken.isEmpty() || currentToken.isExpired() || !currentToken.equals(lockTokenHash)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentToken); } - logger.debug("Unlock lock with token " + lockToken); - if (!clearLock(row, column, lockValue)) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentToken, lockValue); + logger.debug("Unlock lock with token " + lockTokenHash); + if (!clearLock(row, column, currentToken)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentToken); } } - private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[] qualifier, byte[] lockValue, String type) + private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[] qualifier, LockToken currentLock, String type) throws IOException { return hbaseManager.act(tableName, table -> { Put put = new Put(row) @@ -288,30 +246,31 @@ private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[ + token + LOCK_EXPIRING_DATE_SEPARATOR_STR + (System.currentTimeMillis() + lockDuration))); - return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockValue, put); + return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, currentLock.lockValue, put); }); } - private boolean clearLock(byte[] row, byte[] qualifier, byte[] lockValue) throws IOException { + private boolean clearLock(byte[] row, byte[] qualifier, LockToken lockToken) throws IOException { return hbaseManager.act(tableName, table -> { Put put = new Put(row) .addColumn(columnFamily, qualifier, Bytes.toBytes("")); - return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockValue, put); + return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockToken.lockValue, put); }); } /** - * Parse non-expired lock token. + * Parse lock token. * @param lockValue lock values - * @return Current lock token, if any + * @return Current lock token. */ - protected static String parseValidLockToken(byte[] lockValue) { + protected static LockToken parseLockToken(byte[] lockValue) { if (lockValue == null || lockValue.length == 0) { - return null; + return new LockToken(); } int idx1 = Bytes.indexOf(lockValue, LOCK_PREFIX_SEPARATOR_BYTE); int idx2 = Bytes.indexOf(lockValue, LOCK_EXPIRING_DATE_SEPARATOR_BYTE); + String type = Bytes.toString(lockValue, 0, idx1); String token = Bytes.toString(lockValue, idx1 + 1, idx2 - idx1 - 1); long expireDate; try { @@ -319,45 +278,82 @@ protected static String parseValidLockToken(byte[] lockValue) { } catch (NumberFormatException e) { // Deprecated token. Assume expired token if (Bytes.contains(lockValue, DEPRECATED_LOCK_SEPARATOR_BYTE)) { - return null; + return new LockToken(); } throw e; } + return new LockToken(lockValue, type, token, expireDate); + } + + protected static final class LockToken { + protected final byte[] lockValue; + protected final String type; + protected final String token; + protected final Long expireDate; + + private LockToken() { + this.lockValue = new byte[0]; + this.type = null; + this.token = null; + this.expireDate = null; + } + + private LockToken(byte[] lockValue, String type, String token, long expireDate) { + this.lockValue = lockValue; + this.type = type; + this.token = token; + this.expireDate = expireDate; + } + + /** + * A lock is taken if there is any lockValue, and + * the token has not expired. + * + * @return if the lock is taken + */ + public boolean isTaken() { + return token != null && !isExpired(); + } + + public boolean isExpired() { + return expireDate != null && expireDate < System.currentTimeMillis(); + } - if (isExpired(expireDate)) { - return null; - } else { + public boolean isEmpty() { + return token == null; + } + + public boolean equals(String token) { + return !isEmpty() && this.token.equals(token); + } + + public boolean equals(long tokenHash) { + return !isEmpty() && this.token.hashCode() == tokenHash; + } + + public byte[] getLockValue() { + return lockValue; + } + + public String getType() { + return type; + } + + public String getToken() { return token; } - } - protected static long parseExpireDate(byte[] lockValue) { - int idx2 = Bytes.indexOf(lockValue, LOCK_EXPIRING_DATE_SEPARATOR_BYTE); - try { - return Long.parseLong(Bytes.toString(lockValue, idx2 + 1)); - } catch (NumberFormatException e) { - // Deprecated token. Assume expired token - if (Bytes.contains(lockValue, DEPRECATED_LOCK_SEPARATOR_BYTE)) { - return -1; - } - throw e; + public Long getExpireDate() { + return expireDate; } - } - /** - * A lock is taken if there is any lockValue in the array, and - * the token has not expired. - * - * - * @param lockValue lock values - * @return if the lock is taken - */ - protected static boolean isLockTaken(byte[] lockValue) { - return parseValidLockToken(lockValue) != null; + public long getRemainingTime() { + return expireDate == null ? 0 : expireDate - System.currentTimeMillis(); + } } - private static boolean isExpired(long expireDate) { - return expireDate < System.currentTimeMillis(); + private LockToken readLockToken(byte[] row, byte[] qualifier) throws IOException { + return parseLockToken(readLockValue(row, qualifier)); } private byte[] readLockValue(byte[] row, byte[] qualifier) throws IOException { @@ -380,18 +376,22 @@ public IllegalLockStatusException(String s) { super(s); } - public static IllegalLockStatusException inconsistentLock(byte[] row, byte[] column, long lockToken, String currentLock, - byte[] lockValue) { - if (StringUtils.isEmpty(currentLock)) { - return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! " + private static IllegalLockStatusException inconsistentLock(byte[] row, byte[] column, long lockTokenHash, LockToken currentLock) { + if (currentLock.isEmpty()) { + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Empty lock. " + + "Row: '" + Bytes.toStringBinary(row) + "', " + + "column: '" + Bytes.toStringBinary(column) + "'. " + + "Lock: " + Bytes.toString(currentLock.lockValue) + "."); + } else if (currentLock.isExpired()) { + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Expired lock. " + "Row: '" + Bytes.toStringBinary(row) + "', " + "column: '" + Bytes.toStringBinary(column) + "'. " - + "Lock: " + Bytes.toString(lockValue) + "."); + + "Lock: " + Bytes.toString(currentLock.lockValue) + "."); } else { - return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! " + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Lock is taken. " + "Row: '" + Bytes.toStringBinary(row) + "', " + "column: '" + Bytes.toStringBinary(column) + "'. " - + lockToken + " != " + currentLock.hashCode() + " from " + Bytes.toString(lockValue)); + + lockTokenHash + " != " + currentLock.token.hashCode() + " from " + Bytes.toString(currentLock.lockValue)); } } } @@ -403,4 +403,38 @@ protected static ExecutorService buildThreadPool() { .build()); } + private final class HBaseLock extends Lock { + private final long lockDuration; + private final String token; + private final long tokenHash; + private final byte[] row; + private final byte[] column; + + private HBaseLock(long lockDuration, String token, byte[] row, byte[] column) { + super(HBaseLockManager.threadPool, (int) (lockDuration / 4), token.hashCode()); + this.lockDuration = lockDuration; + this.token = token; + this.tokenHash = token.hashCode(); + this.row = row; + this.column = column; + } + + @Override + public void unlock0() { + try { + synchronized (this) { + HBaseLockManager.this.unlock(row, column, tokenHash); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void refresh() throws IOException { + synchronized (this) { + HBaseLockManager.this.refresh(row, column, tokenHash, lockDuration); + } + } + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java index f9bfa4d0a3f..1f6cd77efd8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java @@ -17,24 +17,28 @@ package org.opencb.opencga.storage.hadoop.utils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.core.common.ExceptionUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.auth.HBaseCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.nio.ByteBuffer; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.opencb.opencga.storage.hadoop.utils.PersistentResultScanner.isValid; @@ -388,6 +392,139 @@ public boolean createTableIfNeeded(String tableName, byte[] columnFamily, return createTableIfNeeded(getConnection(), tableName, columnFamily, preSplits, compressionType); } + public boolean splitAndMove(Admin admin, TableName name, byte[] expectedSplit) throws IOException { + return splitAndMove(admin, name, expectedSplit, 3, true); + } + + public boolean splitAndMove(Admin admin, TableName tableName, byte[] expectedSplit, int retries, boolean ignoreExceptions) + throws IOException { + StopWatch stopWatch = StopWatch.createStarted(); + int count = 0; + while (count < retries) { + count++; + try { + // Check if split point exists + RegionInfo regionInfo = getRegionInfo(admin, tableName, expectedSplit); + + if (regionInfo == null) { + LOGGER.info("Splitting table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + admin.split(tableName, expectedSplit); + regionInfo = getRegionInfo(admin, tableName, expectedSplit); + int getRegionInfoAttempts = 10; + while (regionInfo == null) { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + regionInfo = getRegionInfo(admin, tableName, expectedSplit); + getRegionInfoAttempts--; + if (getRegionInfoAttempts == 0) { + throw new DoNotRetryRegionException( + "Split point " + Bytes.toStringBinary(expectedSplit) + " not found after creation"); + } + } + } else if (count == 1) { + // First try and split point exists. Skip moving region + LOGGER.info("Split point {} already exists. Nothing to do.", Bytes.toStringBinary(expectedSplit)); + return false; + } + LOGGER.info("Moving region '{}' to another region server", regionInfo.getRegionNameAsString()); + admin.move(regionInfo.getEncodedNameAsBytes(), (byte[]) null); + LOGGER.info("New region created '{}' in {}", regionInfo.getRegionNameAsString(), TimeUtils.durationToString(stopWatch)); + return true; + } catch (IOException | RuntimeException e) { + if (ignoreExceptions) { + if (e instanceof DoNotRetryRegionException) { + LOGGER.warn("Error splitting table {} at {}. Retry {}/{} : {}", tableName, + Bytes.toStringBinary(expectedSplit), count, retries, e.getMessage()); + } else { + LOGGER.warn("Error splitting table {} at {}. Retry {}/{}", tableName, + Bytes.toStringBinary(expectedSplit), count, retries, e); + } + } else { + throw e; + } + } + try { + // Wait before retry + LOGGER.info("Waiting before retrying split table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + LOGGER.warn("Unable to split table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + return false; + } + + public static RegionInfo getRegionInfo(Admin admin, TableName name, byte[] expectedSplit) throws IOException { + return admin.getRegions(name) + .stream() + .filter(region -> Bytes.equals(region.getStartKey(), expectedSplit)) + .findFirst() + .orElse(null); + } + + public int expandTableIfNeeded(String tableName, int batch, + Function> batchSplitsGenerator, + int extraBatches, Function batchPlaceholderSplitGenerator) throws IOException { + return expandTableIfNeeded(tableName, Collections.singletonList(batch), batchSplitsGenerator, extraBatches, + batchPlaceholderSplitGenerator); + } + public int expandTableIfNeeded(String tableName, Collection batches, + Function> batchSplitsGenerator, + int extraBatches, Function batchPlaceholderSplitGenerator) throws IOException { + if (batches.isEmpty()) { + throw new IllegalArgumentException("No batches provided"); + } + // Get the expected splits for these batches + Collection expectedSplits = new LinkedHashSet<>(); + for (Integer batch : batches) { + expectedSplits.addAll(batchSplitsGenerator.apply(batch)); + } + int lastBatch = batches.stream().max(Integer::compareTo).get(); + + // Add some split placeholders for the extra batches + for (int i = lastBatch + 1; i < lastBatch + 1 + extraBatches; i++) { + expectedSplits.add(batchPlaceholderSplitGenerator.apply(i)); + } + // Shuffle the splits + List shuffledSplits = new ArrayList<>(expectedSplits); + Collections.shuffle(shuffledSplits); + + // Ensure that the table is split at least until the next expected split + return act(tableName, (table, admin) -> { + int newSplits = 0; + Set existingSplits = Arrays.stream(HBaseCompat.getInstance().getTableStartKeys(admin, table)) + .map(ByteBuffer::wrap) + .collect(Collectors.toSet()); + + int expectedNewSplits = 0; + for (byte[] expectedSplit : expectedSplits) { + if (!existingSplits.contains(ByteBuffer.wrap(expectedSplit))) { + expectedNewSplits++; + LOGGER.info("Missing split point '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + } + } + if (expectedNewSplits == 0) { + return 0; + } else { + LOGGER.info("Found {} missing split points. Splitting table '{}'", expectedNewSplits, tableName); + } + + for (byte[] expectedSplit : expectedSplits) { + if (!existingSplits.contains(ByteBuffer.wrap(expectedSplit))) { + if (splitAndMove(admin, table.getName(), expectedSplit)) { + newSplits++; + } + } + } + return newSplits; + }); + } + + /** * Create default HBase table layout with one column family. * diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java index ada36f36d49..b1a21badba6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java @@ -37,7 +37,6 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -96,83 +95,6 @@ public HadoopLocalLoadVariantStoragePipeline(StorageConfiguration configuration, @Override protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata variantFileMetadata) throws StorageEngineException { super.preLoadRegisterAndValidateFile(studyId, variantFileMetadata); - boolean loadSampleIndex = YesNoAuto.parse(getOptions(), LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); - FileMetadata fileMetadata = getMetadataManager().getFileMetadata(studyId, getFileId()); - - int version = getMetadataManager().getStudyMetadata(studyId).getSampleIndexConfigurationLatest().getVersion(); - Set alreadyIndexedSamples = new LinkedHashSet<>(); - Set processedSamples = new LinkedHashSet<>(); - Set samplesWithoutSplitData = new LinkedHashSet<>(); - VariantStorageEngine.SplitData splitData = VariantStorageEngine.SplitData.from(options); - for (String sample : variantFileMetadata.getSampleIds()) { - Integer sampleId = getMetadataManager().getSampleId(studyId, sample); - SampleMetadata sampleMetadata = getMetadataManager().getSampleMetadata(studyId, sampleId); - if (splitData != null && sampleMetadata.getSplitData() != null) { - if (!splitData.equals(sampleMetadata.getSplitData())) { - throw new StorageEngineException("Incompatible split data methods. " - + "Unable to mix requested " + splitData - + " with existing " + sampleMetadata.getSplitData()); - } - } - if (sampleMetadata.isIndexed()) { - if (sampleMetadata.getFiles().size() == 1 && sampleMetadata.getFiles().contains(fileMetadata.getId())) { - // It might happen that the sample is marked as INDEXED, but not the file. - // If the sample only belongs to this file (i.e. it's only file is this file), then ignore - // the overwrite the current sample metadata index status - sampleMetadata = getMetadataManager().updateSampleMetadata(studyId, sampleId, - sm -> sm.setIndexStatus(fileMetadata.getIndexStatus())); - } - } - if (sampleMetadata.isIndexed()) { - alreadyIndexedSamples.add(sample); - if (sampleMetadata.isAnnotated() - || !loadSampleIndex && sampleMetadata.getSampleIndexStatus(version) == Status.READY - || sampleMetadata.getSampleIndexAnnotationStatus(version) == Status.READY - || sampleMetadata.getFamilyIndexStatus(version) == Status.READY - || sampleMetadata.isFamilyIndexDefined()) { - processedSamples.add(sampleMetadata.getId()); - } - } - - if (splitData != null && splitData != sampleMetadata.getSplitData()) { - samplesWithoutSplitData.add(sampleId); - } - } - - if (!alreadyIndexedSamples.isEmpty()) { - if (splitData != null) { - logger.info("Loading split data"); - } else { - String fileName = Paths.get(variantFileMetadata.getPath()).getFileName().toString(); - throw StorageEngineException.alreadyLoadedSamples(fileName, new ArrayList<>(alreadyIndexedSamples)); - } - for (Integer sampleId : processedSamples) { - getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - if (!loadSampleIndex) { - for (Integer v : sampleMetadata.getSampleIndexVersions()) { - sampleMetadata.setSampleIndexStatus(Status.NONE, v); - } - } - for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { - sampleMetadata.setSampleIndexAnnotationStatus(Status.NONE, v); - } - for (Integer v : sampleMetadata.getFamilyIndexVersions()) { - sampleMetadata.setFamilyIndexStatus(Status.NONE, v); - } - sampleMetadata.setAnnotationStatus(Status.NONE); - sampleMetadata.setMendelianErrorStatus(Status.NONE); - }); - } - } - - if (splitData != null) { - // Register loadSplitData - for (Integer sampleId : samplesWithoutSplitData) { - getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.setSplitData(splitData); - }); - } - } } @Override diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java index 6aa472ee735..45682df31f4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java @@ -5,7 +5,6 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; -import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; import java.util.List; @@ -18,8 +17,8 @@ public HadoopVariantQueryParser(CellBaseUtils cellBaseUtils, VariantStorageMetad } @Override - protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { - Query query = super.preProcessQuery(originalQuery, options, projection); + public Query preProcessQuery(Query originalQuery, QueryOptions options) { + Query query = super.preProcessQuery(originalQuery, options); List studyNames = metadataManager.getStudyNames(); if (isValidParam(query, STUDY) && studyNames.size() == 1) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index e0cbbfb3963..84f00b042e9 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -31,6 +31,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.DatabaseCredentials; @@ -38,6 +39,7 @@ import org.opencb.opencga.core.config.storage.StorageEngineConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.StoragePipelineException; @@ -77,6 +79,7 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchemaManager; import org.opencb.opencga.storage.hadoop.variant.adaptors.sample.HBaseVariantSampleDataManager; import org.opencb.opencga.storage.hadoop.variant.annotation.HadoopDefaultVariantAnnotationManager; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDeleteHBaseColumnTask; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutorFactory; @@ -826,7 +829,10 @@ private void remove(String study, List files, List samples, URI archiveColumns.add(family + ':' + ArchiveTableHelper.getRefColumnName(fileId)); archiveColumns.add(family + ':' + ArchiveTableHelper.getNonRefColumnName(fileId)); } - String[] deleteFromArchiveArgs = DeleteHBaseColumnDriver.buildArgs(archiveTable, archiveColumns, options); + ObjectMap thisOptions = new ObjectMap(options); + ArchiveDeleteHBaseColumnTask.configureTask(thisOptions, fileIds); + + String[] deleteFromArchiveArgs = DeleteHBaseColumnDriver.buildArgs(archiveTable, archiveColumns, thisOptions); getMRExecutor().run(DeleteHBaseColumnDriver.class, deleteFromArchiveArgs, "Delete from archive table"); return stopWatch.now(TimeUnit.MILLISECONDS); }); @@ -1082,6 +1088,49 @@ protected List initVariantAggregationExecutors() { return executors; } + @Override + public ObjectMap inferConfigurationParams(VariantSetupParams params) { + ObjectMap options = super.inferConfigurationParams(params); + ObjectMap configuredOptions = getOptions(); + + long expectedHBaseRegionSize = IOUtils.fromHumanReadableToByte("7.5GiB"); + + options.put(EXPECTED_SAMPLES_NUMBER.key(), params.getExpectedSamples()); + options.put(EXPECTED_FILES_NUMBER.key(), params.getExpectedFiles()); + + // Variant pre-split + int defaultVariantPreSplit = configuredOptions + .getInt(VARIANT_TABLE_PRESPLIT_SIZE.key(), VARIANT_TABLE_PRESPLIT_SIZE.defaultValue()); + float variantsFileToHBaseMultiplier = 1.3f; + Long averageFileSize = IOUtils.fromHumanReadableToByte(params.getAverageFileSize(), true); + float variantsTableSize = params.getExpectedFiles() * averageFileSize * variantsFileToHBaseMultiplier; + int variantPreSplit = (int) (variantsTableSize / expectedHBaseRegionSize); + options.put(VARIANT_TABLE_PRESPLIT_SIZE.key(), Math.max(defaultVariantPreSplit, variantPreSplit)); + + // Archive pre-split + int filesPerBatch = configuredOptions + .getInt(ARCHIVE_FILE_BATCH_SIZE.key(), ARCHIVE_FILE_BATCH_SIZE.defaultValue()); + float archiveFileToHBaseMultiplier = 1.2f; + float archiveTableSize = filesPerBatch * averageFileSize.floatValue() * archiveFileToHBaseMultiplier; + int archiveTablePreSplit = (int) (archiveTableSize / expectedHBaseRegionSize); + options.put(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), Math.max(1, archiveTablePreSplit)); + + // SampleIndex pre-split + long averageSizePerVariant; + if (params.getVariantsPerSample() > 3500000) { + // With this many variants per sample, most of them won't have much data + averageSizePerVariant = IOUtils.fromHumanReadableToByte("13B"); + } else { + // With a small number of variants per sample, most of them will have a lot of data + averageSizePerVariant = IOUtils.fromHumanReadableToByte("25B"); + } + long sampleIndexSize = params.getVariantsPerSample() * averageSizePerVariant; + int samplesPerSplit = (int) (expectedHBaseRegionSize / sampleIndexSize); + options.put(SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), samplesPerSplit); + + return options; + } + @Override public void close() throws IOException { super.close(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java index 9c1f22ed349..817605be87c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java @@ -8,7 +8,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { HADOOP_LOAD_FILES_IN_PARALLEL("storage.hadoop.load.filesInParallel", 1), HBASE_NAMESPACE("storage.hadoop.hbase.namespace"), - EXPECTED_FILES_NUMBER("expected_files_number", 5000), + EXPECTED_FILES_NUMBER("expected_files_number", 50), EXPECTED_SAMPLES_NUMBER("expected_samples_number"), DBADAPTOR_PHOENIX_FETCH_SIZE("storage.hadoop.phoenix.fetchSize", -1), DBADAPTOR_PHOENIX_QUERY_COMPLEXITY_THRESHOLD("storage.hadoop.phoenix.queryComplexityThreshold", 250), @@ -64,7 +64,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Variant table configuration ///////////////////////// VARIANT_TABLE_COMPRESSION("storage.hadoop.variant.table.compression", Compression.Algorithm.SNAPPY.getName()), - VARIANT_TABLE_PRESPLIT_SIZE("storage.hadoop.variant.table.preSplit.numSplits", 500), + VARIANT_TABLE_PRESPLIT_SIZE("storage.hadoop.variant.table.preSplit.numSplits", 50), // Do not create phoenix indexes. Testing purposes only VARIANT_TABLE_INDEXES_SKIP("storage.hadoop.variant.table.indexes.skip"), VARIANT_TABLE_LOAD_REFERENCE("storage.hadoop.variant.table.load.reference", false), @@ -78,7 +78,8 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Archive table configuration ///////////////////////// ARCHIVE_TABLE_COMPRESSION("storage.hadoop.archive.table.compression", Compression.Algorithm.GZ.getName()), - ARCHIVE_TABLE_PRESPLIT_SIZE("storage.hadoop.archive.table.preSplit.splitsPerBatch", 500), + ARCHIVE_TABLE_PRESPLIT_SIZE("storage.hadoop.archive.table.preSplit.splitsPerBatch", 10), + ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.archive.table.preSplit.extraSplits", 3), ARCHIVE_CHUNK_SIZE("storage.hadoop.archive.table.chunkSize", 1000), ARCHIVE_FILE_BATCH_SIZE("storage.hadoop.archive.table.fileBatchSize", 1000), @@ -92,7 +93,8 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Sample index table configuration ///////////////////////// SAMPLE_INDEX_TABLE_COMPRESSION("storage.hadoop.sampleIndex.table.compression", Compression.Algorithm.SNAPPY.getName()), - SAMPLE_INDEX_TABLE_PRESPLIT_SIZE("storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit", 15), + SAMPLE_INDEX_TABLE_PRESPLIT_SIZE("storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit", 200), + SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.sampleIndex.table.preSplit.extraSplits", 5), SAMPLE_INDEX_BUILD_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.build.maxSamplesPerMR", 2000), SAMPLE_INDEX_ANNOTATION_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.annotation.maxSamplesPerMR", 2000), SAMPLE_INDEX_FAMILY_MAX_TRIOS_PER_MR("storage.hadoop.sampleIndex.family.maxTriosPerMR", 1000), diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java index 9b86cba3f6a..c3f5ebf550e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java @@ -170,6 +170,7 @@ public URI preLoad(URI input, URI output) throws StorageEngineException { try { if (YesNoAuto.parse(getOptions(), LOAD_ARCHIVE.key()).orYes().booleanValue()) { ArchiveTableHelper.createArchiveTableIfNeeded(getOptions(), getArchiveTable(), dbAdaptor.getConnection()); + ArchiveTableHelper.expandTableIfNeeded(getOptions(), getArchiveTable(), getFileId(), dbAdaptor.getHBaseManager()); } else { logger.info("Skip archive table"); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java index c672e94fb58..ceb52e5eeaa 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java @@ -7,6 +7,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.query.ParsedQuery; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; @@ -43,8 +44,8 @@ public HBaseColumnIntersectVariantQueryExecutor(VariantDBAdaptor dbAdaptor, Stri } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { - + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { + VariantQuery query = variantQuery.getQuery(); if (!options.getBoolean(HBASE_COLUMN_INTERSECT, ACTIVE_BY_DEFAULT)) { // HBase column intersect not active return false; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java index 9e06ebc4b51..977e6fbbf93 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java @@ -146,7 +146,7 @@ public static boolean mightHashAlleles(Variant variant) { */ public static byte[] generateVariantRowKey(String chrom, int start, Integer end, String ref, String alt, StructuralVariation sv) { chrom = Region.normalizeChromosome(chrom); - alt = buildSymbolicAlternate(ref, alt, end, sv); + alt = buildSymbolicAlternate(ref, alt, start, end, sv); int size = getSize(chrom, ref, alt); if (size > HConstants.MAX_ROW_LENGTH) { @@ -201,11 +201,11 @@ public static String buildAlleles(Variant v) { } public static String buildSymbolicAlternate(Variant v) { - return buildSymbolicAlternate(v.getReference(), v.getAlternate(), v.getEnd(), v.getSv()); + return buildSymbolicAlternate(v.getReference(), v.getAlternate(), v.getStart(), v.getEnd(), v.getSv()); } // visible for test - public static String buildSymbolicAlternate(String reference, String alternate, Integer end, StructuralVariation sv) { + public static String buildSymbolicAlternate(String reference, String alternate, int start, Integer end, StructuralVariation sv) { if (sv != null) { byte[] alternateBytes = alternate.getBytes(); if (!Allele.wouldBeSymbolicAllele(alternateBytes) && emptyCiStartEnd(sv)) { @@ -216,14 +216,16 @@ public static String buildSymbolicAlternate(String reference, String alternate, if (StructuralVariantType.TANDEM_DUPLICATION.equals(sv.getType())) { alternate = VariantBuilder.DUP_TANDEM_ALT; } - boolean bnd = Allele.wouldBeBreakpoint(alternateBytes); + // Ignore CIEND on variants without an actual END. This includes Breakends and INSERTIONS + // These variants are not expected to have CIEND. This is a redundant check, as the CIEND should be empty after normalization. + boolean ignoreCiend = Allele.wouldBeBreakpoint(alternateBytes) || end < start; alternate = alternate + SV_ALTERNATE_SEPARATOR + end + SV_ALTERNATE_SEPARATOR + (sv.getCiStartLeft() == null ? 0 : sv.getCiStartLeft()) + SV_ALTERNATE_SEPARATOR + (sv.getCiStartRight() == null ? 0 : sv.getCiStartRight()) - + SV_ALTERNATE_SEPARATOR + (bnd | sv.getCiEndLeft() == null ? 0 : sv.getCiEndLeft()) - + SV_ALTERNATE_SEPARATOR + (bnd | sv.getCiEndRight() == null ? 0 : sv.getCiEndRight()); + + SV_ALTERNATE_SEPARATOR + ((ignoreCiend || sv.getCiEndLeft() == null) ? 0 : sv.getCiEndLeft()) + + SV_ALTERNATE_SEPARATOR + ((ignoreCiend || sv.getCiEndRight() == null) ? 0 : sv.getCiEndRight()); if (StringUtils.isNotEmpty(sv.getLeftSvInsSeq()) || StringUtils.isNotEmpty(sv.getRightSvInsSeq())) { alternate = alternate diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java index ce29e3f18fe..102638aab8c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java @@ -25,7 +25,7 @@ public void run() throws ToolException { List samples1 = getSampleList1(); List samples2 = getSampleList2(); - if (getConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { + if (getGwasConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { addWarning("Unable to calculate chi-square test."); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java new file mode 100644 index 00000000000..0fb9fb0adbf --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.storage.hadoop.variant.archive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.storage.hadoop.utils.DeleteHBaseColumnDriver; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ArchiveDeleteHBaseColumnTask extends DeleteHBaseColumnDriver.DeleteHBaseColumnTask { + + public static final String FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX = "fileBatchesWithFilesToDeleteFromArchiveIndex"; + + public static void configureTask(ObjectMap options, List fileIds) { + ArchiveRowKeyFactory keyFactory = new ArchiveRowKeyFactory(options); + Set fileBatchesSet = new HashSet<>(); + for (Integer fileId : fileIds) { + fileBatchesSet.add(keyFactory.getFileBatch(fileId)); + } + List fileBatches = new ArrayList<>(fileBatchesSet); + fileBatches.sort(Integer::compareTo); + options.put(DeleteHBaseColumnDriver.DELETE_HBASE_COLUMN_TASK_CLASS, ArchiveDeleteHBaseColumnTask.class.getName()); + options.put(ArchiveDeleteHBaseColumnTask.FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX, fileBatches); + } + + @Override + public List> getRegionsToDelete(Configuration configuration) { + int[] fileBatches = configuration.getInts(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX); + List> regions = new ArrayList<>(); + ArchiveRowKeyFactory archiveRowKeyFactory = new ArchiveRowKeyFactory(configuration); + for (int fileBatch : fileBatches) { + regions.add(new Pair<>( + Bytes.toBytes(archiveRowKeyFactory.generateBlockIdFromBatch(fileBatch)), + Bytes.toBytes(archiveRowKeyFactory.generateBlockIdFromBatch(fileBatch + 1)))); + } + return regions; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java index 43e9cf97cf7..d20e5eee924 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java @@ -143,6 +143,27 @@ public String generateBlockId(int fileId) { return sb.toString(); } + /** + * Generates a Row key based on Chromosome and position adjusted for the + * Chunk size.
+ *
    + *
  • Using {@link Region#normalizeChromosome(String)} to get standard chromosome + * name + *
  • Using {@link #getSliceId(long)} to return slice position + *
+ * e.g. using chunk size 100, separator _ with chr2 and 1234 would result in + * 2_12 + * + * @param fileBatch File batch + * @return {@link String} Row key string + */ + public String generateBlockIdFromBatch(int fileBatch) { + StringBuilder sb = new StringBuilder(FILE_BATCH_PAD + 1); + sb.append(StringUtils.leftPad(String.valueOf(fileBatch), FILE_BATCH_PAD, '0')); + sb.append(getSeparator()); + return sb.toString(); + } + public String generateBlockIdFromSlice(int fileId, String chrom, long slice) { return generateBlockIdFromSliceAndBatch(getFileBatch(fileId), chrom, slice); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java index 3af44e97188..348d4d33e00 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java @@ -56,7 +56,7 @@ public class ArchiveTableHelper extends GenomeHelper { public static final byte[] REF_COLUMN_SUFIX_BYTES = Bytes.toBytes(REF_COLUMN_SUFIX); public static final String CONFIG_ARCHIVE_TABLE_NAME = "opencga.archive.table.name"; - private final Logger logger = LoggerFactory.getLogger(ArchiveTableHelper.class); + private static Logger logger = LoggerFactory.getLogger(ArchiveTableHelper.class); private final AtomicReference meta = new AtomicReference<>(); private final ArchiveRowKeyFactory keyFactory; private final byte[] nonRefColumn; @@ -188,17 +188,13 @@ public static String getRefColumnName(int fileId) { public static boolean createArchiveTableIfNeeded(Configuration conf, String tableName) throws IOException { try (Connection con = ConnectionFactory.createConnection(conf)) { - return createArchiveTableIfNeeded(conf, tableName, con); + Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( + conf.get(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); + final List preSplits = generateArchiveTableBootPreSplitHuman(conf); + return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); } } - public static boolean createArchiveTableIfNeeded(Configuration conf, String tableName, Connection con) throws IOException { - Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( - conf.get(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); - final List preSplits = generateArchiveTableBootPreSplitHuman(conf); - return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); - } - public static boolean createArchiveTableIfNeeded(ObjectMap conf, String tableName, Connection con) throws IOException { Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( conf.getString(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); @@ -206,6 +202,20 @@ public static boolean createArchiveTableIfNeeded(ObjectMap conf, String tableNam return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); } + public static void expandTableIfNeeded(ObjectMap options, String archiveTable, int fileId, HBaseManager hBaseManager) + throws IOException { + int splitsPerBatch = options.getInt(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), ARCHIVE_TABLE_PRESPLIT_SIZE.defaultValue()); + int extraBatches = options.getInt(ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.key(), ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); + ArchiveRowKeyFactory rowKeyFactory = new ArchiveRowKeyFactory(options); + int thisBatch = rowKeyFactory.getFileBatch(fileId); + int newRegions = hBaseManager.expandTableIfNeeded(archiveTable, thisBatch, + batch -> generateBatchSplitsHuman(rowKeyFactory, splitsPerBatch, batch), + extraBatches, batch -> Bytes.toBytes(rowKeyFactory.generateBlockIdFromBatch(batch))); + if (newRegions > 0) { + logger.info("Archive table '" + archiveTable + "' expanded with " + newRegions + " new regions for batch " + thisBatch); + } + } + public static List generateArchiveTableBootPreSplitHuman(Configuration conf) { final ArchiveRowKeyFactory rowKeyFactory = new ArchiveRowKeyFactory(conf); int nSplits = conf.getInt(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), ARCHIVE_TABLE_PRESPLIT_SIZE.defaultValue()); @@ -226,15 +236,18 @@ private static List generateArchiveTableBootPreSplitHuman(ArchiveRowKeyF final List preSplits = new ArrayList<>(nSplits * expectedNumBatches); for (int batch = 0; batch <= expectedNumBatches; batch++) { - int finalBatch = batch; - preSplits.addAll(generateBootPreSplitsHuman(nSplits, (chr, position) -> { - long slice = rowKeyFactory.getSliceId(position); - return Bytes.toBytes(rowKeyFactory.generateBlockIdFromSliceAndBatch(finalBatch, chr, slice)); - })); + preSplits.addAll(generateBatchSplitsHuman(rowKeyFactory, nSplits, batch)); } return preSplits; } + public static List generateBatchSplitsHuman(ArchiveRowKeyFactory rowKeyFactory, int nSplits, int batch) { + return generateBootPreSplitsHuman(nSplits, (chr, position) -> { + long slice = rowKeyFactory.getSliceId(position); + return Bytes.toBytes(rowKeyFactory.generateBlockIdFromSliceAndBatch(batch, chr, slice)); + }); + } + public VariantFileMetadata getFileMetadata() { return meta.get(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java index 4dd50c9eacf..5063ca1fe4b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java @@ -9,7 +9,6 @@ import org.opencb.biodata.tools.pedigree.MendelianError; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.Trio; @@ -33,10 +32,10 @@ public SampleIndexMendelianErrorQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { - if (VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_MENDELIAN_ERROR) - || VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_DE_NOVO) - || VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_DE_NOVO_STRICT)) { + public boolean canUseThisExecutor(ParsedVariantQuery query, QueryOptions options) { + if (VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_MENDELIAN_ERROR) + || VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_DE_NOVO) + || VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_DE_NOVO_STRICT)) { return super.canUseThisExecutor(query, options); } else { return false; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java index 371d57ac4da..7b1cdc5dceb 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java @@ -86,7 +86,8 @@ public SampleIndexOnlyVariantQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, Sam } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { + VariantQuery query = variantQuery.getQuery(); if (SampleIndexQueryParser.validSampleIndexQuery(query)) { if (isFullyCoveredQuery(query, options)) { @@ -182,7 +183,8 @@ private boolean isFullyCoveredQuery(Query inputQuery, QueryOptions options) { // ParsedVariantQuery parsedVariantQuery = variantQueryProjectionParser.parseQuery(query, options, true); SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); - return isQueryCovered(query) && isIncludeCovered(sampleIndexQuery, inputQuery, options); + return isQueryCovered(sampleIndexQuery.getUncoveredQuery()) + && isIncludeCovered(sampleIndexQuery, inputQuery, options); } private boolean isQueryCovered(Query query) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java index 6175128018d..03e8a5fe880 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java @@ -58,9 +58,9 @@ public SampleIndexVariantQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, SampleI } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery query, QueryOptions options) { if (options.getBoolean(SAMPLE_INDEX_INTERSECT, true)) { - return SampleIndexQueryParser.validSampleIndexQuery(query); + return SampleIndexQueryParser.validSampleIndexQuery(query.getQuery()); } return false; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java index af632d69436..ad09896e627 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java @@ -112,10 +112,9 @@ protected void parseAndValidateParameters() throws IOException { } else { trioList.add(metadataManager.getSampleIdOrFail(getStudyId(), trio.getMother())); } - int childId = metadataManager.getSampleIdOrFail(getStudyId(), trio.getChild()); - trioList.add(childId); - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), childId); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), trio.getChild()); + trioList.add(sampleMetadata.getId()); if (!overwrite && sampleMetadata.getFamilyIndexStatus(sampleIndexVersion) == TaskMetadata.Status.READY) { LOGGER.info("Skip sample " + sampleMetadata.getName() + ". Already precomputed!"); } else { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java index 8b18a5e9a9f..8047fb49cf1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java @@ -85,6 +85,7 @@ public void buildSampleIndex(List samples, ObjectMap options, boolean ov } sampleIndexDBAdaptor.createTableIfNeeded(studyId, schema.getVersion(), options); + sampleIndexDBAdaptor.expandTableIfNeeded(studyId, schema.getVersion(), sampleIds, options); if (finalSamplesList.size() < 20) { logger.info("Run sample index build on samples " + finalSamplesList); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java index 06a3ca79b9a..0d785856776 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java @@ -2,10 +2,7 @@ import com.google.common.collect.Iterators; import org.apache.commons.collections4.CollectionUtils; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.util.Bytes; import org.opencb.biodata.models.core.Region; @@ -796,8 +793,11 @@ public boolean createTableIfNeeded(int studyId, int version, ObjectMap options) int preSplitSize = options.getInt( HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.defaultValue()); + int sampleIndexExtraSplits = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); - int splits = samples / preSplitSize; + int splits = (samples / preSplitSize) + sampleIndexExtraSplits; ArrayList preSplits = new ArrayList<>(splits); for (int i = 0; i < splits; i++) { preSplits.add(SampleIndexSchema.toRowKey(i * preSplitSize)); @@ -815,6 +815,32 @@ public boolean createTableIfNeeded(int studyId, int version, ObjectMap options) } } + public void expandTableIfNeeded(int studyId, int version, List sampleIds, ObjectMap options) { + String sampleIndexTable = getSampleIndexTableName(studyId, version); + int preSplitSize = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.defaultValue()); + int sampleIndexExtraBatches = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); + Set batches = new HashSet<>(); + for (Integer sampleId : sampleIds) { + batches.add((sampleId) / preSplitSize); + } + + try { + int newRegions = hBaseManager.expandTableIfNeeded(sampleIndexTable, batches, + batch -> Collections.singletonList(SampleIndexSchema.toRowKey(batch * preSplitSize)), + sampleIndexExtraBatches, batch -> SampleIndexSchema.toRowKey(batch * preSplitSize)); + if (newRegions != 0) { + // Log number of new regions + logger.info("Sample index table '" + sampleIndexTable + "' expanded with " + newRegions + " new regions"); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public void updateSampleIndexSchemaStatus(int studyId, int version) throws StorageEngineException { StudyMetadata studyMetadata = metadataManager.getStudyMetadata(studyId); if (studyMetadata.getSampleIndexConfiguration(version).getStatus() diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java index f2700831e94..23c94cf0c28 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java @@ -188,6 +188,7 @@ public String toString() { public boolean open() { super.open(); dbAdaptor.createTableIfNeeded(studyId, schema.getVersion(), options); + dbAdaptor.expandTableIfNeeded(studyId, schema.getVersion(), sampleIds, options); return true; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java index 027d241282c..d4605b918f4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java @@ -1286,6 +1286,7 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema intergenic == Boolean.FALSE && (!ctFilterCoveredBySummary || (!ctBtCombinationCoveredBySummary && combination.isBiotype()) || combination.isFlag()); + if (useCtIndexFilter) { ctCovered = completeIndex; consequenceTypeFilter = schema.getCtIndex().getField().buildFilter(new OpValue<>("=", soNames)); @@ -1515,6 +1516,8 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema if (intergenic == null || intergenic) { // If intergenic is undefined, or true, CT and BT filters can not be used. + biotypeFilter = schema.getBiotypeIndex().getField().noOpFilter(); + consequenceTypeFilter = schema.getCtIndex().getField().noOpFilter(); if (!biotypeFilter.isNoOp()) { throw new IllegalStateException("Unexpected BT filter for intergenic=" + intergenic); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java index ef2aecb803b..a087420f425 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java @@ -78,8 +78,7 @@ public Collection getSampleIndexConfigurationVersions(int studyId, Coll private Collection getSampleIndexConfigurationVersions(int studyId, Object sample, boolean withAnnotation, boolean withFamilyIndex) { - int sampleId = metadataManager.getSampleIdOrFail(studyId, sample); - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sample); Collection versions = sampleMetadata.getSampleIndexVersions(); if (withAnnotation) { versions = CollectionUtils.intersection( diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java index f1b167ae9c1..6edc5c9abf2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java @@ -688,7 +688,7 @@ protected int getRelativeStart(Variant variant) { } protected String getAlternate(Variant v) { - return VariantPhoenixKeyFactory.buildSymbolicAlternate(v.getReference(), v.getAlternate(), v.getEnd(), v.getSv()); + return VariantPhoenixKeyFactory.buildSymbolicAlternate(v); } /** diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java index 3f6a23c5abf..fbdfdf920ec 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java @@ -279,14 +279,5 @@ protected Lock lockToken(byte[] rowKey, byte[] lockName, long lockDuration, long } } - protected void unLock(byte[] rowKey, byte[] lockName, long token) { - try { - this.lock.unlock(rowKey, lockName, token); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java index 85a1d0cec0c..6ddae38be65 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java @@ -49,20 +49,14 @@ public HBaseProjectMetadataDBAdaptor(HBaseManager hBaseManager, String metaTable @Override public Lock lockProject(long lockDuration, long timeout, String lockName) - throws InterruptedException, TimeoutException, StorageEngineException { + throws StorageEngineException { try { ensureTableExists(); return lock.lock(getProjectRowKey(), getLockColumn(lockName), lockDuration, timeout); - } catch (IOException e) { - throw new StorageEngineException("Error locking project in HBase", e); - } - } - - @Override - public void unLockProject(long lockId) throws StorageEngineException { - try { - lock.unlock(getProjectRowKey(), getLockColumn(), lockId); - } catch (IOException e) { + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new StorageEngineException("Unable to lock the Project", e); + } catch (IOException | TimeoutException e) { throw new StorageEngineException("Error locking project in HBase", e); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java index 17c61739496..39c63923c02 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java @@ -68,7 +68,7 @@ protected void add(List batch) throws Exception { return PhoenixHelper.toBytes(studyIds, PIntegerArray.INSTANCE); }); - byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("id").toString())); + byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("attr_id").toString())); variantRows.add(row); mutations.add(new Put(row) .addColumn(family, VariantPhoenixSchema.VariantColumn.INDEX_STUDIES.bytes(), bytes)); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java index 0f421081ff6..5943a43519f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java @@ -214,23 +214,47 @@ public void testLockRefreshExpiredRefresh() throws Exception { @Test public void testGetCurrent() { long e = System.currentTimeMillis() + 1000; - String s; + HBaseLockManager.LockToken s; + + // null token + s = HBaseLockManager.parseLockToken(null); + assertTrue(s.isEmpty()); + assertFalse(s.isTaken()); + assertEquals(null, s.getType()); + assertArrayEquals(new byte[0], s.getLockValue()); + + // Empty token + s = HBaseLockManager.parseLockToken(Bytes.toBytes("")); + assertTrue(s.isEmpty()); + assertFalse(s.isTaken()); + assertEquals(null, s.getType()); + assertArrayEquals(new byte[0], s.getLockValue()); // Expired current token - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("CURRENT-abc:123")); - assertNull(s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("CURRENT-abc:123")); + assertFalse(s.isEmpty()); + assertEquals("CURRENT", s.getType()); // Valid current token - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("CURRENT-abc:" + e)); - assertEquals("abc", s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("CURRENT-abc:" + e)); + assertEquals("abc", s.token); + assertEquals("CURRENT", s.getType()); + assertFalse(s.isExpired()); + assertTrue(s.isTaken()); // Current expired, first refresh valid - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("REFRESH-abc:" + e)); - assertEquals("abc", s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("REFRESH-abc:" + e)); + assertEquals("abc", s.token); + assertEquals("REFRESH", s.getType()); + assertFalse(s.isExpired()); + assertTrue(s.isTaken()); // Expired refresh - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("REFRESH-abc:200")); - assertNull(s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("REFRESH-abc:200")); + assertEquals("abc", s.token); + assertEquals("REFRESH", s.getType()); + assertTrue(s.isExpired()); + assertFalse(s.isTaken()); } } \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java new file mode 100644 index 00000000000..ae3ddabd3fb --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java @@ -0,0 +1,119 @@ +package org.opencb.opencga.storage.hadoop.variant; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.ExternalResource; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.hadoop.HBaseCompat; +import org.opencb.opencga.storage.hadoop.utils.HBaseManager; +import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; + +import static org.junit.Assert.assertEquals; + +@Category(LongTests.class) +public class AutoScaleHBaseTableTest extends VariantStorageBaseTest implements HadoopVariantStorageTest { + + @ClassRule + public static ExternalResource externalResource = new HadoopExternalResource(); + + private VariantHadoopDBAdaptor dbAdaptor; + private SampleIndexDBAdaptor sampleIndexDBAdaptor; + private HadoopVariantStorageEngine engine; + + @Before + public void before() throws Exception { + clearDB(DB_NAME); + engine = getVariantStorageEngine(); + dbAdaptor = engine.getDBAdaptor(); + sampleIndexDBAdaptor = engine.getSampleIndexDBAdaptor(); + + } + + @Test + public void testAutoScaleTables() throws Exception { + + int archiveSplitsPerBatch = 10; + int samplesPerSplit = 2; + int extraSplits = 3; + + ObjectMap params = new ObjectMap() + .append(VariantStorageOptions.STUDY.key(), STUDY_NAME) + .append(VariantStorageOptions.ANNOTATE.key(), false) + .append(VariantStorageOptions.STATS_CALCULATE.key(), false) + .append(HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), samplesPerSplit) + .append(HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), extraSplits) + .append(HadoopVariantStorageOptions.ARCHIVE_TABLE_PRESPLIT_SIZE.key(), archiveSplitsPerBatch) + .append(HadoopVariantStorageOptions.ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.key(), extraSplits) + .append(HadoopVariantStorageOptions.ARCHIVE_FILE_BATCH_SIZE.key(), 2) + .append(HadoopVariantStorageOptions.EXPECTED_SAMPLES_NUMBER.key(), 1) + .append(HadoopVariantStorageOptions.EXPECTED_FILES_NUMBER.key(), 1); + + + // -- Batch 1 + int batches = 1; + runETL(engine, getPlatinumFile(1), outputUri, params); + // Each batch starts with one extra split, expect the first batch. So, -1 + // Then, a fixed number of extra splits + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 2 + // First batch has 1 fewer elements than the rest of the batches. + batches = 2; + runETL(engine, getPlatinumFile(2), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + runETL(engine, getPlatinumFile(3), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 3 + batches = 3; + runETL(engine, getPlatinumFile(4), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + runETL(engine, getPlatinumFile(5), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 4 + batches = 4; + runETL(engine, getPlatinumFile(6), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + +// VariantHbaseTestUtils.printVariants(dbAdaptor, newOutputUri()); + } + + private void checkArchiveTableSplits(int expectedSplits) throws Exception { + int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); + + String archiveTableName = engine.getArchiveTableName(studyId); + HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); + int archiveNumRegions = hBaseManager.act(archiveTableName, + (table, admin) -> HBaseCompat.getInstance().getTableStartKeys(admin, table).length); + // numRegions == numSplits + 1 + assertEquals(archiveTableName, expectedSplits + 1, archiveNumRegions); + } + + private void checkSampleIndexTableSplits(int expectedSplits) throws Exception { + int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); + + String sampleIndexTableName = sampleIndexDBAdaptor.getSampleIndexTableName(studyId, 1); + HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); + int sampleIndexNumRegions = hBaseManager.act(sampleIndexTableName, + (table, admin) -> HBaseCompat.getInstance().getTableStartKeys(admin, table).length); + // numRegions == numSplits + 1 + assertEquals(sampleIndexTableName, expectedSplits + 1, sampleIndexNumRegions); + } + + +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java index aea720d356a..b613df935ba 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java @@ -19,7 +19,7 @@ public class HadoopVariantStorageEngineBNDTest extends VariantStorageEngineBNDTe @Override protected void loadFiles() throws Exception { super.loadFiles(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); + VariantHbaseTestUtils.printVariants(((HadoopVariantStorageEngine) variantStorageEngine).getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java index fbcbd773477..26bcb49bb0f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java @@ -1,8 +1,6 @@ package org.opencb.opencga.storage.hadoop.variant; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.*; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; @@ -12,13 +10,15 @@ import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import static org.opencb.opencga.core.api.ParamConstants.OVERWRITE; /** @@ -33,9 +33,46 @@ public class HadoopVariantStorageEngineSVTest extends VariantStorageEngineSVTest public static HadoopExternalResource externalResource = new HadoopExternalResource(); private Logger logger = LoggerFactory.getLogger(getClass()); + public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); + + @BeforeClass + public static void beforeClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.before(); + solr.configure(externalResource.getVariantStorageEngine()); + System.out.println("Start embedded solr"); + } else { + System.out.println("Skip embedded solr tests"); + } + } + + @AfterClass + public static void afterClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.after(); + } + } + + @Override + public void before() throws Exception { + super.before(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } + } + @Override protected void loadFiles() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } super.loadFiles(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + variantStorageEngine.secondaryIndex(); + assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } else { + assertFalse(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java index d11635abe6d..cf4d7984ada 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java @@ -232,8 +232,8 @@ public void before() throws Exception { Configurator.setLevel(MapTask.class.getName(), Level.WARN); Configurator.setLevel(TableInputFormatBase.class.getName(), Level.WARN); - utility.set(new HBaseTestingUtility()); - Configuration conf = utility.get().getConfiguration(); + HBaseTestingUtility testingUtility = new HBaseTestingUtility(); + Configuration conf = testingUtility.getConfiguration(); HadoopVariantStorageTest.configuration.set(conf); @@ -277,7 +277,8 @@ public void before() throws Exception { } //org.apache.commons.configuration2.Configuration - utility.get().startMiniCluster(1); + testingUtility.startMiniCluster(1); + utility.set(testingUtility); // MiniMRCluster miniMRCluster = utility.startMiniMapReduceCluster(); // MiniMRClientCluster miniMRClientCluster = MiniMRClientClusterFactory.create(HadoopVariantStorageManagerTestUtils.class, 1, configuration); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java index d609e769f0d..8cead56116a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java @@ -119,6 +119,7 @@ public void before() throws Exception { e.printStackTrace(); } } + variantStorageEngine.getOptions().append(VariantStorageOptions.ASSEMBLY.key(), "GRCH38"); cellBaseUtils = variantStorageEngine.getCellBaseUtils(); expectedConnections = GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getValue(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java index 05b6d81a5b3..5f514f7483c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java @@ -129,7 +129,7 @@ public byte[] generateVariantRowKeyPhoenix(Variant variant) { ImmutableBytesWritable key = new ImmutableBytesWritable(); String reference = variant.getReference(); - String alternate = VariantPhoenixKeyFactory.buildSymbolicAlternate(reference, variant.getAlternate(), variant.getEnd(), variant.getSv()); + String alternate = VariantPhoenixKeyFactory.buildSymbolicAlternate(variant); table.newKey(key, new byte[][]{ Bytes.toBytes(variant.getChromosome()), Bytes.toBytes(variant.getStart()), diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java new file mode 100644 index 00000000000..5513bcdcb7a --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java @@ -0,0 +1,59 @@ +package org.opencb.opencga.storage.hadoop.variant.archive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Assert; +import org.junit.Test; +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDeleteHBaseColumnTask.*; + +public class ArchiveDeleteHBaseColumnTaskTest { + + private ArchiveDeleteHBaseColumnTask task; + + @Test + public void testConfigureTask() { + ObjectMap options = new ObjectMap(); + task = new ArchiveDeleteHBaseColumnTask(); + configureTask(options, Arrays.asList(1, 2, 3)); + assertEquals(Arrays.asList(0), options.getAsIntegerList(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX)); + List> regionsToDelete = task.getRegionsToDelete(toConf(options)); + assertEquals(1, regionsToDelete.size()); + + assertArrayEquals(Bytes.toBytes("00000_"), regionsToDelete.get(0).getFirst()); + assertArrayEquals(Bytes.toBytes("00001_"), regionsToDelete.get(0).getSecond()); + } + + @Test + public void testConfigureTaskMultiRegions() { + ObjectMap options = new ObjectMap(); + task = new ArchiveDeleteHBaseColumnTask(); + configureTask(options, Arrays.asList(5300, 6053, 9032)); + assertEquals(Arrays.asList(5, 6, 9), options.getAsIntegerList(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX)); + List> regionsToDelete = task.getRegionsToDelete(toConf(options)); + assertEquals(3, regionsToDelete.size()); + + assertArrayEquals(Bytes.toBytes("00005_"), regionsToDelete.get(0).getFirst()); + assertArrayEquals(Bytes.toBytes("00006_"), regionsToDelete.get(0).getSecond()); + assertArrayEquals(Bytes.toBytes("00006_"), regionsToDelete.get(1).getFirst()); + assertArrayEquals(Bytes.toBytes("00007_"), regionsToDelete.get(1).getSecond()); + assertArrayEquals(Bytes.toBytes("00009_"), regionsToDelete.get(2).getFirst()); + assertArrayEquals(Bytes.toBytes("00010_"), regionsToDelete.get(2).getSecond()); + } + + private static Configuration toConf(ObjectMap options) { + Configuration conf = new Configuration(); + for (String key : options.keySet()) { + conf.set(key, options.getString(key)); + } + return conf; + } + +} \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java index f716adb0110..c1dc22a6bb4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/mr/VariantLocalConflictResolverTest.java @@ -169,7 +169,7 @@ public void resolveConflictIndelCase1() throws Exception { se.setSampleDataKeys(Arrays.asList(GENOTYPE_KEY, GENOTYPE_FILTER_KEY)); se.setSamplesPosition(asMap("S1", 0)); se.addSampleData("S1", Arrays.asList("1/2", "LowGQXHetDel")); - se.getSecondaryAlternates().add(new AlternateCoordinate(null, null, 328, "CTT", "CTTTC", INDEL)); + se.getSecondaryAlternates().add(new AlternateCoordinate(null, 328, null, "CTT", "CTTTC", INDEL)); addAttribute(v1, FILTER, "LowGQXHetDel"); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java index 3d9db73719e..63f579cb99e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java @@ -13,6 +13,7 @@ import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; import org.opencb.biodata.models.variant.protobuf.VcfSliceProtos; import org.opencb.biodata.tools.variant.converters.proto.VariantToVcfSliceConverter; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -44,6 +45,7 @@ public class FillGapsTaskTest { public void setUp() throws Exception { DummyVariantStorageMetadataDBAdaptorFactory.clear(); metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); studyMetadata = metadataManager.createStudy("S"); metadataManager.updateStudyMetadata("S", sm -> { sm.getAttributes().put(VariantStorageOptions.EXTRA_FORMAT_FIELDS.key(), "DP"); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index f67d5734d7c..a6aeba91145 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -63,13 +63,13 @@ public class FamilyIndexTest extends VariantStorageBaseTest implements HadoopVar @Before public void before() throws Exception { + HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.reloadCellbaseConfiguration(); if (!loaded) { - HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); - variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); - variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); - variantStorageEngine.reloadCellbaseConfiguration(); URI outputUri = newOutputUri(); ObjectMap params = new ObjectMap(VariantStorageOptions.ANNOTATE.key(), false) @@ -91,7 +91,7 @@ public void before() throws Exception { variantStorageEngine.annotate(outputUri, new ObjectMap()); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(variantStorageEngine.getDBAdaptor(), newOutputUri(getTestName().getMethodName())); mendelianErrorVariants = new HashSet<>(); deNovoVariants = new HashSet<>(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java index 99cf76e71cc..e9c46d5214f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java @@ -1387,7 +1387,6 @@ public void parseIntergenicTest() { checkIntergenic(null, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT)); checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT) .append(ANNOT_BIOTYPE.key(), "protein_coding")); - // Nonsense combination checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), "intergenic_variant").append(ANNOT_BIOTYPE.key(), "protein_coding")); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index 55d9d98a0f8..98062c27b8f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -1067,7 +1067,11 @@ public void testFamilyIndexQueryCount() { .append(QueryOptions.LIMIT, 10) .append(QueryOptions.COUNT, true)); - System.out.println(result.getResults().stream().map(Variant::getAnnotation).flatMap(v -> v.getConsequenceTypes().stream()).map(ConsequenceType::getGeneName).collect(Collectors.toSet())); + System.out.println(result.getResults().stream() + .map(Variant::getAnnotation) + .flatMap(v -> v.getConsequenceTypes().stream()) + .map(ConsequenceType::getGeneName) + .collect(Collectors.toSet())); result = variantStorageEngine.get( new Query() @@ -1081,7 +1085,11 @@ public void testFamilyIndexQueryCount() { .append(QueryOptions.LIMIT, 10) .append(QueryOptions.COUNT, true)); - System.out.println(result.getResults().stream().map(Variant::getAnnotation).flatMap(v -> v.getConsequenceTypes().stream()).map(ConsequenceType::getGeneName).collect(Collectors.toSet())); + System.out.println(result.getResults().stream() + .map(Variant::getAnnotation) + .flatMap(v -> v.getConsequenceTypes().stream()) + .map(ConsequenceType::getGeneName) + .collect(Collectors.toSet())); } @Test diff --git a/pom.xml b/pom.xml index bd9d0af08fb..911c6ab2dcd 100644 --- a/pom.xml +++ b/pom.xml @@ -43,14 +43,13 @@ - 4.0.0_dev 4.0.0_dev 7.0.0-SNAPSHOT 4.0.0-SNAPSHOT 6.0.0-SNAPSHOT 4.0.0-SNAPSHOT - 0.2.0 + 0.2.0 2.14.3 2.30.1 @@ -154,7 +153,6 @@ --> true 2.23.0 - 1.18.30 3.4.0 @@ -1369,7 +1367,7 @@ opencga LOCAL - 5 + 5 https://ws.opencb.org/opencga-prod